[codex] Add configurable liveness auto-recovery controls (#4587)

## Thinking Path

> - Paperclip orchestrates AI agents for zero-human companies.
> - Heartbeat liveness recovery decides when stalled issue trees need
manager-visible follow-up.
> - Automatic recovery issue creation is useful, but operators need
instance-level controls for how aggressive it is.
> - Without controls, recovery behavior is harder to tune for local
development, production operations, and noisy edge cases.
> - This pull request adds configurable liveness auto-recovery settings
across shared contracts, API routes, services, and the instance
experimental settings UI.
> - The benefit is that operators can keep liveness findings advisory or
enable bounded recovery automation with explicit intervals and lookback
windows.

## What Changed

- Added shared types and validators for liveness auto-recovery settings.
- Extended instance settings routes and services to persist and validate
the new controls.
- Wired heartbeat/recovery services to honor enablement, minimum
interval, and lookback settings.
- Added UI controls for liveness recovery under instance experimental
settings.
- Covered the new server behavior with instance settings and liveness
escalation tests.

## Verification

- `pnpm exec vitest run --project @paperclipai/server
server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts
server/src/__tests__/instance-settings-routes.test.ts --pool=forks
--poolOptions.forks.isolate=true`
- `pnpm --filter @paperclipai/shared typecheck`
- `pnpm --filter @paperclipai/server typecheck`
- `pnpm --filter @paperclipai/ui typecheck`

## Risks

- Moderate behavioral risk because recovery automation timing changes
when enabled; defaults keep existing advisory behavior unless the
setting is turned on.
- No database migration in this PR; settings are stored through the
existing instance settings path.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI Codex, `gpt-5`, coding model with tool use and local command
execution; context window not exposed by the runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
2026-04-27 08:46:44 -05:00
committed by GitHub
parent f0f9460d1d
commit fda296ee4f
14 changed files with 679 additions and 54 deletions
@@ -103,6 +103,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
await instanceSettingsService(db).updateExperimental({
enableIssueGraphLivenessAutoRecovery: false,
enableIsolatedWorkspaces: false,
issueGraphLivenessAutoRecoveryLookbackHours: 24,
});
});
@@ -116,7 +117,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
});
}
async function seedBlockedChain(opts: { stale?: boolean } = {}) {
async function seedBlockedChain(opts: { outsideLookback?: boolean } = {}) {
const companyId = randomUUID();
const managerId = randomUUID();
const coderId = randomUUID();
@@ -157,9 +158,9 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
},
]);
const issueTimestamp = opts.stale === false
? new Date()
: new Date(Date.now() - 25 * 60 * 60 * 1000);
const issueTimestamp = opts.outsideLookback === true
? new Date(Date.now() - 25 * 60 * 60 * 1000)
: new Date(Date.now() - 60 * 60 * 1000);
await db.insert(issues).values([
{
id: blockedIssueId,
@@ -197,6 +198,9 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
}
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
await instanceSettingsService(db).updateExperimental({
enableIssueGraphLivenessAutoRecovery: false,
});
const { companyId } = await seedBlockedChain();
const heartbeat = heartbeatService(db);
@@ -214,16 +218,16 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
expect(escalations).toHaveLength(0);
});
it("does not create recovery issues until the dependency path is stale for 24 hours", async () => {
it("does not create recovery issues outside the configured lookback window", async () => {
await enableAutoRecovery();
const { companyId } = await seedBlockedChain({ stale: false });
const { companyId } = await seedBlockedChain({ outsideLookback: true });
const heartbeat = heartbeatService(db);
const result = await heartbeat.reconcileIssueGraphLiveness();
expect(result.findings).toBe(1);
expect(result.escalationsCreated).toBe(0);
expect(result.skippedAutoRecoveryTooYoung).toBe(1);
expect(result.skippedOutsideLookback).toBe(1);
const escalations = await db
.select()
@@ -424,7 +428,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
const dependentExecutionWorkspaceId = randomUUID();
const blockerExecutionWorkspaceId = randomUUID();
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
const issueTimestamp = new Date(Date.now() - 60 * 60 * 1000);
await db.insert(companies).values({
id: companyId,
@@ -559,7 +563,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
const { companyId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
const secondBlockedIssueId = randomUUID();
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
const issueTimestamp = new Date(Date.now() - 60 * 60 * 1000);
await db.insert(issues).values({
id: secondBlockedIssueId,
companyId,