Files
paperclip/server/src/__tests__/issue-recovery-actions.test.ts
T
Dotta c91a062326 [codex] Runtime control-plane fixes (#6380)
## Thinking Path

> - Paperclip orchestrates AI agents through a server-side control plane
> - That control plane depends on reliable issue state transitions,
plugin lifecycle behavior, import limits, and startup/shutdown handling
> - Several small runtime fixes had accumulated on the working branch
and were mixed with larger feature work
> - Keeping them separate makes the correctness fixes reviewable and
mergeable without waiting for cloud-sync UI work
> - This pull request groups the server/runtime control-plane fixes into
one standalone branch
> - The benefit is a tighter, safer runtime baseline for retries,
imports, plugin migrations, feedback flushing, and trusted cloud import
handling

## What Changed

- Fixed updated issue list pagination sorting and scheduled retry
comment handling.
- Re-applied pending plugin migrations during hot reload and fixed
plugin-schema worktree seed restore.
- Hardened public tenant DB startup, portable import body limits,
trusted cloud import errors, and trusted cloud tenant import mutation
access.
- Expired stale request confirmations after user comments.
- Added feedback export shutdown hardening so database-unavailable flush
loops stop cleanly.
- Guarded plugin worker `error` event emission when no listener is
registered.

## Verification

- `pnpm install --frozen-lockfile --ignore-scripts`
- `pnpm --filter @paperclipai/plugin-sdk build`
- `npm run install --prefix
node_modules/.pnpm/sqlite3@5.1.7/node_modules/sqlite3`
- `pnpm exec vitest run server/src/__tests__/issues-service.test.ts
server/src/__tests__/plugin-lifecycle-restart.test.ts
server/src/__tests__/server-startup-feedback-export.test.ts
server/src/__tests__/issue-comment-reopen-routes.test.ts
server/src/__tests__/issue-thread-interactions-service.test.ts
server/src/__tests__/issue-thread-interaction-routes.test.ts
server/src/__tests__/body-limits.test.ts
server/src/__tests__/feedback-flush-controller.test.ts
server/src/__tests__/error-handler.test.ts
server/src/__tests__/board-mutation-guard.test.ts
packages/db/src/backup-lib.test.ts` initially exposed local setup issues
and two 5s test timeouts.
- Rerun after local prereq build: `pnpm exec vitest run --testTimeout
15000 server/src/__tests__/issue-comment-reopen-routes.test.ts
server/src/__tests__/issue-thread-interaction-routes.test.ts
server/src/__tests__/feedback-flush-controller.test.ts
server/src/__tests__/server-startup-feedback-export.test.ts` passed.
- Some embedded Postgres-backed tests skipped on this host because local
Postgres init was unavailable.

## Risks

- Runtime-touching branch: startup/shutdown and issue interaction
behavior should be reviewed carefully.
- The feedback export change disables repeated flush attempts only for
database connection-refused failures; other upload failures still log
normally.
- The plugin worker error guard avoids process crashes from unhandled
EventEmitter errors but may hide errors from code paths that expected an
emitted listener.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI Codex, GPT-5-based coding agent with local shell/git/tool use.
Exact hosted model ID and context-window size are not exposed by the
local Paperclip adapter runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-20 10:37:11 -05:00

1181 lines
40 KiB
TypeScript

import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
agents,
agentWakeupRequests,
activityLog,
companies,
createDb,
environmentLeases,
environments,
heartbeatRuns,
issueComments,
issueRecoveryActions,
issueRelations,
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { errorHandler } from "../middleware/index.js";
import { issueRoutes } from "../routes/issues.js";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { recoveryService } from "../services/recovery/service.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
function makeRecoveryActionRow(overrides: Record<string, unknown> = {}) {
const now = new Date("2026-05-09T19:30:00.000Z");
return {
id: randomUUID(),
companyId: "company-1",
sourceIssueId: "source-1",
recoveryIssueId: null,
kind: "missing_disposition",
status: "active",
ownerType: "agent",
ownerAgentId: "agent-1",
ownerUserId: null,
previousOwnerAgentId: null,
returnOwnerAgentId: null,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: {},
nextAction: "Choose a valid issue disposition.",
wakePolicy: null,
monitorPolicy: null,
attemptCount: 1,
maxAttempts: null,
timeoutAt: null,
lastAttemptAt: now,
outcome: null,
resolutionNote: null,
resolvedAt: null,
createdAt: now,
updatedAt: now,
...overrides,
};
}
describe("issueRecoveryActionService", () => {
it("does not reactivate an action resolved between the active read and update", async () => {
const existingRow = makeRecoveryActionRow({ id: "existing-action", attemptCount: 1 });
const createdRow = makeRecoveryActionRow({ id: "new-action", attemptCount: 1 });
const selectResults = [[existingRow], []];
const makeSelectQuery = (rows: unknown[]) => ({
from() {
return this;
},
where() {
return this;
},
orderBy() {
return this;
},
limit() {
return Promise.resolve(rows);
},
});
const fakeDb = {
select: vi.fn(() => makeSelectQuery(selectResults.shift() ?? [])),
update: vi.fn(() => ({
set: vi.fn(() => ({
where: vi.fn(() => ({
returning: vi.fn(async () => []),
})),
})),
})),
insert: vi.fn(() => ({
values: vi.fn(() => ({
returning: vi.fn(async () => [createdRow]),
})),
})),
};
const result = await issueRecoveryActionService(fakeDb as never).upsertSourceScoped({
companyId: "company-1",
sourceIssueId: "source-1",
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: "agent-1",
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
nextAction: "Choose a valid issue disposition.",
});
expect(result).toMatchObject({ id: "new-action", status: "active" });
expect(fakeDb.update).toHaveBeenCalledTimes(1);
expect(fakeDb.insert).toHaveBeenCalledTimes(1);
});
});
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres issue recovery action tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("issue recovery actions", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let db: ReturnType<typeof createDb>;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-");
db = createDb(tempDb.connectionString);
}, 30_000);
afterEach(async () => {
await db.delete(issueRecoveryActions);
await db.delete(issueComments);
await db.delete(environmentLeases);
await db.delete(activityLog);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(environments);
await db.delete(issues);
await db.delete(agents);
await db.delete(companies);
});
afterAll(async () => {
await tempDb?.cleanup();
});
async function seedCompany() {
const companyId = randomUUID();
const managerId = randomUUID();
const coderId = randomUUID();
const sourceIssueId = randomUUID();
const prefix = `RA${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`;
await db.insert(companies).values({
id: companyId,
name: "Recovery Co",
issuePrefix: prefix,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
{
id: managerId,
companyId,
name: "CTO",
role: "cto",
status: "idle",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
},
{
id: coderId,
companyId,
name: "Coder",
role: "engineer",
status: "idle",
reportsTo: managerId,
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
},
]);
await db.insert(issues).values({
id: sourceIssueId,
companyId,
title: "Implement backend recovery",
status: "in_progress",
priority: "medium",
assigneeAgentId: coderId,
issueNumber: 1,
identifier: `${prefix}-1`,
});
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
return { companyId, managerId, coderId, sourceIssueId, prefix, sourceIssue: sourceIssue! };
}
async function seedHeartbeatRun(input: {
companyId: string;
agentId: string;
runId: string;
issueId?: string;
status?: string;
}) {
await db.insert(heartbeatRuns).values({
id: input.runId,
companyId: input.companyId,
agentId: input.agentId,
invocationSource: "manual",
status: input.status ?? "running",
startedAt: new Date("2026-05-13T18:00:00.000Z"),
contextSnapshot: input.issueId ? { issueId: input.issueId } : undefined,
});
}
function createApp(actor: any = { type: "board", source: "local_implicit" }) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
(req as any).actor = actor;
next();
});
app.use("/api", issueRoutes(db, {} as any));
app.use(errorHandler);
return app;
}
it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const svc = issueRecoveryActionService(db);
const first = await svc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "stranded_assigned_issue",
ownerType: "agent",
ownerAgentId: managerId,
cause: "stranded_assigned_issue",
fingerprint: "recovery:fingerprint",
evidence: { latestRunId: "run-1" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "wake_owner" },
});
const second = await svc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "stranded_assigned_issue",
ownerType: "agent",
ownerAgentId: managerId,
cause: "stranded_assigned_issue",
fingerprint: "recovery:fingerprint",
evidence: { latestRunId: "run-2" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "wake_owner" },
});
expect(second.id).toBe(first.id);
expect(second.attemptCount).toBe(2);
expect(second.evidence).toMatchObject({ latestRunId: "run-2" });
expect(await svc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({ id: first.id });
expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull();
});
it("escalates stranded assigned work into a source action instead of a recovery issue", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
const enqueueWakeup = vi.fn(async () => null);
const recovery = recoveryService(db, { enqueueWakeup });
const latestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun,
comment: "Automatic continuation recovery failed.",
});
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(updatedIssue).toMatchObject({
status: "blocked",
});
const recoveryIssues = await db
.select()
.from(issues)
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
expect(recoveryIssues).toHaveLength(0);
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({
issueId: sourceIssue.id,
sourceIssueId: sourceIssue.id,
recoveryCause: "stranded_assigned_issue",
});
});
it("reuses the same source-scoped action when latest run IDs change while the cause stays the same", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
const enqueueWakeup = vi.fn(async () => null);
const recovery = recoveryService(db, { enqueueWakeup });
const firstLatestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
const secondLatestRun = {
...firstLatestRun,
id: randomUUID(),
};
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: firstLatestRun,
comment: "Automatic continuation recovery failed.",
});
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: secondLatestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
expect(actionRows[0]?.evidence).toMatchObject({ latestRunId: secondLatestRun.id });
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
expect(enqueueWakeup.mock.calls[1]?.[1]?.payload).toMatchObject({
issueId: sourceIssue.id,
sourceIssueId: sourceIssue.id,
strandedRunId: secondLatestRun.id,
recoveryCause: "stranded_assigned_issue",
});
});
it("keeps the source issue blocked when source-scoped wakeup is claimed synchronously", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, managerId));
const enqueueWakeup = vi.fn(async () => {
await db
.update(issues)
.set({ status: "in_progress" })
.where(eq(issues.id, sourceIssue.id));
return null;
});
const recovery = recoveryService(db, { enqueueWakeup });
const firstLatestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: firstLatestRun,
comment: "Automatic continuation recovery failed.",
});
const [afterFirst] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(afterFirst?.status).toBe("blocked");
expect(afterFirst?.assigneeAgentId).toBe(coderId);
const secondLatestRun = {
...firstLatestRun,
id: randomUUID(),
};
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: secondLatestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
const [afterSecond] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(afterSecond?.status).toBe("blocked");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("Recovery action:");
});
it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => {
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
const recoveryIssueId = randomUUID();
await db.insert(issues).values({
id: recoveryIssueId,
companyId,
title: "Recover stalled issue",
status: "in_progress",
priority: "medium",
assigneeAgentId: managerId,
parentId: sourceIssueId,
issueNumber: 2,
identifier: `${prefix}-2`,
originKind: "stranded_issue_recovery",
originId: sourceIssueId,
originFingerprint: `stranded_issue_recovery:${sourceIssueId}`,
});
const [recoveryIssue] = await db.select().from(issues).where(eq(issues.id, recoveryIssueId));
const recovery = recoveryService(db, { enqueueWakeup: vi.fn(async () => null) });
await recovery.escalateStrandedAssignedIssue({
issue: recoveryIssue!,
previousStatus: "in_progress",
latestRun: {
id: randomUUID(),
agentId: managerId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
},
});
const actionRows = await db.select().from(issueRecoveryActions);
expect(actionRows).toHaveLength(0);
const recoveryIssues = await db
.select()
.from(issues)
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
expect(recoveryIssues).toHaveLength(1);
expect(recoveryIssues[0]?.status).toBe("blocked");
});
it("exposes active recovery actions on the issue read API", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toMatchObject({
id: action.id,
sourceIssueId,
kind: "missing_disposition",
ownerAgentId: managerId,
});
const list = await request(app).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);
expect(list.body.active).toMatchObject({ id: action.id });
expect(list.body.actions).toHaveLength(1);
});
it("resolves an active recovery action and removes it from active projections", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
resolutionNote: "Operator confirmed the source issue is complete.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "done",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "restored",
resolutionNote: "Operator confirmed the source issue is complete.",
});
expect(resolved.body.recoveryAction.resolvedAt).toBeTruthy();
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toBeNull();
const activityRows = await db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, sourceIssueId));
expect(activityRows.map((row) => row.action)).toEqual(
expect.arrayContaining(["issue.updated", "issue.recovery_action_resolved"]),
);
});
it("resolves an active recovery action by returning the source issue to todo", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:try-again",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "todo",
resolutionNote: "Try the source issue again.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "todo",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "restored",
resolutionNote: "Try the source issue again.",
});
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
});
it("marks a recovery action stale when a blocked source issue is manually moved to todo", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:manual-restore",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const patched = await request(app)
.patch(`/api/issues/${sourceIssueId}`)
.send({ status: "todo" })
.expect(200);
expect(patched.body).toMatchObject({
id: sourceIssueId,
status: "todo",
activeRecoveryAction: null,
});
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "cancelled",
outcome: "cancelled",
resolutionNote: "Recovery action became stale because the source issue was manually moved from blocked to todo.",
});
expect(actionRow?.resolvedAt).toBeTruthy();
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toBeNull();
const activityRows = await db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, sourceIssueId));
expect(activityRows.map((row) => row.action)).toEqual(
expect.arrayContaining(["issue.updated", "issue.recovery_action_resolved"]),
);
expect(activityRows.find((row) => row.action === "issue.recovery_action_resolved")?.details).toMatchObject({
source: "source_revalidation",
trigger: "issue_update",
});
});
it("folds stale recovery during read projection after the source issue reaches done", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:done-projection",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
await db.update(issues).set({ status: "done" }).where(eq(issues.id, sourceIssueId));
const app = createApp();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body).toMatchObject({
id: sourceIssueId,
status: "done",
activeRecoveryAction: null,
});
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "cancelled",
outcome: "cancelled",
resolutionNote: "Recovery action became stale because the source issue reached done.",
});
expect(actionRow?.resolvedAt).toBeTruthy();
const activityRows = await db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, sourceIssueId));
expect(activityRows.find((row) => row.action === "issue.recovery_action_resolved")?.details).toMatchObject({
source: "source_revalidation",
trigger: "read_projection",
recoveryActionId: action.id,
});
});
it("keeps active recovery visible when a plain comment does not create a live path", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:plain-comment",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
await request(app)
.post(`/api/issues/${sourceIssueId}/comments`)
.send({ body: "I am looking at this, but not changing the disposition." })
.expect(201);
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({
id: action.id,
status: "active",
});
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toMatchObject({ id: action.id });
});
it("folds stale recovery when a structured resume comment restores todo dispatch", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:resume-comment",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
await request(app)
.post(`/api/issues/${sourceIssueId}/comments`)
.send({ body: "Resume this now.", resume: true })
.expect(201);
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("todo");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "cancelled",
outcome: "cancelled",
resolutionNote: "Recovery action became stale because the source issue was manually moved from blocked to todo.",
});
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
const activityRows = await db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, sourceIssueId));
expect(activityRows.find((row) => row.action === "issue.recovery_action_resolved")?.details).toMatchObject({
source: "source_revalidation",
trigger: "comment",
recoveryActionId: action.id,
});
});
it("rejects peer-agent source issue updates that would hide another owner's recovery action", async () => {
const { companyId, managerId, coderId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:peer-status-update",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp({
type: "agent",
agentId: coderId,
companyId,
runId: randomUUID(),
source: "agent_jwt",
});
await request(app)
.patch(`/api/issues/${sourceIssueId}`)
.send({ status: "todo" })
.expect(403);
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("blocked");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolvedAt: null,
});
});
it("rejects peer-agent recovery action resolution on a board-owned source issue", async () => {
const { companyId, managerId, coderId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:peer-resolution",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const app = createApp({
type: "agent",
agentId: coderId,
companyId,
runId: randomUUID(),
source: "agent_jwt",
});
await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
resolutionNote: "Peer agent should not be able to clear this recovery.",
})
.expect(403);
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("blocked");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolvedAt: null,
});
});
it("allows the named recovery owner to resolve a board-owned source recovery action", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db
.update(issues)
.set({ status: "blocked", assigneeAgentId: null, assigneeUserId: "board-user" })
.where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:owner-resolution",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "manual" },
});
const runId = randomUUID();
const app = createApp({
type: "agent",
agentId: managerId,
companyId,
runId,
source: "agent_jwt",
});
await seedHeartbeatRun({
companyId,
agentId: managerId,
runId,
issueId: sourceIssueId,
});
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
resolutionNote: "Recovery owner verified the work was intentionally completed.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "done",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "restored",
});
});
it("rejects blocked recovery resolution when the source issue has no first-class blockers", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:blocked-without-blocker",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Choose a disposition with a live continuation path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const rejected = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "blocked",
sourceIssueStatus: "blocked",
})
.expect(422);
expect(rejected.body.error).toContain("requires an unresolved first-class blocker");
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("in_progress");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolvedAt: null,
});
});
it("allows blocked recovery resolution when the source issue has an unresolved first-class blocker", async () => {
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
const blockerIssueId = randomUUID();
await db.insert(issues).values({
id: blockerIssueId,
companyId,
title: "Unblock recovery disposition",
status: "todo",
priority: "medium",
assigneeAgentId: managerId,
issueNumber: 2,
identifier: `${prefix}-2`,
});
await db.insert(issueRelations).values({
companyId,
issueId: blockerIssueId,
relatedIssueId: sourceIssueId,
type: "blocks",
});
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:blocked-with-blocker",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Wait for the blocker before continuing.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "blocked",
sourceIssueStatus: "blocked",
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "blocked",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "blocked",
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
});
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
});
it("rejects false-positive recovery resolution without an explicit source issue status", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:fingerprint",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Confirm whether the issue is actually stranded.",
wakePolicy: { type: "manual" },
});
const app = createApp();
await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "false_positive",
resolutionNote: "The source issue still has a live execution path.",
})
.expect(400);
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("in_progress");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolutionNote: null,
});
});
it("allows false-positive recovery resolution to restore a blocked source issue in the same request", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:false-positive-unblock",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Confirm whether the issue is actually stranded.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "false_positive",
sourceIssueStatus: "in_review",
resolutionNote: "Recovery signal was stale; return to review.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "in_review",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "false_positive",
resolutionNote: "Recovery signal was stale; return to review.",
});
});
it("enforces company scope when resolving recovery actions", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp({
type: "agent",
agentId: randomUUID(),
companyId: randomUUID(),
runId: randomUUID(),
source: "agent_jwt",
});
await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
})
.expect(403);
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow?.status).toBe("active");
});
});