Merge upstream/master into dev (13 commits — includes #5922, #5938, blocked inbox, recovery actions)

This commit is contained in:
2026-05-13 22:35:18 -04:00
180 changed files with 31626 additions and 545 deletions
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { runChildProcess } from "@paperclipai/adapter-utils/server-utils";
import { execute } from "@paperclipai/adapter-claude-local/server";
import { claudeSessionCwdMatchesExecutionTarget, execute } from "@paperclipai/adapter-claude-local/server";
async function writeFailingClaudeCommand(
commandPath: string,
@@ -580,7 +580,7 @@ describe("claude execute", () => {
const remoteWorkspace = path.join(root, "sandbox-$HOME");
const binDir = path.join(root, "bin");
const commandPath = path.join(binDir, "claude");
const capturePath = path.join(remoteWorkspace, "capture.json");
const capturePath1 = path.join(remoteWorkspace, "capture-1.json");
const claudeRoot = path.join(root, ".claude");
const previousHome = process.env.HOME;
const previousPath = process.env.PATH;
@@ -615,7 +615,7 @@ describe("claude execute", () => {
command: commandPath,
cwd: localWorkspace,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath,
PAPERCLIP_TEST_CAPTURE_PATH: capturePath1,
},
promptTemplate: "Follow the paperclip heartbeat.",
},
@@ -635,7 +635,17 @@ describe("claude execute", () => {
});
expect(result.exitCode).toBe(0);
const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload;
expect(result.sessionParams).toMatchObject({
cwd: localWorkspace,
remoteExecution: {
transport: "sandbox",
providerKey: "e2b",
environmentId: "env-1",
leaseId: "lease-1",
remoteCwd: remoteWorkspace,
},
});
const capture = JSON.parse(await fs.readFile(capturePath1, "utf8")) as CapturePayload;
expect(capture.argv).toContain("--allowedTools");
expect(capture.argv).toContain(
"Task AskUserQuestion Bash(*) CronCreate CronDelete CronList Edit EnterPlanMode EnterWorktree ExitPlanMode ExitWorktree Glob Grep Monitor NotebookEdit PushNotification Read RemoteTrigger ScheduleWakeup Skill TaskOutput TaskStop TodoWrite ToolSearch WebFetch WebSearch Write",
@@ -655,6 +665,19 @@ describe("claude execute", () => {
}
}, 10_000);
it("allows remote session resumes when saved cwd is the host workspace", () => {
expect(claudeSessionCwdMatchesExecutionTarget({
runtimeSessionCwd: "/host/workspace",
effectiveExecutionCwd: "/remote/workspace",
executionTargetIsRemote: true,
})).toBe(true);
expect(claudeSessionCwdMatchesExecutionTarget({
runtimeSessionCwd: "/host/workspace",
effectiveExecutionCwd: "/remote/workspace",
executionTargetIsRemote: false,
})).toBe(false);
});
it("reuses a stable Paperclip-managed Claude prompt bundle across equivalent runs", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-bundle-"));
const workspace = path.join(root, "workspace");
@@ -80,6 +80,10 @@ vi.mock("../services/index.js", () => ({
listApprovalsForIssue: vi.fn(),
unlink: vi.fn(),
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
documentService: () => ({}),
routineService: () => ({}),
workProductService: () => ({}),
@@ -20,6 +20,7 @@ import {
heartbeatRuns,
issueComments,
issueDocuments,
issueRecoveryActions,
issueRelations,
issueTreeHoldMembers,
issueTreeHolds,
@@ -328,6 +329,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await db.delete(documentRevisions);
await db.delete(documents);
await db.delete(issueRelations);
await db.delete(issueRecoveryActions);
await db.delete(issueTreeHoldMembers);
await db.delete(issueTreeHolds);
for (let attempt = 0; attempt < 5; attempt += 1) {
@@ -692,67 +694,76 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, issueId };
}
async function expectStrandedRecoveryArtifacts(input: {
async function expectSourceScopedStrandedRecoveryAction(input: {
companyId: string;
agentId: string;
issueId: string;
runId: string;
previousStatus: "todo" | "in_progress";
retryReason: "assignment_recovery" | "issue_continuation_needed";
retryReason?: "assignment_recovery" | "issue_continuation_needed" | null;
cause?: string;
kind?: string;
}) {
const recovery = await waitForValue(async () =>
db.select().from(issues).where(
const action = await waitForValue(async () =>
db.select().from(issueRecoveryActions).where(
and(
eq(issues.companyId, input.companyId),
eq(issues.originKind, "stranded_issue_recovery"),
eq(issues.originId, input.issueId),
eq(issueRecoveryActions.companyId, input.companyId),
eq(issueRecoveryActions.sourceIssueId, input.issueId),
),
).then((rows) => rows[0] ?? null),
);
if (!recovery) throw new Error("Expected stranded issue recovery issue to be created");
if (!action) throw new Error("Expected source-scoped stranded recovery action to be created");
expect(recovery).toMatchObject({
expect(action).toMatchObject({
companyId: input.companyId,
parentId: input.issueId,
assigneeAgentId: input.agentId,
originKind: "stranded_issue_recovery",
originId: input.issueId,
originRunId: input.runId,
priority: "medium",
assigneeAdapterOverrides: { modelProfile: "cheap" },
sourceIssueId: input.issueId,
recoveryIssueId: null,
kind: input.kind ?? "stranded_assigned_issue",
status: "active",
ownerType: "agent",
ownerAgentId: input.agentId,
previousOwnerAgentId: input.agentId,
returnOwnerAgentId: input.agentId,
cause: input.cause ?? "stranded_assigned_issue",
attemptCount: 1,
maxAttempts: null,
});
expect(recovery.title).toContain("Recover stalled issue");
expect(recovery.description).toContain(`Previous source status: \`${input.previousStatus}\``);
expect(recovery.description).toContain(`Retry reason: \`${input.retryReason}\``);
expect(recovery.description).toContain("Fix the runtime/adapter problem");
expect(action.evidence).toMatchObject({
sourceIssueId: input.issueId,
previousStatus: input.previousStatus,
latestRunId: input.runId,
retryReason: input.retryReason ?? null,
});
expect(action.nextAction).toContain(
input.kind === "missing_disposition" ? "valid issue disposition" : "Restore a live execution path",
);
const relation = await db
const recoveryIssues = await db
.select()
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, input.companyId),
eq(issueRelations.issueId, recovery.id),
eq(issueRelations.relatedIssueId, input.issueId),
eq(issueRelations.type, "blocks"),
),
)
.then((rows) => rows[0] ?? null);
expect(relation).toBeTruthy();
.from(issues)
.where(and(
eq(issues.companyId, input.companyId),
eq(issues.originKind, "stranded_issue_recovery"),
eq(issues.originId, input.issueId),
));
expect(recoveryIssues).toHaveLength(0);
const wakeups = await db
.select()
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.agentId, input.agentId));
const recoveryWakeup = wakeups.find((wakeup) => {
const payload = wakeup.payload as Record<string, unknown> | null;
return payload?.issueId === recovery.id &&
payload?.sourceIssueId === input.issueId &&
payload?.strandedRunId === input.runId;
const recoveryWakeup = await waitForValue(async () => {
const wakeups = await db
.select()
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.agentId, input.agentId));
return wakeups.find((wakeup) => {
const payload = wakeup.payload as Record<string, unknown> | null;
return payload?.issueId === input.issueId &&
payload?.sourceIssueId === input.issueId &&
payload?.recoveryActionId === action.id &&
payload?.strandedRunId === input.runId;
}) ?? null;
});
expect(recoveryWakeup).toMatchObject({
companyId: input.companyId,
reason: "issue_assigned",
reason: "source_scoped_recovery_action",
source: "assignment",
payload: expect.objectContaining({ modelProfile: "cheap" }),
});
@@ -765,15 +776,23 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
.then((rows) => rows[0] ?? null)
: null;
expect(recoveryRun?.contextSnapshot).toMatchObject({
issueId: recovery.id,
taskId: recovery.id,
source: "stranded_issue_recovery",
issueId: input.issueId,
taskId: input.issueId,
source: "issue_recovery_action",
recoveryActionId: action.id,
sourceIssueId: input.issueId,
strandedRunId: input.runId,
modelProfile: "cheap",
});
await waitForHeartbeatIdle(db);
const sourceIssue = await db
.select()
.from(issues)
.where(eq(issues.id, input.issueId))
.then((rows) => rows[0] ?? null);
expect(sourceIssue?.status).toBe("blocked");
return recovery;
return action;
}
async function sourceBlockerIssueIds(companyId: string, sourceIssueId: string) {
@@ -1056,7 +1075,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(blockedIssue?.checkoutRunId).toBeNull();
if (!continuationRun?.id) throw new Error("Expected continuation recovery run to exist");
const recovery = await expectStrandedRecoveryArtifacts({
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
@@ -1065,17 +1084,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
retryReason: "issue_continuation_needed",
});
const blockerRelations = await db
.select()
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, companyId),
eq(issueRelations.relatedIssueId, issueId),
eq(issueRelations.type, "blocks"),
),
);
expect(blockerRelations.map((relation) => relation.issueId)).toEqual([recovery.id]);
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
const comments = await waitForValue(async () => {
const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
@@ -1083,7 +1092,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried continuation");
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
});
it("blocks failed recovery work in place during immediate terminal-run cleanup", async () => {
@@ -1600,27 +1610,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(result.successfulRunHandoffEscalated).toBe(1);
expect(result.issueIds).toEqual([issueId]);
const recovery = await waitForValue(async () =>
db.select().from(issues).where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, "stranded_issue_recovery"),
eq(issues.originId, issueId),
),
).then((rows) => rows[0] ?? null),
);
expect(recovery?.assigneeAgentId).toBe(agentId);
expect(recovery?.title).toContain("Recover missing next step");
expect(recovery?.description).toContain("Normalized cause: `successful_run_missing_state`");
expect(recovery?.description).toContain("not a runtime/adapter crash report");
expect(recovery?.description).toContain(`Source run: [\`${sourceRunId}\`]`);
expect(recovery?.description).toContain("Missing disposition: `clear_next_step`");
expect(recovery?.description).toContain("Source assignee: [CodexCoder]");
expect(recovery?.description).not.toContain("sk-test-successful-handoff-secret");
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
runId,
previousStatus: "in_progress",
retryReason: null,
cause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
kind: "missing_disposition",
});
expect(recoveryAction.evidence).toMatchObject({
sourceRunId,
missingDisposition: "clear_next_step",
latestRunStatus: "failed",
latestRunErrorCode: "adapter_failed",
recoveryCause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
});
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("sk-test-successful-handoff-secret");
const sourceIssue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
expect(sourceIssue?.status).toBe("blocked");
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([recovery?.id]);
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments[0]?.body).toBe(SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY);
@@ -1636,7 +1647,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect.objectContaining({
title: "Recovery owner",
rows: expect.arrayContaining([
expect.objectContaining({ type: "issue_link", identifier: recovery?.identifier }),
expect.objectContaining({ type: "key_value", label: "Recovery action", value: recoveryAction.id }),
expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CodexCoder" }),
]),
}),
@@ -1657,7 +1668,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
it("escalates an exhausted successful handoff run that still leaves no disposition", async () => {
const { companyId, runId, issueId } = await seedStrandedIssueFixture({
const { companyId, agentId, runId, issueId } = await seedStrandedIssueFixture({
status: "in_progress",
runStatus: "succeeded",
livenessState: "advanced",
@@ -1687,17 +1698,21 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(result.successfulContinuationObserved).toBe(0);
expect(result.successfulRunHandoffEscalated).toBe(1);
const recovery = await waitForValue(async () =>
db.select().from(issues).where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, "stranded_issue_recovery"),
eq(issues.originId, issueId),
),
).then((rows) => rows[0] ?? null),
);
expect(recovery?.description).toContain("Latest handoff run status: `succeeded`");
expect(recovery?.description).toContain("Suggested");
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
runId,
previousStatus: "in_progress",
retryReason: null,
cause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
kind: "missing_disposition",
});
expect(recoveryAction.evidence).toMatchObject({
sourceRunId,
latestRunStatus: "succeeded",
missingDisposition: "clear_next_step",
});
});
it("clears the detached warning when the run reports activity again", async () => {
@@ -2063,7 +2078,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
expect(issue?.status).toBe("blocked");
const recovery = await expectStrandedRecoveryArtifacts({
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
@@ -2071,13 +2086,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
previousStatus: "todo",
retryReason: "assignment_recovery",
});
expect(recovery.description ?? "").not.toContain("sk-test-recovery-secret");
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("sk-test-recovery-secret");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried dispatch");
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
});
it("blocks an already stranded recovery issue without creating a recovery child", async () => {
@@ -2457,7 +2473,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
expect(issue?.status).toBe("blocked");
const recovery = await expectStrandedRecoveryArtifacts({
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
@@ -2470,7 +2486,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried continuation");
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
});
it("redacts error-code-only stranded recovery failures in issue copy", async () => {
@@ -2486,7 +2503,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const result = await heartbeat.reconcileStrandedAssignedIssues();
expect(result.escalated).toBe(1);
const recovery = await expectStrandedRecoveryArtifacts({
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
@@ -2494,8 +2511,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
previousStatus: "in_progress",
retryReason: "issue_continuation_needed",
});
expect(recovery.description).toContain("Latest retry failure details were withheld from the issue thread");
expect(recovery.description).not.toContain("- Failure: none recorded");
expect(recoveryAction.evidence).toMatchObject({
latestRunErrorCode: "adapter_exit_code",
});
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("- Failure: none recorded");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments).toHaveLength(1);
@@ -2516,6 +2535,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
);
expect(results.every((result) => result.status === "fulfilled")).toBe(true);
const actions = await db
.select()
.from(issueRecoveryActions)
.where(and(
eq(issueRecoveryActions.companyId, companyId),
eq(issueRecoveryActions.sourceIssueId, issueId),
));
expect(actions).toHaveLength(1);
expect(actions[0]?.attemptCount).toBeGreaterThanOrEqual(1);
const recoveries = await db
.select()
.from(issues)
@@ -2524,8 +2552,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
eq(issues.originKind, "stranded_issue_recovery"),
eq(issues.originId, issueId),
));
expect(recoveries).toHaveLength(1);
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([recoveries[0]?.id]);
expect(recoveries).toHaveLength(0);
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
});
it("blocks stranded recovery issues in place instead of creating nested recovery issues", async () => {
@@ -2783,7 +2811,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
expect(issue?.status).toBe("blocked");
const recovery = await expectStrandedRecoveryArtifacts({
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
companyId,
agentId,
issueId,
@@ -2796,7 +2824,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("automatically retried continuation");
expect(comments[0]?.body).toContain("still has no live execution path");
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
});
it("allows one productive-terminal recovery after regular continuation recovery made progress", async () => {
@@ -89,6 +89,10 @@ function registerModuleMocks() {
heartbeatService: () => mockHeartbeatService,
instanceSettingsService: () => mockInstanceSettingsService,
issueApprovalService: () => ({}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueReferenceService: () => ({
deleteDocumentSource: async () => undefined,
diffIssueReferenceSummary: () => ({
@@ -60,6 +60,9 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
}));
const mockIssueRecoveryActionService = vi.hoisted(() => ({
getActiveForIssue: vi.fn(async () => null),
}));
function registerRouteMocks() {
vi.doMock("@paperclipai/shared/telemetry", () => ({
@@ -124,6 +127,7 @@ function registerRouteMocks() {
listCompanyIds: vi.fn(async () => [companyId]),
}),
issueApprovalService: () => ({}),
issueRecoveryActionService: () => mockIssueRecoveryActionService,
issueReferenceService: () => ({
deleteDocumentSource: async () => undefined,
diffIssueReferenceSummary: () => ({
@@ -259,6 +263,8 @@ describe("agent issue mutation checkout ownership", () => {
mockIssueService.getWakeableParentAfterChildCompletion.mockReset();
mockIssueService.listAttachments.mockReset();
mockIssueService.listWakeableBlockedDependents.mockReset();
mockIssueRecoveryActionService.getActiveForIssue.mockReset();
mockIssueRecoveryActionService.getActiveForIssue.mockResolvedValue(null);
mockIssueService.remove.mockReset();
mockIssueService.removeAttachment.mockReset();
mockIssueService.update.mockReset();
@@ -59,6 +59,10 @@ vi.mock("../services/index.js", () => ({
listCompanyIds: vi.fn(async () => ["company-1"]),
}),
issueApprovalService: () => ({}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueReferenceService: () => ({
deleteDocumentSource: async () => undefined,
diffIssueReferenceSummary: () => ({
@@ -81,6 +81,10 @@ function registerRouteMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
logActivity: mockLogActivity,
projectService: () => ({}),
@@ -2,12 +2,16 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
activityLog,
agents,
agentWakeupRequests,
approvals,
companies,
createDb,
heartbeatRuns,
issueApprovals,
issueRelations,
issueThreadInteractions,
issues,
} from "@paperclipai/db";
import {
@@ -15,6 +19,7 @@ import {
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { issueService } from "../services/issues.js";
import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -37,6 +42,10 @@ describeEmbeddedPostgres("issue blocker attention", () => {
}, 20_000);
afterEach(async () => {
await db.delete(issueThreadInteractions);
await db.delete(issueApprovals);
await db.delete(approvals);
await db.delete(activityLog);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(issueRelations);
@@ -52,20 +61,30 @@ describeEmbeddedPostgres("issue blocker attention", () => {
async function createCompany(prefix = "PBA") {
const companyId = randomUUID();
const agentId = randomUUID();
const pausedAgentId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: `Company ${prefix}`,
issuePrefix: prefix,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
id: agentId,
companyId,
name: `${prefix} Agent`,
role: "engineer",
status: "idle",
});
return { companyId, agentId };
await db.insert(agents).values([
{
id: agentId,
companyId,
name: `${prefix} Agent`,
role: "engineer",
status: "idle",
},
{
id: pausedAgentId,
companyId,
name: `${prefix} Paused`,
role: "engineer",
status: "paused",
},
]);
return { companyId, agentId, pausedAgentId };
}
async function insertIssue(input: {
@@ -80,6 +99,8 @@ describeEmbeddedPostgres("issue blocker attention", () => {
originKind?: string | null;
originId?: string | null;
originFingerprint?: string | null;
executionState?: Record<string, unknown> | null;
description?: string | null;
}) {
const id = input.id ?? randomUUID();
await db.insert(issues).values({
@@ -95,6 +116,8 @@ describeEmbeddedPostgres("issue blocker attention", () => {
originKind: input.originKind ?? "manual",
originId: input.originId ?? null,
originFingerprint: input.originFingerprint ?? "default",
executionState: input.executionState ?? null,
description: input.description ?? null,
});
return id;
}
@@ -483,4 +506,192 @@ describeEmbeddedPostgres("issue blocker attention", () => {
sampleBlockerIdentifier: "PBY-2",
});
});
it("returns blocked inbox attention for an unassigned blocker leaf and supports count/search", async () => {
const { companyId } = await createCompany("BIA");
const parentId = await insertIssue({ companyId, identifier: "BIA-1", title: "Blocked source", status: "blocked" });
const blockerId = await insertIssue({
companyId,
identifier: "BIA-2",
title: "Unassigned leaf",
status: "todo",
});
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
const rows = await svc.list(companyId, { attention: "blocked", q: "BIA-2" });
expect(rows).toHaveLength(1);
expect(rows[0]?.id).toBe(parentId);
expect(rows[0]?.blockedBy).toEqual([
expect.objectContaining({ id: blockerId, identifier: "BIA-2" }),
]);
expect(rows[0]?.blockedInboxAttention).toMatchObject({
kind: "blocked",
state: "needs_attention",
reason: "blocked_by_unassigned_issue",
severity: "critical",
owner: { type: "unknown", agentId: null, userId: null },
action: { label: "Assign blocker" },
leafIssue: { id: blockerId, identifier: "BIA-2" },
redaction: { secretFieldsOmitted: true },
});
await expect(svc.count(companyId, { attention: "blocked" })).resolves.toBe(1);
});
it("redacts external wait details from blocked inbox payloads and search", async () => {
const { companyId } = await createCompany("BIX");
const owner = "Private Vendor Security Team";
const action = "Send the confidential access token for customer Alpha";
const issueId = await insertIssue({
companyId,
identifier: "BIX-1",
title: "Blocked on vendor",
status: "blocked",
description: [
"Public context stays visible.",
`external owner: ${owner}`,
`external action: ${action}`,
"Continue after the vendor confirms receipt.",
].join("\n"),
});
const rows = await svc.list(companyId, { attention: "blocked" });
const issue = rows.find((row) => row.id === issueId);
expect(issue?.description).toContain("Public context stays visible.");
expect(issue?.description).toContain("Continue after the vendor confirms receipt.");
expect(issue?.description).not.toContain(owner);
expect(issue?.description).not.toContain(action);
expect(issue?.blockedInboxAttention).toMatchObject({
state: "external_wait",
reason: "external_owner_action",
owner: { type: "external", label: null },
action: { label: "External owner action", detail: null },
redaction: { externalDetailsRedacted: true, secretFieldsOmitted: true },
});
expect(JSON.stringify(issue?.blockedInboxAttention)).not.toContain(owner);
expect(JSON.stringify(issue?.blockedInboxAttention)).not.toContain(action);
await expect(svc.list(companyId, { attention: "blocked", q: owner })).resolves.toEqual([]);
await expect(svc.count(companyId, { attention: "blocked", q: action })).resolves.toBe(0);
await expect(svc.count(companyId, { attention: "blocked", q: "Public context" })).resolves.toBe(1);
});
it("excludes healthy active blockers from blocked inbox attention", async () => {
const { companyId, agentId } = await createCompany("BIB");
const parentId = await insertIssue({ companyId, identifier: "BIB-1", title: "Blocked source", status: "blocked" });
const blockerId = await insertIssue({
companyId,
identifier: "BIB-2",
title: "Running leaf",
status: "todo",
assigneeAgentId: agentId,
});
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
await activeRun({ companyId, agentId, issueId: blockerId });
expect(await svc.list(companyId, { attention: "blocked" })).toEqual([]);
});
it("classifies assigned backlog and invalid review leaves for blocked inbox attention", async () => {
const { companyId, agentId, pausedAgentId } = await createCompany("BIC");
const backlogParentId = await insertIssue({ companyId, identifier: "BIC-1", title: "Blocked by parked work", status: "blocked" });
const backlogLeafId = await insertIssue({
companyId,
identifier: "BIC-2",
title: "Parked blocker",
status: "backlog",
assigneeAgentId: agentId,
});
await block({ companyId, blockerIssueId: backlogLeafId, blockedIssueId: backlogParentId });
const reviewId = await insertIssue({
companyId,
identifier: "BIC-3",
title: "Invalid review",
status: "in_review",
assigneeAgentId: agentId,
executionState: {
status: "pending",
currentStageId: null,
currentStageIndex: null,
currentStageType: "review",
currentParticipant: { type: "agent", agentId: pausedAgentId },
returnAssignee: null,
reviewRequest: null,
completedStageIds: [],
lastDecisionId: null,
lastDecisionOutcome: null,
},
});
const rows = await svc.list(companyId, { attention: "blocked" });
const byId = new Map(rows.map((row) => [row.id, row]));
expect(byId.get(backlogParentId)?.blockedInboxAttention).toMatchObject({
reason: "blocked_by_assigned_backlog_issue",
severity: "high",
owner: { type: "agent", agentId },
leafIssue: { id: backlogLeafId },
});
expect(byId.get(reviewId)?.blockedInboxAttention).toMatchObject({
reason: "invalid_review_participant",
severity: "critical",
action: { label: "Repair review participant" },
});
});
it("classifies recovery issues and missing successful-run dispositions", async () => {
const { companyId, agentId } = await createCompany("BID");
const sourceId = await insertIssue({ companyId, identifier: "BID-1", title: "Stopped source", status: "blocked" });
const leafId = await insertIssue({ companyId, identifier: "BID-2", title: "Stopped leaf", status: "todo" });
const recoveryId = await insertIssue({
companyId,
identifier: "BID-3",
title: "Recovery issue",
status: "todo",
assigneeAgentId: agentId,
originKind: "harness_liveness_escalation",
originId: buildIssueGraphLivenessIncidentKey({
companyId,
issueId: sourceId,
state: "blocked_by_unassigned_issue",
blockerIssueId: leafId,
}),
});
const handoffId = await insertIssue({
companyId,
identifier: "BID-4",
title: "Needs disposition",
status: "in_progress",
assigneeAgentId: agentId,
});
await db.insert(activityLog).values({
companyId,
actorType: "system",
actorId: "system",
action: "issue.successful_run_handoff_required",
entityType: "issue",
entityId: handoffId,
agentId,
details: { sourceRunId: randomUUID(), detectedProgressSummary: "Progress was made" },
});
const rows = await svc.list(companyId, { attention: "blocked" });
const byId = new Map(rows.map((row) => [row.id, row]));
expect(byId.get(recoveryId)?.blockedInboxAttention).toMatchObject({
state: "recovery_open",
reason: "open_recovery_issue",
sourceIssue: { id: sourceId },
leafIssue: { id: leafId },
recoveryIssue: { id: recoveryId },
});
expect(byId.get(handoffId)?.blockedInboxAttention).toMatchObject({
state: "missing_disposition",
reason: "missing_successful_run_disposition",
owner: { type: "agent", agentId },
action: { label: "Choose disposition" },
});
});
});
@@ -116,6 +116,10 @@ function registerServiceMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
logActivity: mockLogActivity,
projectService: () => mockProjectService,
@@ -86,6 +86,10 @@ function registerModuleMocks() {
heartbeatService: () => mockHeartbeatService,
instanceSettingsService: () => mockInstanceSettingsService,
issueApprovalService: () => ({}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueReferenceService: () => ({
deleteDocumentSource: async () => undefined,
diffIssueReferenceSummary: () => ({
@@ -67,6 +67,9 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
}));
const mockIssueRecoveryActionService = vi.hoisted(() => ({
getActiveForIssue: vi.fn(async () => null),
}));
const mockIssueTreeControlService = vi.hoisted(() => ({
getActivePauseHoldGate: vi.fn(async () => null),
}));
@@ -125,6 +128,7 @@ vi.mock("../services/index.js", () => ({
heartbeatService: () => mockHeartbeatService,
instanceSettingsService: () => mockInstanceSettingsService,
issueApprovalService: () => ({}),
issueRecoveryActionService: () => mockIssueRecoveryActionService,
issueReferenceService: () => ({
deleteDocumentSource: async () => undefined,
diffIssueReferenceSummary: () => ({
@@ -238,6 +242,7 @@ describe.sequential("issue comment reopen routes", () => {
mockInstanceSettingsService.get.mockReset();
mockInstanceSettingsService.listCompanyIds.mockReset();
mockRoutineService.syncRunStatusForIssue.mockReset();
mockIssueRecoveryActionService.getActiveForIssue.mockReset();
mockIssueTreeControlService.getActivePauseHoldGate.mockReset();
mockTxInsertValues.mockReset();
mockTxInsert.mockReset();
@@ -274,6 +279,7 @@ describe.sequential("issue comment reopen routes", () => {
});
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1"]);
mockRoutineService.syncRunStatusForIssue.mockResolvedValue(undefined);
mockIssueRecoveryActionService.getActiveForIssue.mockResolvedValue(null);
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue(null);
mockIssueService.addComment.mockResolvedValue({
id: "comment-1",
@@ -61,6 +61,10 @@ vi.mock("../services/index.js", () => ({
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
logActivity: vi.fn(async () => undefined),
projectService: () => ({
@@ -133,6 +133,10 @@ function registerModuleMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockIssueThreadInteractionService,
logActivity: mockLogActivity,
@@ -82,6 +82,10 @@ function registerModuleMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockIssueThreadInteractionService,
logActivity: mockLogActivity,
@@ -93,6 +93,10 @@ function registerModuleMocks() {
heartbeatService: () => mockHeartbeatService,
issueApprovalService: () => ({}),
issueReferenceService: () => mockIssueReferenceService,
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockIssueThreadInteractionService,
logActivity: mockLogActivity,
@@ -0,0 +1,767 @@
import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
agents,
activityLog,
companies,
createDb,
issueComments,
issueRecoveryActions,
issueRelations,
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { errorHandler } from "../middleware/index.js";
import { issueRoutes } from "../routes/issues.js";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { recoveryService } from "../services/recovery/service.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
function makeRecoveryActionRow(overrides: Record<string, unknown> = {}) {
const now = new Date("2026-05-09T19:30:00.000Z");
return {
id: randomUUID(),
companyId: "company-1",
sourceIssueId: "source-1",
recoveryIssueId: null,
kind: "missing_disposition",
status: "active",
ownerType: "agent",
ownerAgentId: "agent-1",
ownerUserId: null,
previousOwnerAgentId: null,
returnOwnerAgentId: null,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: {},
nextAction: "Choose a valid issue disposition.",
wakePolicy: null,
monitorPolicy: null,
attemptCount: 1,
maxAttempts: null,
timeoutAt: null,
lastAttemptAt: now,
outcome: null,
resolutionNote: null,
resolvedAt: null,
createdAt: now,
updatedAt: now,
...overrides,
};
}
describe("issueRecoveryActionService", () => {
it("does not reactivate an action resolved between the active read and update", async () => {
const existingRow = makeRecoveryActionRow({ id: "existing-action", attemptCount: 1 });
const createdRow = makeRecoveryActionRow({ id: "new-action", attemptCount: 1 });
const selectResults = [[existingRow], []];
const makeSelectQuery = (rows: unknown[]) => ({
from() {
return this;
},
where() {
return this;
},
orderBy() {
return this;
},
limit() {
return Promise.resolve(rows);
},
});
const fakeDb = {
select: vi.fn(() => makeSelectQuery(selectResults.shift() ?? [])),
update: vi.fn(() => ({
set: vi.fn(() => ({
where: vi.fn(() => ({
returning: vi.fn(async () => []),
})),
})),
})),
insert: vi.fn(() => ({
values: vi.fn(() => ({
returning: vi.fn(async () => [createdRow]),
})),
})),
};
const result = await issueRecoveryActionService(fakeDb as never).upsertSourceScoped({
companyId: "company-1",
sourceIssueId: "source-1",
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: "agent-1",
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
nextAction: "Choose a valid issue disposition.",
});
expect(result).toMatchObject({ id: "new-action", status: "active" });
expect(fakeDb.update).toHaveBeenCalledTimes(1);
expect(fakeDb.insert).toHaveBeenCalledTimes(1);
});
});
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres issue recovery action tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("issue recovery actions", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let db: ReturnType<typeof createDb>;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-");
db = createDb(tempDb.connectionString);
}, 30_000);
afterEach(async () => {
await db.delete(issueRecoveryActions);
await db.delete(issueComments);
await db.delete(activityLog);
await db.delete(issues);
await db.delete(agents);
await db.delete(companies);
});
afterAll(async () => {
await tempDb?.cleanup();
});
async function seedCompany() {
const companyId = randomUUID();
const managerId = randomUUID();
const coderId = randomUUID();
const sourceIssueId = randomUUID();
const prefix = `RA${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`;
await db.insert(companies).values({
id: companyId,
name: "Recovery Co",
issuePrefix: prefix,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
{
id: managerId,
companyId,
name: "CTO",
role: "cto",
status: "idle",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
},
{
id: coderId,
companyId,
name: "Coder",
role: "engineer",
status: "idle",
reportsTo: managerId,
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
},
]);
await db.insert(issues).values({
id: sourceIssueId,
companyId,
title: "Implement backend recovery",
status: "in_progress",
priority: "medium",
assigneeAgentId: coderId,
issueNumber: 1,
identifier: `${prefix}-1`,
});
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
return { companyId, managerId, coderId, sourceIssueId, prefix, sourceIssue: sourceIssue! };
}
function createApp(actor: any = { type: "board", source: "local_implicit" }) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
(req as any).actor = actor;
next();
});
app.use("/api", issueRoutes(db, {} as any));
app.use(errorHandler);
return app;
}
it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const svc = issueRecoveryActionService(db);
const first = await svc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "stranded_assigned_issue",
ownerType: "agent",
ownerAgentId: managerId,
cause: "stranded_assigned_issue",
fingerprint: "recovery:fingerprint",
evidence: { latestRunId: "run-1" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "wake_owner" },
});
const second = await svc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "stranded_assigned_issue",
ownerType: "agent",
ownerAgentId: managerId,
cause: "stranded_assigned_issue",
fingerprint: "recovery:fingerprint",
evidence: { latestRunId: "run-2" },
nextAction: "Restore a live execution path.",
wakePolicy: { type: "wake_owner" },
});
expect(second.id).toBe(first.id);
expect(second.attemptCount).toBe(2);
expect(second.evidence).toMatchObject({ latestRunId: "run-2" });
expect(await svc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({ id: first.id });
expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull();
});
it("escalates stranded assigned work into a source action instead of a recovery issue", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
const enqueueWakeup = vi.fn(async () => null);
const recovery = recoveryService(db, { enqueueWakeup });
const latestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun,
comment: "Automatic continuation recovery failed.",
});
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(updatedIssue).toMatchObject({
status: "blocked",
});
const recoveryIssues = await db
.select()
.from(issues)
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
expect(recoveryIssues).toHaveLength(0);
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({
issueId: sourceIssue.id,
sourceIssueId: sourceIssue.id,
recoveryCause: "stranded_assigned_issue",
});
});
it("reuses the same source-scoped action when latest run IDs change while the cause stays the same", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
const enqueueWakeup = vi.fn(async () => null);
const recovery = recoveryService(db, { enqueueWakeup });
const firstLatestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
const secondLatestRun = {
...firstLatestRun,
id: randomUUID(),
};
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: firstLatestRun,
comment: "Automatic continuation recovery failed.",
});
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: secondLatestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
expect(actionRows[0]?.evidence).toMatchObject({ latestRunId: secondLatestRun.id });
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
expect(enqueueWakeup.mock.calls[1]?.[1]?.payload).toMatchObject({
issueId: sourceIssue.id,
sourceIssueId: sourceIssue.id,
strandedRunId: secondLatestRun.id,
recoveryCause: "stranded_assigned_issue",
});
});
it("keeps the source issue blocked when source-scoped wakeup is claimed synchronously", async () => {
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, managerId));
const enqueueWakeup = vi.fn(async () => {
await db
.update(issues)
.set({ status: "in_progress" })
.where(eq(issues.id, sourceIssue.id));
return null;
});
const recovery = recoveryService(db, { enqueueWakeup });
const firstLatestRun = {
id: randomUUID(),
agentId: coderId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
} as const;
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: firstLatestRun,
comment: "Automatic continuation recovery failed.",
});
const [afterFirst] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(afterFirst?.status).toBe("blocked");
expect(afterFirst?.assigneeAgentId).toBe(coderId);
const secondLatestRun = {
...firstLatestRun,
id: randomUUID(),
};
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: secondLatestRun,
comment: "Automatic continuation recovery failed.",
});
const actionRows = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
expect(actionRows).toHaveLength(1);
expect(actionRows[0]).toMatchObject({
companyId,
kind: "stranded_assigned_issue",
status: "active",
previousOwnerAgentId: coderId,
returnOwnerAgentId: coderId,
cause: "stranded_assigned_issue",
attemptCount: 2,
});
const [afterSecond] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
expect(afterSecond?.status).toBe("blocked");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("Recovery action:");
});
it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => {
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
const recoveryIssueId = randomUUID();
await db.insert(issues).values({
id: recoveryIssueId,
companyId,
title: "Recover stalled issue",
status: "in_progress",
priority: "medium",
assigneeAgentId: managerId,
parentId: sourceIssueId,
issueNumber: 2,
identifier: `${prefix}-2`,
originKind: "stranded_issue_recovery",
originId: sourceIssueId,
originFingerprint: `stranded_issue_recovery:${sourceIssueId}`,
});
const [recoveryIssue] = await db.select().from(issues).where(eq(issues.id, recoveryIssueId));
const recovery = recoveryService(db, { enqueueWakeup: vi.fn(async () => null) });
await recovery.escalateStrandedAssignedIssue({
issue: recoveryIssue!,
previousStatus: "in_progress",
latestRun: {
id: randomUUID(),
agentId: managerId,
status: "failed",
error: "adapter failed",
errorCode: "adapter_failed",
contextSnapshot: { retryReason: "issue_continuation_needed" },
livenessState: "needs_followup",
},
});
const actionRows = await db.select().from(issueRecoveryActions);
expect(actionRows).toHaveLength(0);
const recoveryIssues = await db
.select()
.from(issues)
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
expect(recoveryIssues).toHaveLength(1);
expect(recoveryIssues[0]?.status).toBe("blocked");
});
it("exposes active recovery actions on the issue read API", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toMatchObject({
id: action.id,
sourceIssueId,
kind: "missing_disposition",
ownerAgentId: managerId,
});
const list = await request(app).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);
expect(list.body.active).toMatchObject({ id: action.id });
expect(list.body.actions).toHaveLength(1);
});
it("resolves an active recovery action and removes it from active projections", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
resolutionNote: "Operator confirmed the source issue is complete.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "done",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "restored",
resolutionNote: "Operator confirmed the source issue is complete.",
});
expect(resolved.body.recoveryAction.resolvedAt).toBeTruthy();
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
expect(detail.body.activeRecoveryAction).toBeNull();
const activityRows = await db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, sourceIssueId));
expect(activityRows.map((row) => row.action)).toEqual(
expect.arrayContaining(["issue.updated", "issue.recovery_action_resolved"]),
);
});
it("rejects blocked recovery resolution when the source issue has no first-class blockers", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:blocked-without-blocker",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Choose a disposition with a live continuation path.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const rejected = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "blocked",
sourceIssueStatus: "blocked",
})
.expect(422);
expect(rejected.body.error).toContain("requires an unresolved first-class blocker");
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("in_progress");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolvedAt: null,
});
});
it("allows blocked recovery resolution when the source issue has an unresolved first-class blocker", async () => {
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
const blockerIssueId = randomUUID();
await db.insert(issues).values({
id: blockerIssueId,
companyId,
title: "Unblock recovery disposition",
status: "todo",
priority: "medium",
assigneeAgentId: managerId,
issueNumber: 2,
identifier: `${prefix}-2`,
});
await db.insert(issueRelations).values({
companyId,
issueId: blockerIssueId,
relatedIssueId: sourceIssueId,
type: "blocks",
});
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:blocked-with-blocker",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Wait for the blocker before continuing.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "blocked",
sourceIssueStatus: "blocked",
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "blocked",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "blocked",
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
});
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
});
it("rejects false-positive recovery resolution without an explicit source issue status", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:fingerprint",
evidence: { latestIssueStatus: "in_progress" },
nextAction: "Confirm whether the issue is actually stranded.",
wakePolicy: { type: "manual" },
});
const app = createApp();
await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "false_positive",
resolutionNote: "The source issue still has a live execution path.",
})
.expect(400);
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
expect(sourceIssue?.status).toBe("in_progress");
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow).toMatchObject({
status: "active",
outcome: null,
resolutionNote: null,
});
});
it("allows false-positive recovery resolution to restore a blocked source issue in the same request", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, sourceIssueId));
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "issue_graph_liveness",
ownerType: "agent",
ownerAgentId: managerId,
cause: "issue_graph_liveness",
fingerprint: "graph-liveness:false-positive-unblock",
evidence: { latestIssueStatus: "blocked" },
nextAction: "Confirm whether the issue is actually stranded.",
wakePolicy: { type: "manual" },
});
const app = createApp();
const resolved = await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "false_positive",
sourceIssueStatus: "in_review",
resolutionNote: "Recovery signal was stale; return to review.",
})
.expect(200);
expect(resolved.body.issue).toMatchObject({
id: sourceIssueId,
status: "in_review",
activeRecoveryAction: null,
});
expect(resolved.body.recoveryAction).toMatchObject({
id: action.id,
status: "resolved",
outcome: "false_positive",
resolutionNote: "Recovery signal was stale; return to review.",
});
});
it("enforces company scope when resolving recovery actions", async () => {
const { companyId, managerId, sourceIssueId } = await seedCompany();
const recoveryActionSvc = issueRecoveryActionService(db);
const action = await recoveryActionSvc.upsertSourceScoped({
companyId,
sourceIssueId,
kind: "missing_disposition",
ownerType: "agent",
ownerAgentId: managerId,
cause: "successful_run_missing_issue_disposition",
fingerprint: "missing-disposition:fingerprint",
evidence: { sourceRunId: "run-1" },
nextAction: "Choose a valid issue disposition.",
wakePolicy: { type: "wake_owner" },
});
const app = createApp({
type: "agent",
agentId: randomUUID(),
companyId: randomUUID(),
runId: randomUUID(),
source: "agent_jwt",
});
await request(app)
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
.send({
actionId: action.id,
outcome: "restored",
sourceIssueStatus: "done",
})
.expect(403);
const [actionRow] = await db
.select()
.from(issueRecoveryActions)
.where(eq(issueRecoveryActions.id, action.id));
expect(actionRow?.status).toBe("active");
});
});
@@ -58,6 +58,10 @@ function registerModuleMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
logActivity: vi.fn(async () => undefined),
projectService: () => ({}),
@@ -86,6 +86,10 @@ function registerModuleMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockInteractionService,
logActivity: mockLogActivity,
@@ -73,6 +73,10 @@ vi.mock("../services/index.js", () => ({
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockIssueThreadInteractionService,
logActivity: vi.fn(async () => undefined),
@@ -131,6 +135,10 @@ function registerModuleMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
issueThreadInteractionService: () => mockIssueThreadInteractionService,
logActivity: vi.fn(async () => undefined),
@@ -115,6 +115,10 @@ function registerRouteMocks() {
syncDocument: async () => undefined,
syncIssue: async () => undefined,
}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueService: () => mockIssueService,
logActivity: mockLogActivity,
projectService: () => ({}),
@@ -111,6 +111,10 @@ vi.mock("../services/index.js", () => ({
heartbeatService: () => mockHeartbeatService,
instanceSettingsService: () => mockInstanceSettingsService,
issueApprovalService: () => ({}),
issueRecoveryActionService: () => ({
getActiveForIssue: vi.fn(async () => null),
listActiveForIssues: vi.fn(async () => new Map()),
}),
issueReferenceService: () => mockIssueReferenceService,
issueService: () => mockIssueService,
logActivity: mockLogActivity,
+132 -1
View File
@@ -24,7 +24,12 @@ import {
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { instanceSettingsService } from "../services/instance-settings.ts";
import { clampIssueListLimit, ISSUE_LIST_MAX_LIMIT, issueService } from "../services/issues.ts";
import {
clampIssueListLimit,
deriveIssueCommentRunLogAttribution,
ISSUE_LIST_MAX_LIMIT,
issueService,
} from "../services/issues.ts";
import { buildProjectMentionHref, MAX_ISSUE_REQUEST_DEPTH } from "@paperclipai/shared";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
@@ -38,6 +43,69 @@ describe("issue list limit helpers", () => {
});
});
describe("deriveIssueCommentRunLogAttribution", () => {
it("recovers agent attribution from run logs that printed the posted comment id", () => {
const commentId = randomUUID();
const runId = randomUUID();
const agentId = randomUUID();
const derived = deriveIssueCommentRunLogAttribution(
[
{
id: commentId,
authorAgentId: null,
authorUserId: "user-1",
createdByRunId: null,
createdAt: new Date("2026-05-11T18:55:40.090Z"),
},
],
[
{
runId,
agentId,
createdAt: new Date("2026-05-11T18:51:56.246Z"),
startedAt: new Date("2026-05-11T18:51:56.257Z"),
finishedAt: new Date("2026-05-11T18:55:45.600Z"),
logContent: `comment id: ${commentId}\n`,
},
],
);
expect(derived.get(commentId)).toEqual({
derivedAuthorAgentId: agentId,
derivedCreatedByRunId: runId,
derivedAuthorSource: "run_log_comment_post",
});
});
it("does not rewrite comments without exact run-log proof", () => {
const commentId = randomUUID();
const derived = deriveIssueCommentRunLogAttribution(
[
{
id: commentId,
authorAgentId: null,
authorUserId: "user-1",
createdByRunId: null,
createdAt: new Date("2026-05-11T18:55:40.090Z"),
},
],
[
{
runId: randomUUID(),
agentId: randomUUID(),
createdAt: new Date("2026-05-11T18:51:56.246Z"),
startedAt: new Date("2026-05-11T18:51:56.257Z"),
finishedAt: new Date("2026-05-11T18:55:45.600Z"),
logContent: "posted results without echoing the comment id",
},
],
);
expect(derived.has(commentId)).toBe(false);
});
});
async function ensureIssueRelationsTable(db: ReturnType<typeof createDb>) {
await db.execute(sql.raw(`
CREATE TABLE IF NOT EXISTS "issue_relations" (
@@ -82,6 +150,7 @@ describeEmbeddedPostgres("issueService.list participantAgentId", () => {
await db.delete(projectWorkspaces);
await db.delete(projects);
await db.delete(goals);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(instanceSettings);
await db.delete(companies);
@@ -1091,6 +1160,68 @@ describeEmbeddedPostgres("issueService.list participantAgentId", () => {
expect(comments.map((comment) => comment.id)).toEqual([firstCommentId]);
});
it("lists user comments when derived run attribution scans a timestamp window", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
const issueId = randomUUID();
const commentId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
id: agentId,
companyId,
name: "CodexCoder",
role: "engineer",
status: "active",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
});
await db.insert(issues).values({
id: issueId,
companyId,
title: "Comments issue",
status: "todo",
priority: "medium",
});
await db.insert(heartbeatRuns).values({
id: randomUUID(),
companyId,
agentId,
contextSnapshot: { issueId },
createdAt: new Date("2026-05-12T22:58:00.000Z"),
startedAt: new Date("2026-05-12T22:58:00.000Z"),
finishedAt: new Date("2026-05-12T23:14:00.000Z"),
});
await db.insert(issueComments).values({
id: commentId,
companyId,
issueId,
authorUserId: "user-1",
body: "Comment should be visible",
createdAt: new Date("2026-05-12T23:00:00.000Z"),
updatedAt: new Date("2026-05-12T23:00:00.000Z"),
});
const comments = await svc.listComments(issueId, {
order: "desc",
limit: 50,
});
expect(comments.map((comment) => comment.id)).toEqual([commentId]);
expect(comments[0]?.body).toBe("Comment should be visible");
});
it("includes blockedBy summaries on list rows in one batched pass", async () => {
const companyId = randomUUID();
const blockerId = randomUUID();
+43 -1
View File
@@ -25,7 +25,7 @@ import {
validatePluginRuntimeExecute,
validatePluginRuntimeQuery,
} from "../services/plugin-database.js";
import { pluginLoader } from "../services/plugin-loader.js";
import { buildPluginWorkerEnv, pluginLoader } from "../services/plugin-loader.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -84,6 +84,48 @@ describe("plugin database SQL validation", () => {
});
});
describe("buildPluginWorkerEnv", () => {
const instanceInfo = {
deploymentMode: "authenticated",
deploymentExposure: "public",
};
it("passes only model provider keys through to environment driver plugins", () => {
const env = buildPluginWorkerEnv({
manifest: { capabilities: ["environment.drivers.register"] },
instanceInfo,
processEnv: {
ANTHROPIC_API_KEY: "anthropic-token",
OPENAI_API_KEY: "openai-token",
GEMINI_API_KEY: " ",
AWS_SECRET_ACCESS_KEY: "aws-secret",
},
});
expect(env).toEqual({
PAPERCLIP_DEPLOYMENT_MODE: "authenticated",
PAPERCLIP_DEPLOYMENT_EXPOSURE: "public",
ANTHROPIC_API_KEY: "anthropic-token",
OPENAI_API_KEY: "openai-token",
});
});
it("does not pass provider keys to non-environment plugins", () => {
const env = buildPluginWorkerEnv({
manifest: { capabilities: ["ui.slots.register"] },
instanceInfo,
processEnv: {
OPENAI_API_KEY: "openai-token",
},
});
expect(env).toEqual({
PAPERCLIP_DEPLOYMENT_MODE: "authenticated",
PAPERCLIP_DEPLOYMENT_EXPOSURE: "public",
});
});
});
describeEmbeddedPostgres("plugin database namespaces", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+98 -18
View File
@@ -1,12 +1,28 @@
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { EventEmitter } from "node:events";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { resolvePluginWatchTargets } from "../services/plugin-dev-watcher.js";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const chokidarMock = vi.hoisted(() => ({
watch: vi.fn(),
}));
vi.mock("chokidar", () => ({
default: chokidarMock,
}));
import { createPluginDevWatcher, resolvePluginWatchTargets } from "../services/plugin-dev-watcher.js";
const tempDirs: string[] = [];
beforeEach(() => {
vi.useRealTimers();
chokidarMock.watch.mockReset();
});
afterEach(() => {
vi.useRealTimers();
while (tempDirs.length > 0) {
const dir = tempDirs.pop();
if (dir) rmSync(dir, { recursive: true, force: true });
@@ -19,25 +35,49 @@ function makeTempPluginDir(): string {
return dir;
}
function writePluginPackage(pluginDir: string): void {
mkdirSync(path.join(pluginDir, "dist", "ui"), { recursive: true });
writeFileSync(
path.join(pluginDir, "package.json"),
JSON.stringify({
name: "@acme/example",
paperclipPlugin: {
manifest: "./dist/manifest.js",
worker: "./dist/worker.js",
ui: "./dist/ui",
},
}),
);
writeFileSync(path.join(pluginDir, "dist", "manifest.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "worker.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "ui", "index.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "ui", "index.css"), "body {}\n");
}
function createLifecycle() {
const emitter = new EventEmitter();
return Object.assign(emitter, {
restartWorker: vi.fn().mockResolvedValue(undefined),
});
}
function installMockFsWatcher() {
const handlers: Record<string, (...args: unknown[]) => void> = {};
const fakeWatcher = {
close: vi.fn(),
on: vi.fn((event: string, listener: (...args: unknown[]) => void) => {
handlers[event] = listener;
return fakeWatcher;
}),
};
chokidarMock.watch.mockReturnValue(fakeWatcher);
return { fakeWatcher, handlers };
}
describe("resolvePluginWatchTargets", () => {
it("watches package metadata plus concrete declared runtime files", () => {
const pluginDir = makeTempPluginDir();
mkdirSync(path.join(pluginDir, "dist", "ui"), { recursive: true });
writeFileSync(
path.join(pluginDir, "package.json"),
JSON.stringify({
name: "@acme/example",
paperclipPlugin: {
manifest: "./dist/manifest.js",
worker: "./dist/worker.js",
ui: "./dist/ui",
},
}),
);
writeFileSync(path.join(pluginDir, "dist", "manifest.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "worker.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "ui", "index.js"), "export default {};\n");
writeFileSync(path.join(pluginDir, "dist", "ui", "index.css"), "body {}\n");
writePluginPackage(pluginDir);
const targets = resolvePluginWatchTargets(pluginDir);
@@ -66,3 +106,43 @@ describe("resolvePluginWatchTargets", () => {
]);
});
});
describe("createPluginDevWatcher", () => {
it("starts watching local plugins announced by lifecycle events", async () => {
const pluginDir = makeTempPluginDir();
writePluginPackage(pluginDir);
installMockFsWatcher();
const lifecycle = createLifecycle();
const devWatcher = createPluginDevWatcher(
lifecycle as never,
async (pluginId) => (pluginId === "plugin-1" ? pluginDir : null),
);
lifecycle.emit("plugin.loaded", { pluginId: "plugin-1", pluginKey: "example" });
await vi.waitFor(() => expect(chokidarMock.watch).toHaveBeenCalledTimes(1));
const [watchedPaths] = chokidarMock.watch.mock.calls[0] ?? [];
expect(watchedPaths).toContain(path.join(pluginDir, "dist", "worker.js"));
devWatcher.close();
});
it("debounces watched file changes and restarts the plugin worker", async () => {
vi.useFakeTimers();
const pluginDir = makeTempPluginDir();
writePluginPackage(pluginDir);
const { handlers } = installMockFsWatcher();
const lifecycle = createLifecycle();
const devWatcher = createPluginDevWatcher(lifecycle as never);
devWatcher.watch("plugin-1", pluginDir);
handlers.all?.("change", path.join(pluginDir, "dist", "worker.js"));
await vi.advanceTimersByTimeAsync(500);
expect(lifecycle.restartWorker).toHaveBeenCalledWith("plugin-1");
devWatcher.close();
});
});
@@ -224,6 +224,46 @@ describe.sequential("plugin install and upgrade authz", () => {
expect(mockLifecycle.disable).not.toHaveBeenCalled();
}, 20_000);
it("resolves plugin keys without probing the UUID id column for core plugin actions", async () => {
const pluginKey = "paperclipqa.hello-plugin";
const plugin = {
id: pluginId,
pluginKey,
version: "1.0.0",
status: "ready",
};
mockRegistry.getById.mockImplementation(() => {
throw new Error("getById should not be called for plugin keys");
});
mockRegistry.getByKey.mockResolvedValue(plugin);
mockLifecycle.unload.mockResolvedValue(plugin);
mockLifecycle.enable.mockResolvedValue(plugin);
mockLifecycle.disable.mockResolvedValue(plugin);
const { app } = await createApp({
type: "board",
userId: "admin-1",
source: "session",
isInstanceAdmin: true,
companyIds: [companyA],
});
const inspectRes = await request(app).get(`/api/plugins/${pluginKey}`);
const disableRes = await request(app).post(`/api/plugins/${pluginKey}/disable`).send({});
const enableRes = await request(app).post(`/api/plugins/${pluginKey}/enable`).send({});
const uninstallRes = await request(app).delete(`/api/plugins/${pluginKey}?purge=true`);
expect(inspectRes.status).toBe(200);
expect(disableRes.status).toBe(200);
expect(enableRes.status).toBe(200);
expect(uninstallRes.status).toBe(200);
expect(mockRegistry.getById).not.toHaveBeenCalled();
expect(mockRegistry.getByKey).toHaveBeenCalledWith(pluginKey);
expect(mockLifecycle.disable).toHaveBeenCalledWith(pluginId, undefined);
expect(mockLifecycle.enable).toHaveBeenCalledWith(pluginId);
expect(mockLifecycle.unload).toHaveBeenCalledWith(pluginId, true);
}, 20_000);
it("rejects plugin config saves that contain secret refs even for instance admins", async () => {
readyPlugin();
+4 -6
View File
@@ -420,12 +420,10 @@ export async function createApp(
void toolDispatcher.initialize().catch((err) => {
logger.error({ err }, "Failed to initialize plugin tool dispatcher");
});
const devWatcher = opts.uiMode === "vite-dev"
? createPluginDevWatcher(
lifecycle,
async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null,
)
: null;
const devWatcher = createPluginDevWatcher(
lifecycle,
async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null,
);
void loader.loadAll().then((result) => {
if (!result) return;
for (const loaded of result.results) {
+8 -4
View File
@@ -42,6 +42,7 @@ import {
heartbeatService,
ISSUE_LIST_DEFAULT_LIMIT,
issueApprovalService,
issueRecoveryActionService,
issueService,
logActivity,
syncInstructionsBundleConfigFromFilePath,
@@ -1742,16 +1743,18 @@ export function agentRoutes(
}
const issuesSvc = issueService(db);
const recoveryActionsSvc = issueRecoveryActionService(db);
const rows = await issuesSvc.list(req.actor.companyId, {
assigneeAgentId: req.actor.agentId,
status: "todo,in_progress,blocked",
includeRoutineExecutions: true,
limit: ISSUE_LIST_DEFAULT_LIMIT,
});
const dependencyReadiness = await issuesSvc.listDependencyReadiness(
req.actor.companyId,
rows.map((issue) => issue.id),
);
const issueIds = rows.map((issue) => issue.id);
const [dependencyReadiness, recoveryActionByIssue] = await Promise.all([
issuesSvc.listDependencyReadiness(req.actor.companyId, issueIds),
recoveryActionsSvc.listActiveForIssues(req.actor.companyId, issueIds),
]);
res.json(
rows.map((issue) => ({
@@ -1765,6 +1768,7 @@ export function agentRoutes(
parentId: issue.parentId,
updatedAt: issue.updatedAt,
activeRun: issue.activeRun,
activeRecoveryAction: recoveryActionByIssue.get(issue.id) ?? null,
dependencyReady: dependencyReadiness.get(issue.id)?.isDependencyReady ?? true,
unresolvedBlockerCount: dependencyReadiness.get(issue.id)?.unresolvedBlockerCount ?? 0,
unresolvedBlockerIssueIds: dependencyReadiness.get(issue.id)?.unresolvedBlockerIssueIds ?? [],
+279 -11
View File
@@ -2,9 +2,16 @@ import { randomUUID } from "node:crypto";
import { Router, type Request, type Response } from "express";
import multer from "multer";
import { z } from "zod";
import { and, desc, eq, inArray } from "drizzle-orm";
import { and, desc, eq, inArray, notInArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { activityLog, executionWorkspaces, issueExecutionDecisions, projectWorkspaces } from "@paperclipai/db";
import {
activityLog,
executionWorkspaces,
issueExecutionDecisions,
issueRelations,
issues as issueRows,
projectWorkspaces,
} from "@paperclipai/db";
import {
addIssueCommentSchema,
acceptIssueThreadInteractionSchema,
@@ -18,6 +25,7 @@ import {
createChildIssueSchema,
createIssueSchema,
resolveCreateIssueStatusDefault,
resolveIssueRecoveryActionSchema,
feedbackTargetTypeSchema,
feedbackTraceStatusSchema,
feedbackVoteValueSchema,
@@ -37,6 +45,7 @@ import {
type CompanySearchQuery,
type CompanySearchResponse,
type ExecutionWorkspace,
type IssueRelationIssueSummary,
type SuccessfulRunHandoffState,
} from "@paperclipai/shared";
import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry";
@@ -53,6 +62,7 @@ import {
goalService,
heartbeatService,
issueApprovalService,
issueRecoveryActionService,
issueThreadInteractionService,
ISSUE_LIST_DEFAULT_LIMIT,
ISSUE_LIST_MAX_LIMIT,
@@ -405,6 +415,47 @@ async function listSuccessfulRunHandoffStates(
return states;
}
type RecoveryActionsLister = {
listActiveForIssues: (
companyId: string,
sourceIssueIds: string[],
) => Promise<Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>>;
};
async function relationRecoveryActionMap(
recoveryActionsSvc: RecoveryActionsLister,
companyId: string,
relations: { blockedBy: IssueRelationIssueSummary[]; blocks: IssueRelationIssueSummary[] },
): Promise<Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>> {
const candidates: IssueRelationIssueSummary[] = [];
const visit = (summary: IssueRelationIssueSummary) => {
candidates.push(summary);
for (const terminal of summary.terminalBlockers ?? []) {
visit(terminal);
}
};
for (const blocker of relations.blockedBy) visit(blocker);
for (const blocking of relations.blocks) visit(blocking);
if (candidates.length === 0) return new Map();
const ids = [...new Set(candidates.map((summary) => summary.id))];
return recoveryActionsSvc.listActiveForIssues(companyId, ids);
}
function withRecoveryActionsOnRelationSummaries(
relations: { blockedBy: IssueRelationIssueSummary[]; blocks: IssueRelationIssueSummary[] },
recoveryActionByIssueId: Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>,
) {
const augment = (summary: IssueRelationIssueSummary): IssueRelationIssueSummary => ({
...summary,
activeRecoveryAction: recoveryActionByIssueId.get(summary.id) ?? summary.activeRecoveryAction ?? null,
terminalBlockers: summary.terminalBlockers?.map(augment),
});
return {
blockedBy: relations.blockedBy.map(augment),
blocks: relations.blocks.map(augment),
};
}
const ACTIVE_REVIEW_APPROVAL_STATUSES = new Set(["pending", "revision_requested"]);
const INVALID_AGENT_IN_REVIEW_DISPOSITION_MESSAGE =
@@ -787,6 +838,7 @@ export function issueRoutes(
const projectsSvc = projectService(db);
const goalsSvc = goalService(db);
const issueApprovalsSvc = issueApprovalService(db);
const recoveryActionsSvc = issueRecoveryActionService(db);
const executionWorkspacesSvc = executionWorkspaceServiceDirect(db);
const workProductsSvc = workProductService(db);
const documentsSvc = documentService(db);
@@ -1392,6 +1444,7 @@ export function issueRoutes(
const parsedOffset = rawOffset !== undefined && /^\d+$/.test(rawOffset)
? Number.parseInt(rawOffset, 10)
: null;
const attention = req.query.attention as string | undefined;
if (assigneeUserFilterRaw === "me" && (!assigneeUserId || req.actor.type !== "board")) {
res.status(403).json({ error: "assigneeUserId=me requires board authentication" });
@@ -1409,6 +1462,10 @@ export function issueRoutes(
res.status(403).json({ error: "unreadForUserId=me requires board authentication" });
return;
}
if (attention !== undefined && attention !== "blocked") {
res.status(400).json({ error: "attention must be 'blocked' when provided" });
return;
}
if (rawLimit !== undefined && (parsedLimit === null || !Number.isInteger(parsedLimit) || parsedLimit <= 0)) {
res.status(400).json({ error: `limit must be a positive integer up to ${ISSUE_LIST_MAX_LIMIT}` });
return;
@@ -1420,6 +1477,7 @@ export function issueRoutes(
const offset = parsedOffset ?? 0;
const result = await svc.list(companyId, {
attention: attention === "blocked" ? "blocked" : undefined,
status: req.query.status as string | undefined,
assigneeAgentId: req.query.assigneeAgentId as string | undefined,
participantAgentId: req.query.participantAgentId as string | undefined,
@@ -1443,21 +1501,65 @@ export function issueRoutes(
includePluginOperations:
req.query.includePluginOperations === "true" || req.query.includePluginOperations === "1",
includeBlockedBy: req.query.includeBlockedBy === "true" || req.query.includeBlockedBy === "1",
includeBlockedInboxAttention:
req.query.includeBlockedInboxAttention === "true" || req.query.includeBlockedInboxAttention === "1",
q: req.query.q as string | undefined,
limit,
offset,
});
const handoffStates = await listSuccessfulRunHandoffStates(
db,
companyId,
result.map((issue) => issue.id),
);
const issueIds = result.map((issue) => issue.id);
const [handoffStates, recoveryActionByIssue] = await Promise.all([
listSuccessfulRunHandoffStates(db, companyId, issueIds),
recoveryActionsSvc.listActiveForIssues(companyId, issueIds),
]);
res.json(result.map((issue) => ({
...issue,
successfulRunHandoff: handoffStates.get(issue.id) ?? null,
activeRecoveryAction: recoveryActionByIssue.get(issue.id) ?? null,
})));
});
router.get("/companies/:companyId/issues/count", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
const attention = req.query.attention as string | undefined;
if (attention !== "blocked") {
res.status(400).json({ error: "issues/count currently requires attention=blocked" });
return;
}
if (req.query.limit !== undefined || req.query.offset !== undefined) {
res.status(400).json({ error: "issues/count does not accept limit or offset" });
return;
}
const count = await svc.count(companyId, {
attention: "blocked",
status: req.query.status as string | undefined,
assigneeAgentId: req.query.assigneeAgentId as string | undefined,
participantAgentId: req.query.participantAgentId as string | undefined,
assigneeUserId: req.query.assigneeUserId as string | undefined,
projectId: req.query.projectId as string | undefined,
workspaceId: req.query.workspaceId as string | undefined,
executionWorkspaceId: req.query.executionWorkspaceId as string | undefined,
parentId: req.query.parentId as string | undefined,
descendantOf: req.query.descendantOf as string | undefined,
labelId: req.query.labelId as string | undefined,
originKind: req.query.originKind as string | undefined,
originKindPrefix: req.query.originKindPrefix as string | undefined,
originId: req.query.originId as string | undefined,
includeRoutineExecutions:
req.query.includeRoutineExecutions === "true" || req.query.includeRoutineExecutions === "1",
excludeRoutineExecutions:
req.query.excludeRoutineExecutions === "true" || req.query.excludeRoutineExecutions === "1",
includePluginOperations:
req.query.includePluginOperations === "true" || req.query.includePluginOperations === "1",
includeBlockedBy: true,
includeBlockedInboxAttention: true,
q: req.query.q as string | undefined,
});
res.json({ count });
});
router.get("/companies/:companyId/labels", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
@@ -1541,6 +1643,7 @@ export function issueRoutes(
attachments,
continuationSummary,
currentExecutionWorkspace,
activeRecoveryAction,
] =
await Promise.all([
resolveIssueProjectAndGoal(issue),
@@ -1554,7 +1657,17 @@ export function issueRoutes(
svc.listAttachments(issue.id),
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
currentExecutionWorkspacePromise,
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
]);
const recoveryActionsByRelationIssue = await relationRecoveryActionMap(
recoveryActionsSvc,
issue.companyId,
relations,
);
const relationsWithRecoveryActions = withRecoveryActionsOnRelationSummaries(
relations,
recoveryActionsByRelationIssue,
);
res.json({
issue: {
@@ -1567,12 +1680,13 @@ export function issueRoutes(
...(blockerAttention ? { blockerAttention } : {}),
productivityReview,
scheduledRetry,
activeRecoveryAction,
priority: issue.priority,
projectId: issue.projectId,
goalId: goal?.id ?? issue.goalId,
parentId: issue.parentId,
blockedBy: relations.blockedBy,
blocks: relations.blocks,
blockedBy: relationsWithRecoveryActions.blockedBy,
blocks: relationsWithRecoveryActions.blocks,
assigneeAgentId: issue.assigneeAgentId,
assigneeUserId: issue.assigneeUserId,
originKind: issue.originKind,
@@ -1649,6 +1763,7 @@ export function issueRoutes(
referenceSummary,
successfulRunHandoffStates,
scheduledRetry,
activeRecoveryAction,
] = await Promise.all([
resolveIssueProjectAndGoal(issue),
svc.getAncestors(issue.id),
@@ -1660,7 +1775,17 @@ export function issueRoutes(
issueReferencesSvc.listIssueReferenceSummary(issue.id),
listSuccessfulRunHandoffStates(db, issue.companyId, [issue.id]),
svc.getCurrentScheduledRetry(issue.id),
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
]);
const recoveryActionsByRelationIssue = await relationRecoveryActionMap(
recoveryActionsSvc,
issue.companyId,
relations,
);
const relationsWithRecoveryActions = withRecoveryActionsOnRelationSummaries(
relations,
recoveryActionsByRelationIssue,
);
const mentionedProjects = mentionedProjectIds.length > 0
? await projectsSvc.listByIds(issue.companyId, mentionedProjectIds)
: [];
@@ -1676,8 +1801,9 @@ export function issueRoutes(
productivityReview,
successfulRunHandoff: successfulRunHandoffStates.get(issue.id) ?? null,
scheduledRetry,
blockedBy: relations.blockedBy,
blocks: relations.blocks,
activeRecoveryAction,
blockedBy: relationsWithRecoveryActions.blockedBy,
blocks: relationsWithRecoveryActions.blocks,
relatedWork: referenceSummary,
referencedIssueIdentifiers: referenceSummary.outbound.map((item) => item.issue.identifier ?? item.issue.id),
...documentPayload,
@@ -1689,6 +1815,148 @@ export function issueRoutes(
});
});
router.get("/issues/:id/recovery-actions", async (req, res) => {
const id = req.params.id as string;
const issue = await svc.getById(id);
if (!issue) {
res.status(404).json({ error: "Issue not found" });
return;
}
assertCompanyAccess(req, issue.companyId);
const active = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id);
res.json({
active,
actions: active ? [active] : [],
});
});
router.post("/issues/:id/recovery-actions/resolve", validate(resolveIssueRecoveryActionSchema), async (req, res) => {
const id = req.params.id as string;
const existing = await svc.getById(id);
if (!existing) {
res.status(404).json({ error: "Issue not found" });
return;
}
assertCompanyAccess(req, existing.companyId);
if (!(await assertAgentIssueMutationAllowed(req, res, existing))) return;
const { actionId, outcome, sourceIssueStatus, resolutionNote } = req.body;
if (outcome === "false_positive" || outcome === "cancelled") {
assertBoard(req);
}
const actor = getActorInfo(req);
const updateFields = sourceIssueStatus ? { status: sourceIssueStatus } : {};
await assertAgentInReviewReviewPath({
existing,
updateFields,
actorType: req.actor.type,
});
const actionStatus = outcome === "cancelled" ? "cancelled" : "resolved";
const result = await db.transaction(async (tx) => {
let issue = existing;
if (outcome === "blocked") {
const unresolvedBlockers = await tx
.select({ id: issueRows.id })
.from(issueRelations)
.innerJoin(issueRows, eq(issueRelations.issueId, issueRows.id))
.where(
and(
eq(issueRelations.companyId, existing.companyId),
eq(issueRelations.relatedIssueId, existing.id),
eq(issueRelations.type, "blocks"),
notInArray(issueRows.status, ["done", "cancelled"]),
),
)
.limit(1);
if (unresolvedBlockers.length === 0) {
throw unprocessable("Blocked recovery resolution requires an unresolved first-class blocker on the source issue");
}
}
if (sourceIssueStatus) {
const updatedIssue = await svc.update(
id,
{
status: sourceIssueStatus,
actorAgentId: actor.agentId ?? null,
actorUserId: actor.actorType === "user" ? actor.actorId : null,
},
tx,
);
if (!updatedIssue) throw notFound("Issue not found");
issue = updatedIssue;
}
const recoveryAction = await recoveryActionsSvc.resolveActiveForIssue(
{
companyId: existing.companyId,
sourceIssueId: existing.id,
actionId: actionId ?? null,
status: actionStatus,
outcome,
resolutionNote: resolutionNote ?? null,
},
tx,
);
if (!recoveryAction) throw notFound("Active recovery action not found");
return { issue, recoveryAction };
});
await routinesSvc.syncRunStatusForIssue(result.issue.id);
if (sourceIssueStatus && existing.status !== result.issue.status) {
await logActivity(db, {
companyId: result.issue.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
runId: actor.runId,
action: "issue.updated",
entityType: "issue",
entityId: result.issue.id,
details: {
identifier: result.issue.identifier,
status: result.issue.status,
source: "recovery_action_resolution",
recoveryActionId: result.recoveryAction.id,
_previous: {
status: existing.status,
},
},
});
}
await logActivity(db, {
companyId: result.issue.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
runId: actor.runId,
action: "issue.recovery_action_resolved",
entityType: "issue",
entityId: result.issue.id,
details: {
identifier: result.issue.identifier,
recoveryActionId: result.recoveryAction.id,
recoveryActionStatus: result.recoveryAction.status,
outcome: result.recoveryAction.outcome,
sourceIssueStatus: sourceIssueStatus ?? null,
resolutionNote: result.recoveryAction.resolutionNote,
},
});
res.json({
issue: {
...result.issue,
activeRecoveryAction: null,
},
recoveryAction: result.recoveryAction,
});
});
router.get("/issues/:id/work-products", async (req, res) => {
const id = req.params.id as string;
const issue = await svc.getById(id);
+6 -20
View File
@@ -199,9 +199,9 @@ function listBundledPluginExamples(): AvailablePluginExample[] {
*
* Lookup order:
* - UUID-like IDs: getById first, then getByKey.
* - Scoped package keys (e.g. "@scope/name"): getByKey only, never getById.
* - Other non-UUID IDs: try getById first (test/memory registries may allow this),
* then fallback to getByKey. Any UUID parse error from getById is ignored.
* - All non-UUID values: getByKey only, never getById. The persisted plugin
* ID column is a PostgreSQL UUID, so probing it with keys such as
* "acme.plugin" raises a database cast error before a key lookup can happen.
*
* @param registry - The plugin registry service instance
* @param pluginId - Either a database UUID or plugin key (manifest id)
@@ -212,27 +212,13 @@ async function resolvePlugin(
pluginId: string,
) {
const isUuid = UUID_REGEX.test(pluginId);
const isScopedPackageKey = pluginId.startsWith("@") || pluginId.includes("/");
// Scoped package IDs are valid plugin keys but invalid UUIDs.
// Skip getById() entirely to avoid Postgres uuid parse errors.
if (isScopedPackageKey && !isUuid) {
if (!isUuid) {
return registry.getByKey(pluginId);
}
try {
const byId = await registry.getById(pluginId);
if (byId) return byId;
} catch (error) {
const maybeCode =
typeof error === "object" && error !== null && "code" in error
? (error as { code?: unknown }).code
: undefined;
// Ignore invalid UUID cast errors and continue with key lookup.
if (maybeCode !== "22P02") {
throw error;
}
}
const byId = await registry.getById(pluginId);
if (byId) return byId;
return registry.getByKey(pluginId);
}
@@ -46,6 +46,7 @@ export async function resolveEnvironmentExecutionTarget(input: {
}
const parsed = await resolveEnvironmentDriverConfigForRuntime(input.db, input.companyId, {
id: input.environment.id,
driver: input.environment.driver as "sandbox",
config: parseObject(input.environment.config),
});
@@ -119,6 +120,7 @@ export async function resolveEnvironmentExecutionTarget(input: {
}
const parsed = await resolveEnvironmentDriverConfigForRuntime(input.db, input.companyId, {
id: input.environment.id,
driver: input.environment.driver as "ssh",
config: parseObject(input.environment.config),
});
+15 -5
View File
@@ -163,6 +163,7 @@ import { extractSkillMentionIds } from "@paperclipai/shared";
import { environmentService } from "./environments.js";
import { environmentRuntimeService } from "./environment-runtime.js";
import { environmentRunOrchestrator } from "./environment-run-orchestrator.js";
import { isUnsafeSessionWorkspaceCwd } from "./session-workspace-cwd.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
@@ -1691,6 +1692,7 @@ function shouldAutoCheckoutIssueForWake(input: {
const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason);
if (!wakeReason) return false;
if (wakeReason === "issue_comment_mentioned") return false;
if (wakeReason === "source_scoped_recovery_action") return false;
if (wakeReason.startsWith("execution_")) return false;
return true;
@@ -3567,7 +3569,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
}
const sessionCwd = readNonEmptyString(previousSessionParams?.cwd);
if (sessionCwd) {
const sessionCwdLooksUnsafe = isUnsafeSessionWorkspaceCwd(sessionCwd);
if (sessionCwd && !sessionCwdLooksUnsafe) {
const sessionCwdExists = await fs
.stat(sessionCwd)
.then((stats) => stats.isDirectory())
@@ -3589,7 +3592,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const cwd = resolveDefaultAgentWorkspaceDir(agent.id);
await fs.mkdir(cwd, { recursive: true });
const warnings: string[] = [];
if (sessionCwd) {
if (sessionCwd && sessionCwdLooksUnsafe) {
warnings.push(
`Saved session workspace "${sessionCwd}" points at a system temp root and was rejected as untrusted. Using fallback workspace "${cwd}" for this run.`,
);
} else if (sessionCwd) {
warnings.push(
`Saved session workspace "${sessionCwd}" is not available. Using fallback workspace "${cwd}" for this run.`,
);
@@ -5883,8 +5890,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
// Fix A (lazy locking): stamp executionRunId now that the run is actually running,
// not at queue time. Guard is idempotent — safe if called more than once.
const claimedIssueId = readNonEmptyString(parseObject(claimed.contextSnapshot).issueId);
if (claimedIssueId) {
const claimedContext = parseObject(claimed.contextSnapshot);
const claimedIssueId = readNonEmptyString(claimedContext.issueId);
const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason);
if (claimedIssueId && claimedWakeReason !== "source_scoped_recovery_action") {
const claimedAgent = await getAgent(claimed.agentId);
await db
.update(issues)
@@ -7873,7 +7882,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
});
const livenessRun = finalizedRun;
await refreshContinuationSummaryForRun(livenessRun, agent);
if (issueId && outcome === "succeeded") {
const skipRunIssueComment = parseObject(livenessRun.contextSnapshot).skipIssueComment === true;
if (issueId && outcome === "succeeded" && !skipRunIssueComment) {
try {
const existingRunComment = await findRunIssueComment(livenessRun.id, livenessRun.companyId, issueId);
if (!existingRunComment) {
+1
View File
@@ -24,6 +24,7 @@ export { issueThreadInteractionService } from "./issue-thread-interactions.js";
export { issueTreeControlService } from "./issue-tree-control.js";
export { issueApprovalService } from "./issue-approvals.js";
export { issueReferenceService } from "./issue-references.js";
export { issueRecoveryActionService } from "./issue-recovery-actions.js";
export { goalService } from "./goals.js";
export { activityService, type ActivityFilters } from "./activity.js";
export { approvalService } from "./approvals.js";
@@ -0,0 +1,295 @@
import { and, desc, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { issueRecoveryActions } from "@paperclipai/db";
import type {
IssueRecoveryAction,
IssueRecoveryActionKind,
IssueRecoveryActionOwnerType,
IssueRecoveryActionOutcome,
IssueRecoveryActionStatus,
} from "@paperclipai/shared";
const ACTIVE_RECOVERY_ACTION_STATUSES = ["active", "escalated"] as const satisfies readonly IssueRecoveryActionStatus[];
const MAX_UPSERT_RETRIES = 3;
type IssueRecoveryActionRow = typeof issueRecoveryActions.$inferSelect;
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type DbOrTransaction = Db | DbTransaction;
export type UpsertIssueRecoveryActionInput = {
companyId: string;
sourceIssueId: string;
recoveryIssueId?: string | null;
kind: IssueRecoveryActionKind;
ownerType?: IssueRecoveryActionOwnerType;
ownerAgentId?: string | null;
ownerUserId?: string | null;
previousOwnerAgentId?: string | null;
returnOwnerAgentId?: string | null;
cause: string;
fingerprint: string;
evidence?: Record<string, unknown>;
nextAction: string;
wakePolicy?: Record<string, unknown> | null;
monitorPolicy?: Record<string, unknown> | null;
maxAttempts?: number | null;
timeoutAt?: Date | null;
lastAttemptAt?: Date | null;
};
export type ResolveIssueRecoveryActionInput = {
companyId: string;
sourceIssueId: string;
actionId?: string | null;
status: Extract<IssueRecoveryActionStatus, "resolved" | "cancelled">;
outcome: IssueRecoveryActionOutcome;
resolutionNote?: string | null;
};
function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction {
return {
id: row.id,
companyId: row.companyId,
sourceIssueId: row.sourceIssueId,
recoveryIssueId: row.recoveryIssueId,
kind: row.kind as IssueRecoveryAction["kind"],
status: row.status as IssueRecoveryAction["status"],
ownerType: row.ownerType as IssueRecoveryAction["ownerType"],
ownerAgentId: row.ownerAgentId,
ownerUserId: row.ownerUserId,
previousOwnerAgentId: row.previousOwnerAgentId,
returnOwnerAgentId: row.returnOwnerAgentId,
cause: row.cause,
fingerprint: row.fingerprint,
evidence: row.evidence,
nextAction: row.nextAction,
wakePolicy: row.wakePolicy,
monitorPolicy: row.monitorPolicy,
attemptCount: row.attemptCount,
maxAttempts: row.maxAttempts,
timeoutAt: row.timeoutAt,
lastAttemptAt: row.lastAttemptAt,
outcome: row.outcome as IssueRecoveryAction["outcome"],
resolutionNote: row.resolutionNote,
resolvedAt: row.resolvedAt,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
function isUniqueRecoveryActionConflict(error: unknown) {
const maybe = error as { code?: string; constraint?: string; message?: string } | null;
return Boolean(
maybe &&
maybe.code === "23505" &&
(
maybe.constraint === "issue_recovery_actions_active_source_uq" ||
maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" ||
typeof maybe.message === "string" && (
maybe.message.includes("issue_recovery_actions_active_source_uq") ||
maybe.message.includes("issue_recovery_actions_active_fingerprint_uq")
)
),
);
}
export function issueRecoveryActionService(db: Db) {
const upsertQueues = new Map<string, Promise<void>>();
async function runExclusiveUpsert<T>(
input: UpsertIssueRecoveryActionInput,
task: () => Promise<T>,
): Promise<T> {
const key = `${input.companyId}:${input.sourceIssueId}`;
const previous = upsertQueues.get(key) ?? Promise.resolve();
let release: () => void = () => {};
const current = new Promise<void>((resolve) => {
release = resolve;
});
const next = previous.catch(() => undefined).then(() => current);
upsertQueues.set(key, next);
await previous.catch(() => undefined);
try {
return await task();
} finally {
release();
if (upsertQueues.get(key) === next) {
upsertQueues.delete(key);
}
}
}
async function getActiveForIssue(companyId: string, sourceIssueId: string): Promise<IssueRecoveryAction | null> {
const row = await db
.select()
.from(issueRecoveryActions)
.where(
and(
eq(issueRecoveryActions.companyId, companyId),
eq(issueRecoveryActions.sourceIssueId, sourceIssueId),
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
),
)
.orderBy(desc(issueRecoveryActions.updatedAt))
.limit(1)
.then((rows) => rows[0] ?? null);
return row ? toReadModel(row) : null;
}
async function listActiveForIssues(companyId: string, sourceIssueIds: string[]) {
if (sourceIssueIds.length === 0) return new Map<string, IssueRecoveryAction>();
const rows = await db
.select()
.from(issueRecoveryActions)
.where(
and(
eq(issueRecoveryActions.companyId, companyId),
inArray(issueRecoveryActions.sourceIssueId, [...new Set(sourceIssueIds)]),
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
),
)
.orderBy(desc(issueRecoveryActions.updatedAt));
const result = new Map<string, IssueRecoveryAction>();
for (const row of rows) {
if (!result.has(row.sourceIssueId)) result.set(row.sourceIssueId, toReadModel(row));
}
return result;
}
async function retryUpsertSourceScoped(
input: UpsertIssueRecoveryActionInput,
retryCount: number,
error?: unknown,
): Promise<IssueRecoveryAction> {
if (retryCount >= MAX_UPSERT_RETRIES) {
if (error) throw error;
throw new Error(
`Failed to upsert active recovery action for issue ${input.sourceIssueId} after ${MAX_UPSERT_RETRIES} retries`,
);
}
return upsertSourceScopedUnlocked(input, retryCount + 1);
}
async function upsertSourceScopedUnlocked(
input: UpsertIssueRecoveryActionInput,
retryCount = 0,
): Promise<IssueRecoveryAction> {
const existing = await getActiveForIssue(input.companyId, input.sourceIssueId);
const now = new Date();
const ownerType = input.ownerType ?? (input.ownerAgentId ? "agent" : "board");
if (existing) {
const [updated] = await db
.update(issueRecoveryActions)
.set({
recoveryIssueId: input.recoveryIssueId ?? null,
kind: input.kind,
status: "active",
ownerType,
ownerAgentId: input.ownerAgentId ?? null,
ownerUserId: input.ownerUserId ?? null,
previousOwnerAgentId: input.previousOwnerAgentId ?? existing.previousOwnerAgentId,
returnOwnerAgentId: input.returnOwnerAgentId ?? existing.returnOwnerAgentId,
cause: input.cause,
fingerprint: input.fingerprint,
evidence: input.evidence ?? existing.evidence,
nextAction: input.nextAction,
wakePolicy: input.wakePolicy ?? null,
monitorPolicy: input.monitorPolicy ?? null,
attemptCount: existing.attemptCount + 1,
maxAttempts: input.maxAttempts ?? null,
timeoutAt: input.timeoutAt ?? null,
lastAttemptAt: input.lastAttemptAt ?? now,
outcome: null,
resolutionNote: null,
resolvedAt: null,
updatedAt: now,
})
.where(
and(
eq(issueRecoveryActions.id, existing.id),
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
),
)
.returning();
if (!updated) {
return retryUpsertSourceScoped(input, retryCount);
}
return toReadModel(updated!);
}
try {
const [created] = await db
.insert(issueRecoveryActions)
.values({
companyId: input.companyId,
sourceIssueId: input.sourceIssueId,
recoveryIssueId: input.recoveryIssueId ?? null,
kind: input.kind,
status: "active",
ownerType,
ownerAgentId: input.ownerAgentId ?? null,
ownerUserId: input.ownerUserId ?? null,
previousOwnerAgentId: input.previousOwnerAgentId ?? null,
returnOwnerAgentId: input.returnOwnerAgentId ?? null,
cause: input.cause,
fingerprint: input.fingerprint,
evidence: input.evidence ?? {},
nextAction: input.nextAction,
wakePolicy: input.wakePolicy ?? null,
monitorPolicy: input.monitorPolicy ?? null,
attemptCount: 1,
maxAttempts: input.maxAttempts ?? null,
timeoutAt: input.timeoutAt ?? null,
lastAttemptAt: input.lastAttemptAt ?? now,
})
.returning();
return toReadModel(created!);
} catch (error) {
if (!isUniqueRecoveryActionConflict(error)) throw error;
return retryUpsertSourceScoped(input, retryCount, error);
}
}
async function upsertSourceScoped(
input: UpsertIssueRecoveryActionInput,
): Promise<IssueRecoveryAction> {
return runExclusiveUpsert(input, () => upsertSourceScopedUnlocked(input));
}
async function resolveActiveForIssue(
input: ResolveIssueRecoveryActionInput,
dbOrTx: DbOrTransaction = db,
): Promise<IssueRecoveryAction | null> {
const now = new Date();
const predicates = [
eq(issueRecoveryActions.companyId, input.companyId),
eq(issueRecoveryActions.sourceIssueId, input.sourceIssueId),
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
];
if (input.actionId) {
predicates.push(eq(issueRecoveryActions.id, input.actionId));
}
const [updated] = await dbOrTx
.update(issueRecoveryActions)
.set({
status: input.status,
outcome: input.outcome,
resolutionNote: input.resolutionNote ?? null,
resolvedAt: now,
updatedAt: now,
})
.where(and(...predicates))
.returning();
return updated ? toReadModel(updated) : null;
}
return {
getActiveForIssue,
listActiveForIssues,
resolveActiveForIssue,
upsertSourceScoped,
};
}
File diff suppressed because it is too large Load Diff
+18 -2
View File
@@ -164,6 +164,10 @@ export function createPluginDevWatcher(
const watchers = new Map<string, FSWatcher>();
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
const fileExists = fsDeps?.existsSync ?? existsSync;
log.info(
{ resolvesInstalledPlugins: Boolean(resolvePluginPackagePath) },
"plugin-dev-watcher: initialized",
);
function watchPlugin(pluginId: string, packagePath: string): void {
// Don't double-watch
@@ -293,11 +297,23 @@ export function createPluginDevWatcher(
}
async function watchLocalPluginById(pluginId: string): Promise<void> {
if (!resolvePluginPackagePath) return;
if (!resolvePluginPackagePath) {
log.debug(
{ pluginId },
"plugin-dev-watcher: no package path resolver configured, skipping lifecycle event",
);
return;
}
try {
const packagePath = await resolvePluginPackagePath(pluginId);
if (!packagePath) return;
if (!packagePath) {
log.debug(
{ pluginId },
"plugin-dev-watcher: plugin is not a local-path install, skipping watch",
);
return;
}
watchPlugin(pluginId, packagePath);
} catch (err) {
log.warn(
+32 -4
View File
@@ -79,6 +79,37 @@ export const DEFAULT_LOCAL_PLUGIN_DIR = path.join(
const DEV_TSX_LOADER_PATH = path.resolve(__dirname, "../../../cli/node_modules/tsx/dist/loader.mjs");
const ADAPTER_ENV_PASSTHROUGH = [
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"GOOGLE_API_KEY",
"GEMINI_API_KEY",
"OPENROUTER_API_KEY",
];
export function buildPluginWorkerEnv(input: {
manifest: Pick<PaperclipPluginManifestV1, "capabilities">;
instanceInfo: { deploymentMode?: string | null; deploymentExposure?: string | null };
processEnv?: NodeJS.ProcessEnv;
}): Record<string, string> {
const processEnv = input.processEnv ?? process.env;
const env: Record<string, string> = {
PAPERCLIP_DEPLOYMENT_MODE: input.instanceInfo.deploymentMode ?? "",
PAPERCLIP_DEPLOYMENT_EXPOSURE: input.instanceInfo.deploymentExposure ?? "",
};
const canRegisterEnvironmentDrivers = Array.isArray(input.manifest.capabilities)
&& input.manifest.capabilities.includes("environment.drivers.register");
if (!canRegisterEnvironmentDrivers) return env;
for (const key of ADAPTER_ENV_PASSTHROUGH) {
const value = processEnv[key];
if (value && value.trim().length > 0) {
env[key] = value;
}
}
return env;
}
// ---------------------------------------------------------------------------
// Discovery result types
// ---------------------------------------------------------------------------
@@ -1820,10 +1851,7 @@ export function pluginLoader(
databaseNamespace,
hostHandlers,
autoRestart: true,
env: {
PAPERCLIP_DEPLOYMENT_MODE: instanceInfo.deploymentMode ?? "",
PAPERCLIP_DEPLOYMENT_EXPOSURE: instanceInfo.deploymentExposure ?? "",
},
env: buildPluginWorkerEnv({ manifest, instanceInfo }),
};
// Repo-local plugin installs can resolve workspace TS sources at runtime
+274 -62
View File
@@ -12,10 +12,12 @@ import {
agentWakeupRequests,
approvals,
companies,
issueComments,
heartbeatRunEvents,
heartbeatRunWatchdogDecisions,
heartbeatRuns,
issueApprovals,
issueRecoveryActions,
issueRelations,
issueThreadInteractions,
issues,
@@ -29,6 +31,7 @@ import { redactSensitiveText } from "../../redaction.js";
import { logActivity } from "../activity-log.js";
import { budgetService } from "../budgets.js";
import { instanceSettingsService } from "../instance-settings.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueTreeControlService } from "../issue-tree-control.js";
import { issueService } from "../issues.js";
import { getRunLogStore } from "../run-log-store.js";
@@ -37,6 +40,7 @@ import {
FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
SUCCESSFUL_RUN_MISSING_STATE_REASON,
buildSuccessfulRunHandoffExhaustedNotice,
noticeMetadataReferencesRecoveryAction,
type SuccessfulRunHandoffNotice,
} from "./successful-run-handoff.js";
import {
@@ -386,6 +390,7 @@ function buildLivenessOriginalIssueComment(finding: IssueLivenessFinding, escala
export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) {
const issuesSvc = issueService(db);
const recoveryActionsSvc = issueRecoveryActionService(db);
const treeControlSvc = issueTreeControlService(db);
const budgets = budgetService(db);
const instanceSettings = instanceSettingsService(db);
@@ -1566,6 +1571,136 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
return recovery;
}
function strandedRecoveryActionKind(cause: StrandedRecoveryCause) {
return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
? "missing_disposition" as const
: "stranded_assigned_issue" as const;
}
function strandedRecoveryActionFingerprint(input: {
issue: typeof issues.$inferSelect;
recoveryCause: StrandedRecoveryCause;
}) {
return [
"source_scoped_recovery",
input.issue.companyId,
input.issue.id,
input.recoveryCause,
].join(":");
}
function buildStrandedRecoveryActionEvidence(input: {
issue: typeof issues.$inferSelect;
latestRun: LatestIssueRun;
previousStatus: "todo" | "in_progress";
recoveryCause: StrandedRecoveryCause;
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
}) {
const context = parseObject(input.latestRun?.contextSnapshot);
return {
sourceIssueId: input.issue.id,
sourceIdentifier: input.issue.identifier,
previousStatus: input.previousStatus,
latestIssueStatus: input.issue.status,
latestRunId: input.latestRun?.id ?? null,
latestRunStatus: input.latestRun?.status ?? null,
latestRunErrorCode: input.latestRun?.errorCode ?? null,
retryReason: readNonEmptyString(context.retryReason) ?? null,
recoveryCause: input.recoveryCause,
sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null,
correctiveRunId: input.successfulRunHandoffEvidence?.correctiveRunId ?? null,
missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null,
handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null,
maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null,
};
}
async function ensureSourceScopedStrandedRecoveryAction(input: {
issue: typeof issues.$inferSelect;
latestRun: LatestIssueRun;
previousStatus: "todo" | "in_progress";
recoveryCause?: StrandedRecoveryCause;
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
}) {
const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue);
const now = new Date();
const action = await recoveryActionsSvc.upsertSourceScoped({
companyId: input.issue.companyId,
sourceIssueId: input.issue.id,
kind: strandedRecoveryActionKind(recoveryCause),
ownerType: ownerAgentId ? "agent" : "board",
ownerAgentId,
previousOwnerAgentId: input.issue.assigneeAgentId,
returnOwnerAgentId: input.issue.assigneeAgentId,
cause: recoveryCause,
fingerprint: strandedRecoveryActionFingerprint({
issue: input.issue,
recoveryCause,
}),
evidence: buildStrandedRecoveryActionEvidence({
issue: input.issue,
latestRun: input.latestRun,
previousStatus: input.previousStatus,
recoveryCause,
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
}),
nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
? "Choose and record a valid issue disposition without copying transcript content."
: "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.",
wakePolicy: ownerAgentId
? {
type: "wake_owner",
reason: "source_scoped_recovery_action",
ownerAgentId,
}
: {
type: "board_escalation",
reason: "no_invokable_recovery_owner",
},
monitorPolicy: null,
maxAttempts: null,
lastAttemptAt: now,
});
return action;
}
async function enqueueSourceScopedStrandedRecoveryWake(input: {
action: Awaited<ReturnType<typeof recoveryActionsSvc.upsertSourceScoped>>;
issue: typeof issues.$inferSelect;
latestRun: LatestIssueRun;
recoveryCause: StrandedRecoveryCause;
}) {
if (!input.action.ownerAgentId) return;
await deps.enqueueWakeup(input.action.ownerAgentId, {
source: "assignment",
triggerDetail: "system",
reason: "source_scoped_recovery_action",
idempotencyKey: `source_scoped_recovery_action:${input.action.id}:${input.action.attemptCount}`,
payload: withRecoveryModelProfileHint({
issueId: input.issue.id,
sourceIssueId: input.issue.id,
recoveryActionId: input.action.id,
strandedRunId: input.latestRun?.id ?? null,
recoveryCause: input.recoveryCause,
}),
requestedByActorType: "system",
requestedByActorId: null,
contextSnapshot: withRecoveryModelProfileHint({
issueId: input.issue.id,
taskId: input.issue.id,
wakeReason: "source_scoped_recovery_action",
skipIssueComment: true,
source: "issue_recovery_action",
recoveryActionId: input.action.id,
sourceIssueId: input.issue.id,
strandedRunId: input.latestRun?.id ?? null,
recoveryCause: input.recoveryCause,
}),
});
}
function buildRecoveryIssueInPlaceEscalationComment(input: {
issue: typeof issues.$inferSelect;
previousStatus: "todo" | "in_progress";
@@ -1682,29 +1817,32 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
recoveryCause?: StrandedRecoveryCause;
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
}) {
const nestedRecoverySuppressed = isStrandedIssueRecoveryIssue(input.issue);
let recoveryIssue: typeof issues.$inferSelect | null = null;
if (!nestedRecoverySuppressed) {
recoveryIssue = await ensureStrandedIssueRecoveryIssue({
if (isStrandedIssueRecoveryIssue(input.issue)) {
return escalateStrandedRecoveryIssueInPlace({
issue: input.issue,
previousStatus: input.previousStatus,
latestRun: input.latestRun,
recoveryCause: input.recoveryCause,
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
});
}
const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({
issue: input.issue,
previousStatus: input.previousStatus,
latestRun: input.latestRun,
recoveryCause,
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
});
const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id);
const nextBlockerIds = recoveryIssue
? [...new Set([...blockerIds, recoveryIssue.id])]
: blockerIds;
const updated = await issuesSvc.update(input.issue.id, {
status: "blocked",
blockedByIssueIds: nextBlockerIds,
blockedByIssueIds: blockerIds,
assigneeAgentId: recoveryAction.ownerAgentId ?? input.issue.assigneeAgentId,
});
if (!updated) return null;
const prefix = await getCompanyIssuePrefix(input.issue.companyId);
const recoveryOwner = recoveryIssue?.assigneeAgentId ? await getAgent(recoveryIssue.assigneeAgentId) : null;
const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null;
const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null;
let notice: SuccessfulRunHandoffNotice | null = null;
if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON && input.successfulRunHandoffEvidence) {
@@ -1715,39 +1853,60 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
: null,
correctiveRun: input.latestRun ? { id: input.latestRun.id, status: input.latestRun.status } : null,
sourceAssignee,
recoveryIssue,
recoveryIssue: null,
recoveryActionId: recoveryAction.id,
recoveryOwner,
latestIssueStatus: input.issue.status,
latestHandoffRunStatus: input.latestRun?.status ?? "unknown",
missingDisposition: input.successfulRunHandoffEvidence.missingDisposition,
});
}
let recoveryLine: string;
if (nestedRecoverySuppressed) {
recoveryLine = await buildNestedStrandedRecoveryLine(input.issue, prefix);
} else if (recoveryIssue) {
recoveryLine = [
const recoveryLine = recoveryAction.ownerAgentId
? [
"",
`- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`,
`- Recovery action: \`${recoveryAction.id}\``,
`- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`,
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.",
].join("\n");
} else {
recoveryLine = [
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.",
].join("\n")
: [
"",
"- Recovery issue: none created because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
`- Recovery action: \`${recoveryAction.id}\``,
"- Recovery owner: board escalation, because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
"- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
].join("\n");
}
if (notice) {
await issuesSvc.addComment(input.issue.id, notice.body, {}, {
authorType: "system",
presentation: notice.presentation,
metadata: notice.metadata,
});
} else {
await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {});
if (recoveryAction.attemptCount === 1) {
const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``;
const hasEscalationComment = await db
.select({ id: issueComments.id, body: issueComments.body, metadata: issueComments.metadata })
.from(issueComments)
.where(
and(
eq(issueComments.issueId, input.issue.id),
eq(issueComments.authorType, "system"),
),
)
.orderBy(desc(issueComments.createdAt))
.limit(50)
.then((rows) => rows.some((row) =>
(row.body ?? "").includes(escalationCommentMarker) ||
noticeMetadataReferencesRecoveryAction(row.metadata, recoveryAction.id),
));
if (!hasEscalationComment) {
if (notice) {
await issuesSvc.addComment(input.issue.id, notice.body, {}, {
authorType: "system",
presentation: notice.presentation,
metadata: notice.metadata,
});
} else {
await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {}, {
authorType: "system",
});
}
}
}
await logActivity(db, {
@@ -1772,12 +1931,44 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
latestRunId: input.latestRun?.id ?? null,
latestRunStatus: input.latestRun?.status ?? null,
latestRunErrorCode: input.latestRun?.errorCode ?? null,
recoveryIssueId: recoveryIssue?.id ?? null,
nestedRecoverySuppressed,
blockerIssueIds: nextBlockerIds,
recoveryActionId: recoveryAction.id,
recoveryOwnerAgentId: recoveryAction.ownerAgentId,
previousOwnerAgentId: recoveryAction.previousOwnerAgentId,
returnOwnerAgentId: recoveryAction.returnOwnerAgentId,
blockerIssueIds: blockerIds,
},
});
await enqueueSourceScopedStrandedRecoveryWake({
action: recoveryAction,
issue: input.issue,
latestRun: input.latestRun,
recoveryCause,
});
if (recoveryAction.ownerAgentId && recoveryAction.ownerAgentId === input.issue.assigneeAgentId) {
const [currentIssue] = await db
.select({
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
})
.from(issues)
.where(eq(issues.id, input.issue.id))
.limit(1);
if (
currentIssue &&
(currentIssue.status !== "blocked" ||
currentIssue.assigneeAgentId !== recoveryAction.ownerAgentId)
) {
const reblocked = await issuesSvc.update(input.issue.id, {
status: "blocked",
blockedByIssueIds: blockerIds,
assigneeAgentId: recoveryAction.ownerAgentId,
});
if (reblocked) return reblocked;
}
}
return updated;
}
@@ -2038,6 +2229,33 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
}
async function collectIssueGraphLivenessFindings() {
const issueRowsPromise = Promise.resolve(db
.select({
id: issues.id,
companyId: issues.companyId,
identifier: issues.identifier,
title: issues.title,
status: issues.status,
projectId: issues.projectId,
goalId: issues.goalId,
parentId: issues.parentId,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
createdByAgentId: issues.createdByAgentId,
createdByUserId: issues.createdByUserId,
executionPolicy: issues.executionPolicy,
executionState: issues.executionState,
monitorNextCheckAt: issues.monitorNextCheckAt,
monitorAttemptCount: issues.monitorAttemptCount,
})
.from(issues)
.where(
and(
isNull(issues.hiddenAt),
notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]),
),
));
const [
issueRows,
relationRows,
@@ -2048,33 +2266,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
interactionRows,
approvalRows,
recoveryIssueRows,
recoveryActionRows,
] = await Promise.all([
db
.select({
id: issues.id,
companyId: issues.companyId,
identifier: issues.identifier,
title: issues.title,
status: issues.status,
projectId: issues.projectId,
goalId: issues.goalId,
parentId: issues.parentId,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
createdByAgentId: issues.createdByAgentId,
createdByUserId: issues.createdByUserId,
executionPolicy: issues.executionPolicy,
executionState: issues.executionState,
monitorNextCheckAt: issues.monitorNextCheckAt,
monitorAttemptCount: issues.monitorAttemptCount,
})
.from(issues)
.where(
and(
isNull(issues.hiddenAt),
notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]),
),
),
issueRowsPromise,
db
.select({
companyId: issueRelations.companyId,
@@ -2164,6 +2358,24 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
notInArray(issues.status, ["done", "cancelled"]),
),
),
issueRowsPromise.then((rows) => {
const issueIdsUnderAnalysis = rows.map((row) => row.id);
return issueIdsUnderAnalysis.length === 0
? []
: db
.select({
companyId: issueRecoveryActions.companyId,
issueId: issueRecoveryActions.sourceIssueId,
status: issueRecoveryActions.status,
})
.from(issueRecoveryActions)
.where(
and(
inArray(issueRecoveryActions.status, ["active", "escalated"]),
inArray(issueRecoveryActions.sourceIssueId, issueIdsUnderAnalysis),
),
);
}),
]);
const openRecoveryIssues = recoveryIssueRows.flatMap((row) => {
@@ -2217,7 +2429,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
})),
pendingInteractions: interactionRows,
pendingApprovals: approvalRows,
openRecoveryIssues,
openRecoveryIssues: openRecoveryIssues.concat(recoveryActionRows),
now: new Date(),
});
}
@@ -10,6 +10,7 @@ import {
decideSuccessfulRunHandoff,
isIdempotentFinishSuccessfulRunHandoffWakeStatus,
isSuccessfulRunHandoffRequiredNoticeBody,
noticeMetadataReferencesRecoveryAction,
} from "./successful-run-handoff.js";
const run = {
@@ -256,6 +257,7 @@ describe("successful run handoff decision", () => {
title: "Recover missing next step PAP-1",
status: "todo",
} as any,
recoveryActionId: "77777777-7777-4777-8777-777777777777",
recoveryOwner: { id: "66666666-6666-4666-8666-666666666666", name: "CTO" } as any,
latestIssueStatus: "in_progress",
latestHandoffRunStatus: "failed",
@@ -273,7 +275,7 @@ describe("successful run handoff decision", () => {
expect.objectContaining({
title: "Recovery owner",
rows: expect.arrayContaining([
expect.objectContaining({ type: "issue_link", identifier: "PAP-2" }),
expect.objectContaining({ type: "key_value", label: "Recovery action", value: "77777777-7777-4777-8777-777777777777" }),
expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CTO" }),
]),
}),
@@ -286,6 +288,8 @@ describe("successful run handoff decision", () => {
]),
}),
]));
expect(noticeMetadataReferencesRecoveryAction(notice.metadata, "77777777-7777-4777-8777-777777777777")).toBe(true);
expect(noticeMetadataReferencesRecoveryAction(notice.metadata, "88888888-8888-4888-8888-888888888888")).toBe(false);
});
it("recognizes new notices and legacy markdown headings for fallback deduplication", () => {
@@ -61,6 +61,19 @@ export type SuccessfulRunHandoffNotice = {
metadata: IssueCommentMetadata;
};
export function noticeMetadataReferencesRecoveryAction(
metadata: IssueCommentMetadata | null | undefined,
recoveryActionId: string,
) {
return (metadata?.sections ?? []).some((section) =>
section.rows.some((row) =>
row.type === "key_value" &&
row.label === "Recovery action" &&
row.value === recoveryActionId,
),
);
}
export type SuccessfulRunHandoffDecision =
| {
kind: "enqueue";
@@ -181,6 +194,7 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: {
correctiveRun: NullableNoticeRun;
sourceAssignee: NullableNoticeAgent;
recoveryIssue: NullableNoticeIssue;
recoveryActionId?: string | null;
recoveryOwner: NullableNoticeAgent;
latestIssueStatus: string;
latestHandoffRunStatus: string;
@@ -200,7 +214,9 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: {
title: "Recovery owner",
rows: [
issueLinkRow("Source issue", input.issue),
issueLinkRow("Recovery issue", input.recoveryIssue),
input.recoveryActionId
? keyValueRow("Recovery action", input.recoveryActionId)
: issueLinkRow("Recovery issue", input.recoveryIssue),
agentLinkRow("Recovery owner", input.recoveryOwner),
agentLinkRow("Source assignee", input.sourceAssignee),
keyValueRow("Suggested action", "choose and record a valid issue disposition without copying transcript content"),
@@ -0,0 +1,27 @@
import { describe, expect, it } from "vitest";
import { isUnsafeSessionWorkspaceCwd } from "./session-workspace-cwd.js";
describe("isUnsafeSessionWorkspaceCwd", () => {
it("rejects system roots that can poison remote sandbox session resumes", () => {
expect(isUnsafeSessionWorkspaceCwd("/")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/tmp")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/tmp/")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/private/tmp")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/var/tmp")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/var/run")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/proc")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/sys")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/dev")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/run")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/tmp/.")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/tmp/..")).toBe(true);
expect(isUnsafeSessionWorkspaceCwd("/var/./run")).toBe(true);
});
it("allows concrete workspace descendants", () => {
expect(isUnsafeSessionWorkspaceCwd("/tmp/paperclip-workspace")).toBe(false);
expect(isUnsafeSessionWorkspaceCwd("/Users/dotta/paperclip")).toBe(false);
expect(isUnsafeSessionWorkspaceCwd(null)).toBe(false);
});
});
@@ -0,0 +1,24 @@
import path from "node:path";
const SESSION_CWD_SYSTEM_ROOTS = new Set([
"/",
"/tmp",
"/var",
"/var/tmp",
"/var/run",
"/usr",
"/etc",
"/proc",
"/sys",
"/dev",
"/run",
"/private",
"/private/tmp",
]);
export function isUnsafeSessionWorkspaceCwd(cwd: string | null | undefined): boolean {
const value = typeof cwd === "string" && cwd.trim().length > 0 ? cwd.trim() : null;
if (!value) return false;
const normalized = path.normalize(value.replace(/\/+$/, "") || "/");
return SESSION_CWD_SYSTEM_ROOTS.has(normalized);
}