forked from farhoodlabs/paperclip
[codex] Improve transient recovery and Codex model refresh (#4383)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Adapter execution and retry classification decide whether agent work pauses, retries, or recovers automatically > - Transient provider failures need to be classified precisely so Paperclip does not convert retryable upstream conditions into false hard failures > - At the same time, operators need an up-to-date model list for Codex-backed agents and prompts should nudge agents toward targeted verification instead of repo-wide sweeps > - This pull request tightens transient recovery classification for Claude and Codex, updates the agent prompt guidance, and adds Codex model refresh support end-to-end > - The benefit is better automatic retry behavior plus fresher operator-facing model configuration ## What Changed - added Codex usage-limit retry-window parsing and Claude extra-usage transient classification - normalized the heartbeat transient-recovery contract across adapter executions and heartbeat scheduling - documented that deferred comment wakes only reopen completed issues for human/comment-reopen interactions, while system follow-ups leave closed work closed - updated adapter-utils prompt guidance to prefer targeted verification - added Codex model refresh support in the server route, registry, shared types, and agent config form - added adapter/server tests covering the new parsing, retry scheduling, and model-refresh behavior ## Verification - `pnpm exec vitest run --project @paperclipai/adapter-utils packages/adapter-utils/src/server-utils.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-claude-local packages/adapters/claude-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-codex-local packages/adapters/codex-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/adapter-model-refresh-routes.test.ts server/src/__tests__/adapter-models.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts` ## Risks - Moderate behavior risk: retry classification affects whether runs auto-recover or block, so mistakes here could either suppress needed retries or over-retry real failures - Low workflow risk: deferred comment wake reopening is intentionally scoped to human/comment-reopen interactions so system follow-ups do not revive completed issues unexpectedly > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex GPT-5-based coding agent with tool use and code execution in the Codex CLI environment ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
@@ -1,9 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execute } from "@paperclipai/adapter-claude-local/server";
|
||||
|
||||
async function writeFailingClaudeCommand(
|
||||
commandPath: string,
|
||||
options: { resultEvent: Record<string, unknown>; exitCode?: number },
|
||||
): Promise<void> {
|
||||
const payload = JSON.stringify(options.resultEvent);
|
||||
const exit = options.exitCode ?? 1;
|
||||
const script = `#!/usr/bin/env node
|
||||
console.log(${JSON.stringify(payload)});
|
||||
process.exit(${exit});
|
||||
`;
|
||||
await fs.writeFile(commandPath, script, "utf8");
|
||||
await fs.chmod(commandPath, 0o755);
|
||||
}
|
||||
|
||||
async function writeFakeClaudeCommand(commandPath: string): Promise<void> {
|
||||
const script = `#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
@@ -398,8 +412,10 @@ describe("claude execute", () => {
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
|
||||
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
process.env.HOME = root;
|
||||
process.env.PAPERCLIP_HOME = paperclipHome;
|
||||
process.env.PAPERCLIP_INSTANCE_ID = "default";
|
||||
|
||||
try {
|
||||
const first = await execute({
|
||||
@@ -534,6 +550,8 @@ describe("claude execute", () => {
|
||||
else process.env.HOME = previousHome;
|
||||
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
|
||||
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -553,8 +571,10 @@ describe("claude execute", () => {
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
|
||||
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
process.env.HOME = root;
|
||||
process.env.PAPERCLIP_HOME = paperclipHome;
|
||||
process.env.PAPERCLIP_INSTANCE_ID = "default";
|
||||
|
||||
try {
|
||||
const first = await execute({
|
||||
@@ -635,7 +655,184 @@ describe("claude execute", () => {
|
||||
else process.env.HOME = previousHome;
|
||||
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
|
||||
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("classifies Claude 'out of extra usage' failures as transient upstream errors", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-transient-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error",
|
||||
session_id: "claude-session-extra",
|
||||
is_error: true,
|
||||
result: "You're out of extra usage · resets 4pm (America/Chicago)",
|
||||
errors: [{ type: "rate_limit_error", message: "You're out of extra usage" }],
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date(2026, 3, 22, 10, 15, 0));
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-transient",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("claude_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBe("2026-04-22T21:00:00.000Z");
|
||||
expect(result.resultJson?.retryNotBefore).toBe("2026-04-22T21:00:00.000Z");
|
||||
expect(result.errorMessage ?? "").toContain("extra usage");
|
||||
expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe(
|
||||
new Date("2026-04-22T21:00:00.000Z").getTime(),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies rate-limit / overloaded failures without reset metadata as transient", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-rate-limit-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error",
|
||||
session_id: "claude-session-overloaded",
|
||||
is_error: true,
|
||||
result: "Overloaded",
|
||||
errors: [{ type: "overloaded_error", message: "Overloaded_error: API is overloaded." }],
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-overloaded",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("claude_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore ?? null).toBeNull();
|
||||
expect(result.resultJson?.retryNotBefore ?? null).toBeNull();
|
||||
expect(result.resultJson?.transientRetryNotBefore ?? null).toBeNull();
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not reclassify deterministic Claude failures (auth, max turns) as transient", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-max-turns-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "claude-session-max-turns",
|
||||
is_error: true,
|
||||
result: "Maximum turns reached.",
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-max-turns",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).not.toBe("claude_transient_upstream");
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user