forked from farhoodlabs/paperclip
[codex] Improve transient recovery and Codex model refresh (#4383)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Adapter execution and retry classification decide whether agent work pauses, retries, or recovers automatically > - Transient provider failures need to be classified precisely so Paperclip does not convert retryable upstream conditions into false hard failures > - At the same time, operators need an up-to-date model list for Codex-backed agents and prompts should nudge agents toward targeted verification instead of repo-wide sweeps > - This pull request tightens transient recovery classification for Claude and Codex, updates the agent prompt guidance, and adds Codex model refresh support end-to-end > - The benefit is better automatic retry behavior plus fresher operator-facing model configuration ## What Changed - added Codex usage-limit retry-window parsing and Claude extra-usage transient classification - normalized the heartbeat transient-recovery contract across adapter executions and heartbeat scheduling - documented that deferred comment wakes only reopen completed issues for human/comment-reopen interactions, while system follow-ups leave closed work closed - updated adapter-utils prompt guidance to prefer targeted verification - added Codex model refresh support in the server route, registry, shared types, and agent config form - added adapter/server tests covering the new parsing, retry scheduling, and model-refresh behavior ## Verification - `pnpm exec vitest run --project @paperclipai/adapter-utils packages/adapter-utils/src/server-utils.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-claude-local packages/adapters/claude-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-codex-local packages/adapters/codex-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/adapter-model-refresh-routes.test.ts server/src/__tests__/adapter-models.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts` ## Risks - Moderate behavior risk: retry classification affects whether runs auto-recover or block, so mistakes here could either suppress needed retries or over-retry real failures - Low workflow risk: deferred comment wake reopening is intentionally scoped to human/comment-reopen interactions so system follow-ups do not revive completed issues unexpectedly > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex GPT-5-based coding agent with tool use and code execution in the Codex CLI environment ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
@@ -56,8 +56,15 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
agentId: string;
|
||||
now: Date;
|
||||
errorCode: string;
|
||||
errorFamily?: "transient_upstream" | null;
|
||||
retryNotBefore?: string | null;
|
||||
scheduledRetryAttempt?: number;
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
adapterType?: "codex_local" | "claude_local";
|
||||
agentName?: string;
|
||||
}) {
|
||||
const adapterType = input.adapterType ?? "codex_local";
|
||||
const agentName = input.agentName ?? (adapterType === "claude_local" ? "ClaudeCoder" : "CodexCoder");
|
||||
await db.insert(companies).values({
|
||||
id: input.companyId,
|
||||
name: "Paperclip",
|
||||
@@ -68,10 +75,10 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
await db.insert(agents).values({
|
||||
id: input.agentId,
|
||||
companyId: input.companyId,
|
||||
name: "CodexCoder",
|
||||
name: agentName,
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterType,
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
@@ -93,6 +100,15 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
finishedAt: input.now,
|
||||
scheduledRetryAttempt: input.scheduledRetryAttempt ?? 0,
|
||||
scheduledRetryReason: input.scheduledRetryAttempt ? "transient_failure" : null,
|
||||
resultJson: input.resultJson ?? {
|
||||
...(input.errorFamily ? { errorFamily: input.errorFamily } : {}),
|
||||
...(input.retryNotBefore
|
||||
? {
|
||||
retryNotBefore: input.retryNotBefore,
|
||||
transientRetryNotBefore: input.retryNotBefore,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
contextSnapshot: {
|
||||
issueId: randomUUID(),
|
||||
wakeReason: "issue_assigned",
|
||||
@@ -299,7 +315,8 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "codex_transient_upstream",
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
scheduledRetryAttempt: index,
|
||||
});
|
||||
|
||||
@@ -335,4 +352,110 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
await db.delete(companies);
|
||||
}
|
||||
});
|
||||
|
||||
it("honors codex retry-not-before timestamps when they exceed the default bounded backoff", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const now = new Date(2026, 3, 22, 22, 29, 0);
|
||||
const retryNotBefore = new Date(2026, 3, 22, 23, 31, 0);
|
||||
|
||||
await seedRetryFixture({
|
||||
runId,
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
retryNotBefore: retryNotBefore.toISOString(),
|
||||
});
|
||||
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
|
||||
now,
|
||||
random: () => 0.5,
|
||||
});
|
||||
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") return;
|
||||
expect(scheduled.dueAt.getTime()).toBe(retryNotBefore.getTime());
|
||||
|
||||
const retryRun = await db
|
||||
.select({
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, scheduled.run.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(retryRun?.scheduledRetryAt?.getTime()).toBe(retryNotBefore.getTime());
|
||||
expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
|
||||
const wakeupRequest = await db
|
||||
.select({ payload: agentWakeupRequests.payload })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, retryRun?.wakeupRequestId ?? ""))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect((wakeupRequest?.payload as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
});
|
||||
|
||||
it("schedules bounded retries for claude_transient_upstream and honors its retry-not-before hint", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const now = new Date(2026, 3, 22, 10, 0, 0);
|
||||
const retryNotBefore = new Date(2026, 3, 22, 16, 0, 0);
|
||||
|
||||
await seedRetryFixture({
|
||||
runId,
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
adapterType: "claude_local",
|
||||
retryNotBefore: retryNotBefore.toISOString(),
|
||||
});
|
||||
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
|
||||
now,
|
||||
random: () => 0.5,
|
||||
});
|
||||
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") return;
|
||||
expect(scheduled.dueAt.getTime()).toBe(retryNotBefore.getTime());
|
||||
|
||||
const retryRun = await db
|
||||
.select({
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, scheduled.run.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(retryRun?.scheduledRetryAt?.getTime()).toBe(retryNotBefore.getTime());
|
||||
const contextSnapshot = (retryRun?.contextSnapshot as Record<string, unknown> | null) ?? {};
|
||||
expect(contextSnapshot.transientRetryNotBefore).toBe(retryNotBefore.toISOString());
|
||||
// Claude does not participate in the Codex fallback-mode ladder.
|
||||
expect(contextSnapshot.codexTransientFallbackMode ?? null).toBeNull();
|
||||
|
||||
const wakeupRequest = await db
|
||||
.select({ payload: agentWakeupRequests.payload })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, retryRun?.wakeupRequestId ?? ""))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect((wakeupRequest?.payload as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user