Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd7dce7239 | |||
| b1878c684e | |||
| 83e105393c | |||
| 49288fa5c7 | |||
| dae9e18659 | |||
| 6923597b31 | |||
| d184a1732b | |||
| be84428226 |
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "paperclip-adapter-claude-k8s",
|
||||
"version": "0.1.48",
|
||||
"version": "0.1.52",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "paperclip-adapter-claude-k8s",
|
||||
"version": "0.1.48",
|
||||
"version": "0.1.52",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@kubernetes/client-node": "^1.0.0",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "paperclip-adapter-claude-k8s",
|
||||
"version": "0.1.48",
|
||||
"version": "0.1.52",
|
||||
"description": "Paperclip adapter plugin that runs Claude Code agents as Kubernetes Jobs",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
|
||||
+24
-13
@@ -1019,7 +1019,8 @@ describe("execute: happy path", () => {
|
||||
const result = await executePromise;
|
||||
|
||||
expect(result.errorCode).toBe("k8s_job_deleted_externally");
|
||||
expect(result.errorMessage).toBe("K8s Job was deleted externally before Claude could complete");
|
||||
expect(result.errorMessage).toMatch(/^K8s Job was deleted externally before Claude could complete \[/);
|
||||
expect(result.errorMessage).toContain("detected_via=");
|
||||
expect(result.exitCode).toBeNull();
|
||||
});
|
||||
|
||||
@@ -1560,16 +1561,24 @@ describe("shouldAbortForCancellation", () => {
|
||||
expect(shouldAbortForCancellation("cancelled")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true when status is 'failed'", () => {
|
||||
expect(shouldAbortForCancellation("failed")).toBe(true);
|
||||
it("returns true when status is 'cancelling'", () => {
|
||||
expect(shouldAbortForCancellation("cancelling")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true when status is 'completed'", () => {
|
||||
expect(shouldAbortForCancellation("completed")).toBe(true);
|
||||
// FAR-107: terminal-but-not-cancelled statuses MUST NOT trigger Job deletion.
|
||||
// The previous "anything but running" guard caused k8s_job_deleted_externally
|
||||
// false positives for in-flight runs whenever the API briefly reported a
|
||||
// transient/stale status.
|
||||
it("returns false for non-cancellation terminal statuses (FAR-107)", () => {
|
||||
expect(shouldAbortForCancellation("succeeded")).toBe(false);
|
||||
expect(shouldAbortForCancellation("failed")).toBe(false);
|
||||
expect(shouldAbortForCancellation("completed")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns true for any non-running non-empty string", () => {
|
||||
expect(shouldAbortForCancellation("unknown")).toBe(true);
|
||||
it("returns false for unknown statuses (FAR-107)", () => {
|
||||
expect(shouldAbortForCancellation("unknown")).toBe(false);
|
||||
expect(shouldAbortForCancellation("queued")).toBe(false);
|
||||
expect(shouldAbortForCancellation("pending")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1770,7 +1779,7 @@ describe("execute: SIGTERM handler best-effort cleanup", () => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("deletes the active Job when SIGTERM fires during execution", async () => {
|
||||
it("does NOT delete active Jobs on SIGTERM — leaves them for orphan reattach (FAR-107)", async () => {
|
||||
// Mock process.kill to prevent the test process from actually being killed.
|
||||
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
|
||||
@@ -1781,17 +1790,19 @@ describe("execute: SIGTERM handler best-effort cleanup", () => {
|
||||
// Flush microtasks through the async setup chain: getSelfPodInfo, listJobs,
|
||||
// readSkillEntries, prepareBundle, createJob, onLog, activeJobs.add(), and
|
||||
// ensureSigtermHandler() all complete before the try block enters streaming.
|
||||
// 30 rounds is more than enough for the ~7 sequential await points.
|
||||
for (let i = 0; i < 30; i++) await Promise.resolve();
|
||||
|
||||
// Emit SIGTERM — the process.once handler fires synchronously and kicks off
|
||||
// async cleanup (deleteNamespacedJob). The mock resolves immediately.
|
||||
// Reset deleteJob spy after setup so we can detect any SIGTERM-driven calls.
|
||||
mockBatchDeleteJob.mockClear();
|
||||
|
||||
// Emit SIGTERM — the handler must re-raise to the default handler without
|
||||
// touching the K8s Job. Deleting the Job here would surface as
|
||||
// k8s_job_deleted_externally in the in-flight run (FAR-107).
|
||||
process.emit("SIGTERM");
|
||||
|
||||
// Flush microtasks for deleteJob to resolve and the .then(process.kill) to run.
|
||||
for (let i = 0; i < 10; i++) await Promise.resolve();
|
||||
|
||||
expect(mockBatchDeleteJob).toHaveBeenCalled();
|
||||
expect(mockBatchDeleteJob).not.toHaveBeenCalled();
|
||||
expect(killSpy).toHaveBeenCalledWith(process.pid, "SIGTERM");
|
||||
|
||||
killSpy.mockRestore();
|
||||
|
||||
+198
-43
@@ -58,30 +58,20 @@ function ensureSigtermHandler(): void {
|
||||
if (sigtermHandlerRegistered) return;
|
||||
sigtermHandlerRegistered = true;
|
||||
process.once("SIGTERM", () => {
|
||||
const jobs = [...activeJobs];
|
||||
void Promise.allSettled(
|
||||
jobs.map(async (ref) => {
|
||||
try {
|
||||
const batchApi = getBatchApi(ref.kubeconfigPath);
|
||||
await batchApi.deleteNamespacedJob({
|
||||
name: ref.jobName,
|
||||
namespace: ref.namespace,
|
||||
body: { propagationPolicy: "Background" },
|
||||
});
|
||||
} catch { /* best-effort */ }
|
||||
if (ref.promptSecretName && ref.promptSecretNamespace) {
|
||||
try {
|
||||
const coreApi = getCoreApi(ref.kubeconfigPath);
|
||||
await coreApi.deleteNamespacedSecret({
|
||||
name: ref.promptSecretName,
|
||||
namespace: ref.promptSecretNamespace,
|
||||
});
|
||||
} catch { /* best-effort */ }
|
||||
}
|
||||
}),
|
||||
).then(() => {
|
||||
process.kill(process.pid, "SIGTERM");
|
||||
});
|
||||
// Do NOT delete active K8s Jobs on SIGTERM (FAR-107). Paperclip itself
|
||||
// receives SIGTERM during rolling deploys, evictions, scale-down, etc.
|
||||
// Deleting the Jobs we own there causes the in-flight heartbeat to surface
|
||||
// a false-positive `k8s_job_deleted_externally` error and tears down work
|
||||
// the user expected to keep running.
|
||||
//
|
||||
// The correct behaviour with `reattachOrphanedJobs=true` (default) is to
|
||||
// leave the Jobs alive: the next paperclip process discovers them via the
|
||||
// orphan-classification path and reattaches their log streams. When
|
||||
// `reattachOrphanedJobs=false` the operator explicitly opted into manual
|
||||
// cleanup and should not have us auto-deleting either. The owning Job's
|
||||
// ownerReference (FAR-15) keeps the prompt Secret tied to the Job, so
|
||||
// both survive together and TTL cleans them up after natural completion.
|
||||
process.kill(process.pid, "SIGTERM");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -100,13 +90,23 @@ export function isK8s404(err: unknown): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when the heartbeat-run status indicates the run is no longer
|
||||
* active and the K8s Job should be cancelled.
|
||||
* Returns true when the heartbeat-run status indicates the run was explicitly
|
||||
* cancelled and the K8s Job must be torn down.
|
||||
*
|
||||
* Only `cancelled` / `cancelling` qualify. Treating any non-`running` status
|
||||
* as cancellation (the previous behaviour) produced spurious
|
||||
* k8s_job_deleted_externally errors for in-flight runs whenever the API
|
||||
* briefly reported a transient or stale status — Nancy's runs at
|
||||
* Privileged Escalation hit this without anyone actually cancelling them
|
||||
* (FAR-107). Other terminal statuses (`succeeded`/`failed`/`completed`)
|
||||
* are unreachable in practice while the adapter is still executing
|
||||
* (the adapter's own return is what flips them) and even if observed,
|
||||
* they do not warrant our deleting a Job that may still be doing work.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function shouldAbortForCancellation(runStatus: string | undefined): boolean {
|
||||
if (!runStatus) return false;
|
||||
return runStatus !== "running";
|
||||
return runStatus === "cancelled" || runStatus === "cancelling";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -582,11 +582,14 @@ async function readPodLogs(
|
||||
* is treated as a soft terminal: succeeded=false, timedOut=false, jobGone=true.
|
||||
* The caller should log this and fall through to stdout parsing.
|
||||
*/
|
||||
type JobConditionSnapshot = { type?: string; status?: string; reason?: string; message?: string };
|
||||
|
||||
async function waitForJobCompletion(
|
||||
namespace: string,
|
||||
jobName: string,
|
||||
timeoutMs: number,
|
||||
kubeconfigPath?: string,
|
||||
observer?: { lastConditions: JobConditionSnapshot[] | null; pollCount: number },
|
||||
): Promise<{ succeeded: boolean; timedOut: boolean; jobGone?: boolean }> {
|
||||
const batchApi = getBatchApi(kubeconfigPath);
|
||||
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : 0;
|
||||
@@ -605,6 +608,12 @@ async function waitForJobCompletion(
|
||||
throw err;
|
||||
}
|
||||
const conditions = job.status?.conditions ?? [];
|
||||
if (observer) {
|
||||
observer.pollCount += 1;
|
||||
observer.lastConditions = conditions.map((c) => ({
|
||||
type: c.type, status: c.status, reason: c.reason, message: c.message,
|
||||
}));
|
||||
}
|
||||
|
||||
const complete = conditions.find((c) => c.type === "Complete" && c.status === "True");
|
||||
if (complete) return { succeeded: true, timedOut: false };
|
||||
@@ -641,30 +650,82 @@ export interface PodTerminatedState {
|
||||
signal: number | null;
|
||||
}
|
||||
|
||||
async function getPodTerminatedState(
|
||||
/**
|
||||
* Result of a pod-state lookup. `state` is the terminated state when available;
|
||||
* `phase` and `podMissing` give the caller enough context to render an honest
|
||||
* truncation-cause message instead of guessing "likely deleted" (FAR-107).
|
||||
*/
|
||||
export interface PodLookupResult {
|
||||
state: PodTerminatedState | null;
|
||||
phase: string | null;
|
||||
podMissing: boolean;
|
||||
}
|
||||
|
||||
async function lookupPodState(
|
||||
namespace: string,
|
||||
jobName: string,
|
||||
kubeconfigPath?: string,
|
||||
): Promise<PodTerminatedState | null> {
|
||||
): Promise<PodLookupResult> {
|
||||
const coreApi = getCoreApi(kubeconfigPath);
|
||||
const podList = await coreApi.listNamespacedPod({
|
||||
namespace,
|
||||
labelSelector: `job-name=${jobName}`,
|
||||
});
|
||||
const pod = podList.items[0];
|
||||
if (!pod) return null;
|
||||
if (!pod) return { state: null, phase: null, podMissing: true };
|
||||
|
||||
const phase = pod.status?.phase ?? null;
|
||||
const containerStatus = pod.status?.containerStatuses?.find((s) => s.name === "claude");
|
||||
const terminated = containerStatus?.state?.terminated;
|
||||
if (!terminated) return null;
|
||||
if (!terminated) return { state: null, phase, podMissing: false };
|
||||
return {
|
||||
exitCode: terminated.exitCode ?? null,
|
||||
reason: terminated.reason ?? null,
|
||||
message: (terminated.message ?? "").trim() || null,
|
||||
signal: terminated.signal ?? null,
|
||||
state: {
|
||||
exitCode: terminated.exitCode ?? null,
|
||||
reason: terminated.reason ?? null,
|
||||
message: (terminated.message ?? "").trim() || null,
|
||||
signal: terminated.signal ?? null,
|
||||
},
|
||||
phase,
|
||||
podMissing: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the claude container's terminated state, retrying briefly when the pod
|
||||
* exists in a terminal phase but kubelet has not yet propagated the
|
||||
* containerStatuses[].state.terminated field. Without this retry, fast
|
||||
* truncated-stream exits surface as "pod state unavailable" (FAR-107) and
|
||||
* mask the real exit code / OOMKilled / SIGTERM cause.
|
||||
*/
|
||||
async function getPodLookupWithRetry(
|
||||
namespace: string,
|
||||
jobName: string,
|
||||
kubeconfigPath?: string,
|
||||
attempts = 4,
|
||||
delayMs = 500,
|
||||
): Promise<PodLookupResult> {
|
||||
let last: PodLookupResult = { state: null, phase: null, podMissing: true };
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
last = await lookupPodState(namespace, jobName, kubeconfigPath);
|
||||
if (last.state) return last;
|
||||
if (last.podMissing) return last;
|
||||
// Pod exists but no terminated state. If it is in a terminal phase the
|
||||
// containerStatuses update is in flight — wait briefly and retry. If it
|
||||
// is still Running/Pending, retrying is unlikely to help, so bail.
|
||||
if (last.phase !== "Succeeded" && last.phase !== "Failed") return last;
|
||||
if (i < attempts - 1) await new Promise((r) => setTimeout(r, delayMs));
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
async function getPodTerminatedState(
|
||||
namespace: string,
|
||||
jobName: string,
|
||||
kubeconfigPath?: string,
|
||||
): Promise<PodTerminatedState | null> {
|
||||
return (await lookupPodState(namespace, jobName, kubeconfigPath)).state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a human-readable explanation for a truncated run, including the
|
||||
* pod's claude-container terminated state when available. Exit code 137
|
||||
@@ -673,9 +734,17 @@ async function getPodTerminatedState(
|
||||
*/
|
||||
export function describeTruncationCause(
|
||||
state: PodTerminatedState | null,
|
||||
lookup?: PodLookupResult,
|
||||
): string {
|
||||
if (!state) {
|
||||
return "pod state unavailable — likely deleted before exit could be read";
|
||||
if (lookup?.podMissing) {
|
||||
return "pod is gone — Job pod was removed (eviction, preemption, or external delete) before exit could be read";
|
||||
}
|
||||
if (lookup && !lookup.podMissing) {
|
||||
const phaseHint = lookup.phase ? `pod phase=${lookup.phase}` : "pod present";
|
||||
return `container terminated state not yet observable (${phaseHint}) — kubelet status update did not land within retry window; exit cause unknown`;
|
||||
}
|
||||
return "pod state unavailable — exit cause unknown";
|
||||
}
|
||||
const parts: string[] = [];
|
||||
if (state.exitCode !== null) {
|
||||
@@ -1112,6 +1181,17 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// Set when the job disappeared (404) or grace-timer fired before we saw a
|
||||
// terminal condition — used to emit a clearer error when stdout parsing fails.
|
||||
let jobDeletedExternally = false;
|
||||
// Forensics for k8s_job_deleted_externally — captures which of the three
|
||||
// detection paths observed the 404, the last successful Job-condition read
|
||||
// before deletion, and timing. Surfaced in the error message so the next
|
||||
// occurrence is self-diagnosing instead of opaque (FAR-107).
|
||||
let jobGoneDetectionPath: string | null = null;
|
||||
let jobGoneAt: number | null = null;
|
||||
const jobObserver: { lastConditions: JobConditionSnapshot[] | null; pollCount: number } = {
|
||||
lastConditions: null,
|
||||
pollCount: 0,
|
||||
};
|
||||
let podRunningAt: number | null = null;
|
||||
|
||||
const activeJobRef: ActiveJobRef = {
|
||||
namespace,
|
||||
@@ -1144,6 +1224,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
podName = await waitForPod(namespace, jobName, scheduleTimeoutMs, onLog, kubeconfigPath);
|
||||
await onLog("stdout", `[paperclip] Pod running: ${podName}\n`);
|
||||
}
|
||||
podRunningAt = Date.now();
|
||||
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
@@ -1279,7 +1360,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// while streamPodLogs reconnects, holding execute() open for minutes.
|
||||
// logStopSignal.stopped is set on every settled path (fulfilled, rejected,
|
||||
// or grace) so streamPodLogs stops reconnecting promptly.
|
||||
type CompletionResult = { succeeded: boolean; timedOut: boolean; jobGone?: boolean };
|
||||
type CompletionResult = { succeeded: boolean; timedOut: boolean; jobGone?: boolean; gracePeriodFired?: boolean };
|
||||
let gracePoller: ReturnType<typeof setInterval> | null = null;
|
||||
const completionWithGrace = new Promise<CompletionResult>((resolve, reject) => {
|
||||
let settled = false;
|
||||
@@ -1297,11 +1378,37 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
logStopSignal.stopped = true;
|
||||
reject(err);
|
||||
};
|
||||
waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath).then(settleOk).catch(settleErr);
|
||||
waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath, jobObserver).then(settleOk).catch(settleErr);
|
||||
gracePoller = setInterval(() => {
|
||||
if (logExitTime !== null && Date.now() - logExitTime >= LOG_EXIT_COMPLETION_GRACE_MS) {
|
||||
void onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago without K8s Job condition update — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
||||
settleOk({ succeeded: false, timedOut: false, jobGone: true });
|
||||
// Stop the grace poller immediately so we don't double-fire while the
|
||||
// verification read below is in flight.
|
||||
if (gracePoller) { clearInterval(gracePoller); gracePoller = null; }
|
||||
// The log stream exiting only means the container stopped producing
|
||||
// output — it does NOT prove the Job was deleted. Verify Job
|
||||
// presence with a one-shot read so we can distinguish:
|
||||
// (a) Job 404 → truly gone (TTL or external deletion)
|
||||
// (b) Job still present → K8s condition propagation lag (FAR-23)
|
||||
// Without this check we mis-classify (b) as "deleted externally" and
|
||||
// emit a false-positive k8s_job_deleted_externally error (FAR-107).
|
||||
void (async () => {
|
||||
try {
|
||||
await getBatchApi(kubeconfigPath).readNamespacedJob({ name: jobName, namespace });
|
||||
await onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago without K8s Job condition update; Job ${jobName} still present — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
||||
settleOk({ succeeded: false, timedOut: false, gracePeriodFired: true });
|
||||
} catch (err: unknown) {
|
||||
if (isK8s404(err)) {
|
||||
jobGoneDetectionPath = "grace-period-verify-404";
|
||||
jobGoneAt = Date.now();
|
||||
await onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago and Job ${jobName} is gone (TTL or external deletion) — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
||||
settleOk({ succeeded: false, timedOut: false, jobGone: true });
|
||||
} else {
|
||||
// K8s API hiccup — bail out without claiming external deletion.
|
||||
await onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago; Job state unverifiable (${err instanceof Error ? err.message : String(err)}) — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
||||
settleOk({ succeeded: false, timedOut: false, gracePeriodFired: true });
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
}, 1_000);
|
||||
});
|
||||
@@ -1369,6 +1476,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// completion), so log streaming has captured the full output — continue
|
||||
// to stdout parsing rather than returning an error.
|
||||
jobDeletedExternally = true;
|
||||
if (!jobGoneDetectionPath) {
|
||||
jobGoneDetectionPath = "completion-poll-404";
|
||||
jobGoneAt = Date.now();
|
||||
}
|
||||
await onLog("stdout", `[paperclip] Job ${jobName} was deleted before terminal condition was observed (TTL or external deletion) — proceeding with captured output.\n`);
|
||||
}
|
||||
} else {
|
||||
@@ -1377,7 +1488,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// (60s) so we don't hang the heartbeat indefinitely if the K8s API is degraded.
|
||||
jobTimedOut = false;
|
||||
const RECHECK_TIMEOUT_MS = 60_000;
|
||||
const actualState = await waitForJobCompletion(namespace, jobName, RECHECK_TIMEOUT_MS, kubeconfigPath);
|
||||
const actualState = await waitForJobCompletion(namespace, jobName, RECHECK_TIMEOUT_MS, kubeconfigPath, jobObserver);
|
||||
if (actualState.timedOut) {
|
||||
// Re-check itself timed out — the job may still be running.
|
||||
// Return an error so the UI knows the run is not done.
|
||||
@@ -1386,6 +1497,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// Job was deleted before we could confirm terminal state — same as the
|
||||
// fulfilled+jobGone case above: proceed with captured output.
|
||||
jobDeletedExternally = true;
|
||||
if (!jobGoneDetectionPath) {
|
||||
jobGoneDetectionPath = "recheck-poll-404";
|
||||
jobGoneAt = Date.now();
|
||||
}
|
||||
await onLog("stdout", `[paperclip] Job ${jobName} was deleted before terminal condition was observed (TTL or external deletion) — proceeding with captured output.\n`);
|
||||
} else if (!actualState.succeeded) {
|
||||
// Job still not terminal — the completion error was likely transient.
|
||||
@@ -1455,11 +1570,35 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
|
||||
if (!parsed) {
|
||||
if (jobDeletedExternally && exitCode === null) {
|
||||
// Forensic context (FAR-107): users sometimes see this error when nothing
|
||||
// actually deleted the Job manually. Surface enough state in the message
|
||||
// to distinguish self-delete (SIGTERM/cancel), TTL-after-completion, and
|
||||
// genuine external deletion without needing cluster shell access.
|
||||
const detailParts: string[] = [];
|
||||
if (jobGoneDetectionPath) detailParts.push(`detected_via=${jobGoneDetectionPath}`);
|
||||
detailParts.push(`job=${jobName}`);
|
||||
detailParts.push(`ns=${namespace}`);
|
||||
if (podRunningAt !== null && jobGoneAt !== null) {
|
||||
detailParts.push(`elapsed_since_pod_running=${Math.round((jobGoneAt - podRunningAt) / 1000)}s`);
|
||||
}
|
||||
detailParts.push(`completion_polls=${jobObserver.pollCount}`);
|
||||
const lastConds = jobObserver.lastConditions;
|
||||
if (lastConds && lastConds.length > 0) {
|
||||
const summary = lastConds
|
||||
.map((c) => `${c.type}=${c.status}${c.reason ? `(${c.reason})` : ""}`)
|
||||
.join(",");
|
||||
detailParts.push(`last_job_conditions=[${summary}]`);
|
||||
} else {
|
||||
detailParts.push("last_job_conditions=none_observed");
|
||||
}
|
||||
detailParts.push(`stdout_bytes=${stdout.length}`);
|
||||
const stdoutLines = stdout.split("\n").filter((l) => l.trim()).length;
|
||||
detailParts.push(`stdout_nonempty_lines=${stdoutLines}`);
|
||||
return {
|
||||
exitCode,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: "K8s Job was deleted externally before Claude could complete",
|
||||
errorMessage: `K8s Job was deleted externally before Claude could complete [${detailParts.join(", ")}]`,
|
||||
errorCode: "k8s_job_deleted_externally",
|
||||
resultJson: { stdout },
|
||||
};
|
||||
@@ -1475,7 +1614,23 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
};
|
||||
}
|
||||
if (parsedStream.truncatedMidStream) {
|
||||
const cause = describeTruncationCause(podTerminatedState);
|
||||
// Re-query pod state with retry — the initial single-shot read can lose
|
||||
// to kubelet propagation lag and surface a useless "pod state unavailable"
|
||||
// message that hides the real exit cause (OOMKilled, SIGTERM, etc). The
|
||||
// retry distinguishes pod-genuinely-gone from terminated-state-lag and
|
||||
// gives the operator the actual exit code/reason where possible (FAR-107).
|
||||
let lookup: PodLookupResult | undefined;
|
||||
let refreshedState = podTerminatedState;
|
||||
try {
|
||||
lookup = await getPodLookupWithRetry(namespace, jobName, kubeconfigPath);
|
||||
refreshedState = lookup.state;
|
||||
if (refreshedState && refreshedState.exitCode !== null) {
|
||||
exitCode = refreshedState.exitCode;
|
||||
}
|
||||
} catch (err) {
|
||||
await onLog("stderr", `[paperclip] truncation diagnostic: pod re-query failed (${err instanceof Error ? err.message : String(err)})\n`).catch(() => {});
|
||||
}
|
||||
const cause = describeTruncationCause(refreshedState, lookup);
|
||||
const modelHint = parsedStream.model ? ` (model: ${parsedStream.model})` : "";
|
||||
return {
|
||||
exitCode,
|
||||
|
||||
Reference in New Issue
Block a user