Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1d894f104f | |||
| fc3866924a | |||
| 368254d75d | |||
| 34756f8215 | |||
| 07ef106c66 |
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "paperclip-adapter-claude-k8s",
|
"name": "paperclip-adapter-claude-k8s",
|
||||||
"version": "0.1.52",
|
"version": "0.1.54",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "paperclip-adapter-claude-k8s",
|
"name": "paperclip-adapter-claude-k8s",
|
||||||
"version": "0.1.52",
|
"version": "0.1.54",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@kubernetes/client-node": "^1.0.0",
|
"@kubernetes/client-node": "^1.0.0",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "paperclip-adapter-claude-k8s",
|
"name": "paperclip-adapter-claude-k8s",
|
||||||
"version": "0.1.52",
|
"version": "0.1.55",
|
||||||
"description": "Paperclip adapter plugin that runs Claude Code agents as Kubernetes Jobs",
|
"description": "Paperclip adapter plugin that runs Claude Code agents as Kubernetes Jobs",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"repository": {
|
"repository": {
|
||||||
|
|||||||
+2
-1
@@ -1,7 +1,8 @@
|
|||||||
export const type = "claude_k8s";
|
export const type = "claude_k8s";
|
||||||
export const label = "Claude (Kubernetes)";
|
export const label = "Claude (Kubernetes)";
|
||||||
|
|
||||||
export const models: undefined = undefined;
|
import { DIRECT_MODELS, BEDROCK_MODELS, isBedrockEnv } from "./server/models.js";
|
||||||
|
export const models = isBedrockEnv() ? BEDROCK_MODELS : DIRECT_MODELS;
|
||||||
|
|
||||||
export const agentConfigurationDoc = `# claude_k8s agent configuration
|
export const agentConfigurationDoc = `# claude_k8s agent configuration
|
||||||
|
|
||||||
|
|||||||
@@ -1512,6 +1512,54 @@ describe("execute: log-stream-exit grace period (FAR-23)", () => {
|
|||||||
// (grace did not fire, real completion arrived)
|
// (grace did not fire, real completion arrived)
|
||||||
expect(result.errorMessage).toBeNull();
|
expect(result.errorMessage).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does NOT fire grace when stream drops mid-output and reconnects with more output (FAR-107)", async () => {
|
||||||
|
// Reproduces Nancy / Privileged Escalation symptom: the K8s log API drops
|
||||||
|
// the streaming connection mid-run; streamPodLogs reconnects and the
|
||||||
|
// container is still producing. Before the fix, the grace timer was
|
||||||
|
// armed on first stream exit and fired 30s later regardless of whether
|
||||||
|
// output had resumed, surfacing claude_truncated even though the pod was
|
||||||
|
// still phase=Running.
|
||||||
|
let attemptIndex = 0;
|
||||||
|
mockLogFn.mockImplementation(
|
||||||
|
async (_ns: string, _pod: string, _ctr: string, writable: import("node:stream").Writable) => {
|
||||||
|
if (attemptIndex === 0) {
|
||||||
|
// Stream a partial init line then "drop" the connection without a
|
||||||
|
// result event — this is the transient API disconnect.
|
||||||
|
writable.write(JSON.stringify({ type: "system", subtype: "init", model: "claude-sonnet-4-6", session_id: "sess_test123" }) + "\n");
|
||||||
|
attemptIndex++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Reconnect produces the rest of the stream including the result event.
|
||||||
|
writable.write(CLAUDE_HAPPY_OUTPUT);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// Job condition arrives only after the reconnect produces output, well
|
||||||
|
// beyond the 30s grace window; the old code would have grace-fired at
|
||||||
|
// ~30s and treated the run as truncated.
|
||||||
|
let readJobCalls = 0;
|
||||||
|
mockBatchReadJob.mockImplementation(async () => {
|
||||||
|
readJobCalls++;
|
||||||
|
// Stay non-terminal until the reconnect has had time to run and the
|
||||||
|
// grace window has fully elapsed since the FIRST disconnect.
|
||||||
|
if (readJobCalls < 25) return { status: { conditions: [] } };
|
||||||
|
return { status: { conditions: [{ type: "Complete", status: "True" }] } };
|
||||||
|
});
|
||||||
|
|
||||||
|
const executePromise = execute(makeCtx());
|
||||||
|
// t=3000: first reconnect sleep fires → second streamPodLogsOnce attempt
|
||||||
|
await vi.advanceTimersByTimeAsync(3_100);
|
||||||
|
// Drive past the old (buggy) 30s grace boundary without firing real completion
|
||||||
|
await vi.advanceTimersByTimeAsync(35_000);
|
||||||
|
// Then let the Job's Complete condition land
|
||||||
|
await vi.advanceTimersByTimeAsync(20_000);
|
||||||
|
const result = await executePromise;
|
||||||
|
|
||||||
|
// Run completed normally — grace must not have falsely truncated it.
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
expect(result.errorCode).toBeUndefined();
|
||||||
|
expect(result.sessionId).toBe("sess_test123");
|
||||||
|
}, 80_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
// ─── execute: concurrency guard — multiple orphan sorting ────────────────────
|
// ─── execute: concurrency guard — multiple orphan sorting ────────────────────
|
||||||
|
|||||||
+76
-29
@@ -401,6 +401,7 @@ export async function streamPodLogsOnce(
|
|||||||
sinceSeconds?: number,
|
sinceSeconds?: number,
|
||||||
dedup?: LogLineDedupFilter,
|
dedup?: LogLineDedupFilter,
|
||||||
stopSignal?: { stopped: boolean },
|
stopSignal?: { stopped: boolean },
|
||||||
|
activity?: { lastActiveAt: number },
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
const logApi = getLogApi(kubeconfigPath);
|
const logApi = getLogApi(kubeconfigPath);
|
||||||
const chunks: string[] = [];
|
const chunks: string[] = [];
|
||||||
@@ -409,6 +410,13 @@ export async function streamPodLogsOnce(
|
|||||||
write(chunk: Buffer, _encoding, callback) {
|
write(chunk: Buffer, _encoding, callback) {
|
||||||
const text = chunk.toString("utf-8");
|
const text = chunk.toString("utf-8");
|
||||||
chunks.push(text);
|
chunks.push(text);
|
||||||
|
// Refresh stream liveness on every chunk received from the container.
|
||||||
|
// This MUST happen here (not just after streamPodLogsOnce returns) —
|
||||||
|
// a streaming attempt that never disconnects can produce output for
|
||||||
|
// hours, and the grace timer in execute() will fire 30s after the
|
||||||
|
// FIRST disconnect even if a new long-running attempt is currently
|
||||||
|
// streaming, unless we keep this timestamp fresh per-chunk (FAR-107).
|
||||||
|
if (activity) activity.lastActiveAt = Date.now();
|
||||||
const emitted = dedup ? dedup.filter(text) : text;
|
const emitted = dedup ? dedup.filter(text) : text;
|
||||||
if (!emitted) {
|
if (!emitted) {
|
||||||
callback();
|
callback();
|
||||||
@@ -481,10 +489,18 @@ export async function streamPodLogsOnce(
|
|||||||
* Capped at MAX_LOG_RECONNECT_ATTEMPTS to prevent infinite reconnect
|
* Capped at MAX_LOG_RECONNECT_ATTEMPTS to prevent infinite reconnect
|
||||||
* loops during sustained API partitions.
|
* loops during sustained API partitions.
|
||||||
*
|
*
|
||||||
* onFirstStreamExit is called the first time streamPodLogsOnce returns
|
* `activity` tracks stream liveness so execute()'s grace timer can
|
||||||
* (container has exited or stream disconnected). Used by execute() to
|
* distinguish a transient K8s log-API reconnect from a real container
|
||||||
* start the LOG_EXIT_COMPLETION_GRACE_MS grace timer (FAR-23) without
|
* exit (FAR-107). Two signals:
|
||||||
* waiting for all reconnects to exhaust.
|
* - `streamHasExited` becomes true on the first return from
|
||||||
|
* streamPodLogsOnce. Until then we are still in the warm-up window
|
||||||
|
* and waitForJobCompletion is the authoritative signal — grace must
|
||||||
|
* not fire.
|
||||||
|
* - `lastActiveAt` advances every time a streamPodLogsOnce attempt
|
||||||
|
* returns non-empty output (the container is still producing).
|
||||||
|
* The grace timer fires only once GRACE_MS have passed since the
|
||||||
|
* last chunk, so output that resumes after a transient drop keeps
|
||||||
|
* the run alive.
|
||||||
*/
|
*/
|
||||||
async function streamPodLogs(
|
async function streamPodLogs(
|
||||||
namespace: string,
|
namespace: string,
|
||||||
@@ -493,7 +509,7 @@ async function streamPodLogs(
|
|||||||
kubeconfigPath?: string,
|
kubeconfigPath?: string,
|
||||||
stopSignal?: { stopped: boolean },
|
stopSignal?: { stopped: boolean },
|
||||||
dedup?: LogLineDedupFilter,
|
dedup?: LogLineDedupFilter,
|
||||||
onFirstStreamExit?: () => void,
|
activity?: { lastActiveAt: number; streamHasExited: boolean },
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
const allChunks: string[] = [];
|
const allChunks: string[] = [];
|
||||||
let attempt = 0;
|
let attempt = 0;
|
||||||
@@ -523,15 +539,16 @@ async function streamPodLogs(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const preStreamTs = Math.floor(Date.now() / 1000);
|
const preStreamTs = Math.floor(Date.now() / 1000);
|
||||||
const result = await streamPodLogsOnce(namespace, podName, onLog, kubeconfigPath, sinceSeconds, dedup, stopSignal);
|
const result = await streamPodLogsOnce(namespace, podName, onLog, kubeconfigPath, sinceSeconds, dedup, stopSignal, activity);
|
||||||
// Signal first stream exit immediately so the grace-period timer in
|
if (activity) activity.streamHasExited = true;
|
||||||
// execute() can start without waiting for all reconnects to complete.
|
|
||||||
if (attempt === 0) onFirstStreamExit?.();
|
|
||||||
if (result) {
|
if (result) {
|
||||||
allChunks.push(result);
|
allChunks.push(result);
|
||||||
// Update last-received timestamp to now (the stream just ended,
|
// Update last-received timestamp to now (the stream just ended,
|
||||||
// so any log lines in `result` were received up to this moment).
|
// so any log lines in `result` were received up to this moment).
|
||||||
lastLogReceivedAt = Math.floor(Date.now() / 1000);
|
lastLogReceivedAt = Math.floor(Date.now() / 1000);
|
||||||
|
// Refresh stream liveness so the grace timer in execute() does not
|
||||||
|
// fire while output is still flowing through reconnects (FAR-107).
|
||||||
|
if (activity) activity.lastActiveAt = Date.now();
|
||||||
} else if (attempt === 0) {
|
} else if (attempt === 0) {
|
||||||
// First attempt returned nothing — update timestamp so reconnect
|
// First attempt returned nothing — update timestamp so reconnect
|
||||||
// window stays reasonable.
|
// window stays reasonable.
|
||||||
@@ -1340,17 +1357,16 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||||||
return onLog(stream, chunk);
|
return onLog(stream, chunk);
|
||||||
};
|
};
|
||||||
|
|
||||||
// Track when the log stream first exits so the grace-period can fire
|
// Track stream liveness so the grace timer below only fires when output
|
||||||
// if the K8s Job condition lags behind container exit (FAR-23).
|
// has actually stopped — not on a transient K8s log-API reconnect that
|
||||||
// Set via onFirstStreamExit callback (called after attempt=0 returns)
|
// streamPodLogs heals on its own (FAR-107).
|
||||||
// rather than in .then() of streamPodLogs, which would create a
|
const streamActivity: { lastActiveAt: number; streamHasExited: boolean } = {
|
||||||
// deadlock: streamPodLogs only resolves after stopSignal is set, but
|
lastActiveAt: Date.now(),
|
||||||
// stopSignal is set by the grace timer which needs logExitTime to be
|
streamHasExited: false,
|
||||||
// non-null.
|
};
|
||||||
let logExitTime: number | null = null;
|
|
||||||
const trackedLogStream = streamPodLogs(
|
const trackedLogStream = streamPodLogs(
|
||||||
namespace, podName, wrappedOnLog, kubeconfigPath, logStopSignal, logDedup,
|
namespace, podName, wrappedOnLog, kubeconfigPath, logStopSignal, logDedup,
|
||||||
() => { logExitTime = Date.now(); },
|
streamActivity,
|
||||||
);
|
);
|
||||||
|
|
||||||
// completionWithGrace races waitForJobCompletion against a grace timer
|
// completionWithGrace races waitForJobCompletion against a grace timer
|
||||||
@@ -1379,19 +1395,50 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||||||
reject(err);
|
reject(err);
|
||||||
};
|
};
|
||||||
waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath, jobObserver).then(settleOk).catch(settleErr);
|
waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath, jobObserver).then(settleOk).catch(settleErr);
|
||||||
|
let graceCheckInFlight = false;
|
||||||
gracePoller = setInterval(() => {
|
gracePoller = setInterval(() => {
|
||||||
if (logExitTime !== null && Date.now() - logExitTime >= LOG_EXIT_COMPLETION_GRACE_MS) {
|
// Only consider grace once the stream has exited at least once.
|
||||||
// Stop the grace poller immediately so we don't double-fire while the
|
// Until then we are still in the warm-up window and
|
||||||
// verification read below is in flight.
|
// waitForJobCompletion is the authoritative signal. Once the
|
||||||
if (gracePoller) { clearInterval(gracePoller); gracePoller = null; }
|
// stream has exited, fire only after GRACE_MS of inactivity
|
||||||
// The log stream exiting only means the container stopped producing
|
// measured against the last received chunk — output that resumes
|
||||||
// output — it does NOT prove the Job was deleted. Verify Job
|
// through a reconnect resets the clock so transient drops do not
|
||||||
// presence with a one-shot read so we can distinguish:
|
// truncate live runs (FAR-107).
|
||||||
// (a) Job 404 → truly gone (TTL or external deletion)
|
if (graceCheckInFlight) return;
|
||||||
// (b) Job still present → K8s condition propagation lag (FAR-23)
|
if (
|
||||||
// Without this check we mis-classify (b) as "deleted externally" and
|
streamActivity.streamHasExited &&
|
||||||
// emit a false-positive k8s_job_deleted_externally error (FAR-107).
|
Date.now() - streamActivity.lastActiveAt >= LOG_EXIT_COMPLETION_GRACE_MS
|
||||||
|
) {
|
||||||
|
graceCheckInFlight = true;
|
||||||
void (async () => {
|
void (async () => {
|
||||||
|
try {
|
||||||
|
// Pod-phase gate (FAR-107): if the pod is still Running/Pending
|
||||||
|
// the container is alive — Claude can be silent for >30s during
|
||||||
|
// long tool calls (web fetches, slow upstream APIs). Refresh
|
||||||
|
// the stream-activity timer, leave the poller armed, and let
|
||||||
|
// waitForJobCompletion remain the authoritative signal. Only
|
||||||
|
// proceed with the grace settlement when the pod has actually
|
||||||
|
// reached a terminal phase or is gone.
|
||||||
|
const podLookup = await lookupPodState(namespace, jobName, kubeconfigPath);
|
||||||
|
if (!podLookup.podMissing && (podLookup.phase === "Running" || podLookup.phase === "Pending")) {
|
||||||
|
streamActivity.lastActiveAt = Date.now();
|
||||||
|
graceCheckInFlight = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
await onLog("stderr", `[paperclip] grace gate: pod state lookup failed (${err instanceof Error ? err.message : String(err)}) — falling through to Job-presence check\n`).catch(() => {});
|
||||||
|
}
|
||||||
|
// Pod is no longer Running — proceed with Job-presence verification.
|
||||||
|
// Stop the grace poller immediately so we don't double-fire while the
|
||||||
|
// verification read below is in flight.
|
||||||
|
if (gracePoller) { clearInterval(gracePoller); gracePoller = null; }
|
||||||
|
// The log stream exiting only means the container stopped producing
|
||||||
|
// output — it does NOT prove the Job was deleted. Verify Job
|
||||||
|
// presence with a one-shot read so we can distinguish:
|
||||||
|
// (a) Job 404 → truly gone (TTL or external deletion)
|
||||||
|
// (b) Job still present → K8s condition propagation lag (FAR-23)
|
||||||
|
// Without this check we mis-classify (b) as "deleted externally" and
|
||||||
|
// emit a false-positive k8s_job_deleted_externally error (FAR-107).
|
||||||
try {
|
try {
|
||||||
await getBatchApi(kubeconfigPath).readNamespacedJob({ name: jobName, namespace });
|
await getBatchApi(kubeconfigPath).readNamespacedJob({ name: jobName, namespace });
|
||||||
await onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago without K8s Job condition update; Job ${jobName} still present — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
await onLog("stdout", `[paperclip] Log stream exited ${LOG_EXIT_COMPLETION_GRACE_MS / 1000}s ago without K8s Job condition update; Job ${jobName} still present — proceeding with captured output (FAR-23)\n`).catch(() => {});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
import { listK8sModels } from "./models.js";
|
import { listK8sModels, DIRECT_MODELS, BEDROCK_MODELS } from "./models.js";
|
||||||
|
|
||||||
describe("listK8sModels", () => {
|
describe("listK8sModels", () => {
|
||||||
const savedEnv: Record<string, string | undefined> = {};
|
const savedEnv: Record<string, string | undefined> = {};
|
||||||
@@ -50,3 +50,22 @@ describe("listK8sModels", () => {
|
|||||||
expect(models.some((m) => m.id === "claude-opus-4-7")).toBe(true);
|
expect(models.some((m) => m.id === "claude-opus-4-7")).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("static model lists", () => {
|
||||||
|
it("DIRECT_MODELS is non-empty and has valid ids", () => {
|
||||||
|
expect(DIRECT_MODELS.length).toBeGreaterThan(0);
|
||||||
|
for (const m of DIRECT_MODELS) {
|
||||||
|
expect(typeof m.id).toBe("string");
|
||||||
|
expect(m.id.length).toBeGreaterThan(0);
|
||||||
|
expect(typeof m.label).toBe("string");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BEDROCK_MODELS is non-empty and all ids contain 'anthropic.'", () => {
|
||||||
|
expect(BEDROCK_MODELS.length).toBeGreaterThan(0);
|
||||||
|
for (const m of BEDROCK_MODELS) {
|
||||||
|
expect(m.id).toContain("anthropic.");
|
||||||
|
expect(typeof m.label).toBe("string");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { AdapterModel } from "@paperclipai/adapter-utils";
|
import type { AdapterModel } from "@paperclipai/adapter-utils";
|
||||||
|
|
||||||
const DIRECT_MODELS: AdapterModel[] = [
|
export const DIRECT_MODELS: AdapterModel[] = [
|
||||||
{ id: "claude-opus-4-7", label: "Claude Opus 4.7" },
|
{ id: "claude-opus-4-7", label: "Claude Opus 4.7" },
|
||||||
{ id: "claude-opus-4-6", label: "Claude Opus 4.6" },
|
{ id: "claude-opus-4-6", label: "Claude Opus 4.6" },
|
||||||
{ id: "claude-sonnet-4-6", label: "Claude Sonnet 4.6" },
|
{ id: "claude-sonnet-4-6", label: "Claude Sonnet 4.6" },
|
||||||
@@ -9,7 +9,7 @@ const DIRECT_MODELS: AdapterModel[] = [
|
|||||||
{ id: "claude-haiku-4-5-20251001", label: "Claude Haiku 4.5" },
|
{ id: "claude-haiku-4-5-20251001", label: "Claude Haiku 4.5" },
|
||||||
];
|
];
|
||||||
|
|
||||||
const BEDROCK_MODELS: AdapterModel[] = [
|
export const BEDROCK_MODELS: AdapterModel[] = [
|
||||||
{ id: "us.anthropic.claude-opus-4-7", label: "Bedrock Opus 4.7" },
|
{ id: "us.anthropic.claude-opus-4-7", label: "Bedrock Opus 4.7" },
|
||||||
{ id: "us.anthropic.claude-opus-4-6-v1", label: "Bedrock Opus 4.6" },
|
{ id: "us.anthropic.claude-opus-4-6-v1", label: "Bedrock Opus 4.6" },
|
||||||
{ id: "us.anthropic.claude-sonnet-4-6", label: "Bedrock Sonnet 4.6" },
|
{ id: "us.anthropic.claude-sonnet-4-6", label: "Bedrock Sonnet 4.6" },
|
||||||
@@ -17,7 +17,7 @@ const BEDROCK_MODELS: AdapterModel[] = [
|
|||||||
{ id: "us.anthropic.claude-haiku-4-5-20251001-v1:0", label: "Bedrock Haiku 4.5" },
|
{ id: "us.anthropic.claude-haiku-4-5-20251001-v1:0", label: "Bedrock Haiku 4.5" },
|
||||||
];
|
];
|
||||||
|
|
||||||
function isBedrockEnv(): boolean {
|
export function isBedrockEnv(): boolean {
|
||||||
return (
|
return (
|
||||||
process.env.CLAUDE_CODE_USE_BEDROCK === "1" ||
|
process.env.CLAUDE_CODE_USE_BEDROCK === "1" ||
|
||||||
process.env.CLAUDE_CODE_USE_BEDROCK === "true" ||
|
process.env.CLAUDE_CODE_USE_BEDROCK === "true" ||
|
||||||
|
|||||||
Reference in New Issue
Block a user