fix: detect stop_reason:null + output_tokens:0 and emit llm_api_error (FAR-30)
parseClaudeStreamJson now tracks assistant events with stop_reason:null and output_tokens:0 (the MiniMax degraded-response pattern). When no result event follows, execute() returns errorCode:"llm_api_error" with a descriptive message instead of the generic adapter_failed. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -958,6 +958,42 @@ describe("execute: happy path", () => {
|
|||||||
expect(result.exitCode).toBeNull();
|
expect(result.exitCode).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns llm_api_error when assistant event has stop_reason:null and output_tokens:0 (FAR-30)", async () => {
|
||||||
|
// Reproduces the MiniMax degradation pattern: init event + assistant event with
|
||||||
|
// stop_reason:null and output_tokens:0, no result event, Claude exits -1.
|
||||||
|
const emptyResponseOutput = [
|
||||||
|
JSON.stringify({ type: "system", subtype: "init", model: "MiniMax-M2.7", session_id: "sess_mm" }),
|
||||||
|
JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
session_id: "sess_mm",
|
||||||
|
message: {
|
||||||
|
id: "msg_empty",
|
||||||
|
stop_reason: null,
|
||||||
|
usage: { input_tokens: 500, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||||
|
content: [],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
].join("\n") + "\n";
|
||||||
|
|
||||||
|
mockLogFn.mockImplementation(
|
||||||
|
async (_ns: string, _pod: string, _ctr: string, writable: Writable) => {
|
||||||
|
writable.write(emptyResponseOutput);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// getPodExitCode: exit code -1 (as reported in the issue)
|
||||||
|
mockCoreListPods.mockResolvedValue({
|
||||||
|
items: [{ metadata: { name: "pod-abc" }, status: { containerStatuses: [{ name: "claude", state: { terminated: { exitCode: -1 } } }] } }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const executePromise = execute(makeCtx());
|
||||||
|
await vi.advanceTimersByTimeAsync(3_100);
|
||||||
|
const result = await executePromise;
|
||||||
|
|
||||||
|
expect(result.errorCode).toBe("llm_api_error");
|
||||||
|
expect(result.errorMessage).toContain("stop_reason: null");
|
||||||
|
expect(result.errorMessage).toContain("output_tokens: 0");
|
||||||
|
});
|
||||||
|
|
||||||
it("reconnects log stream and logs status when job completion takes > 3s", async () => {
|
it("reconnects log stream and logs status when job completion takes > 3s", async () => {
|
||||||
// Make waitForJobCompletion take 4s so the 3s stream reconnect fires first.
|
// Make waitForJobCompletion take 4s so the 3s stream reconnect fires first.
|
||||||
// timeoutSec=4, graceSec=0 → completionTimeoutMs=4000.
|
// timeoutSec=4, graceSec=0 → completionTimeoutMs=4000.
|
||||||
|
|||||||
@@ -154,6 +154,87 @@ more raw output`;
|
|||||||
// Should not be "Hello world\n\nHello world"
|
// Should not be "Hello world\n\nHello world"
|
||||||
expect(result.summary.split("Hello world").length).toBe(2);
|
expect(result.summary.split("Hello world").length).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("sets llmApiEmptyResponse=true when stop_reason:null and usage.output_tokens:0", () => {
|
||||||
|
const initLine = JSON.stringify({ type: "system", subtype: "init", model: "MiniMax-M2.7", session_id: "sess_1" });
|
||||||
|
const assistantEvent = JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
session_id: "sess_1",
|
||||||
|
message: {
|
||||||
|
id: "msg_abc",
|
||||||
|
stop_reason: null,
|
||||||
|
usage: { input_tokens: 100, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||||
|
content: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson([initLine, assistantEvent].join("\n"));
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(true);
|
||||||
|
expect(result.resultJson).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sets llmApiEmptyResponse=true when stop_reason:null and message-level output_tokens:0", () => {
|
||||||
|
const assistantEvent = JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
message: { stop_reason: null, output_tokens: 0, content: [] },
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson(assistantEvent);
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not set llmApiEmptyResponse when stop_reason is non-null", () => {
|
||||||
|
const assistantEvent = JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
message: {
|
||||||
|
stop_reason: "end_turn",
|
||||||
|
usage: { output_tokens: 0 },
|
||||||
|
content: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson(assistantEvent);
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not set llmApiEmptyResponse when output_tokens > 0", () => {
|
||||||
|
const assistantEvent = JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
message: {
|
||||||
|
stop_reason: null,
|
||||||
|
usage: { output_tokens: 5 },
|
||||||
|
content: [{ type: "text", text: "hello" }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson(assistantEvent);
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears llmApiEmptyResponse when a result event follows the empty assistant event", () => {
|
||||||
|
const assistantEvent = JSON.stringify({
|
||||||
|
type: "assistant",
|
||||||
|
message: { stop_reason: null, usage: { output_tokens: 0 }, content: [] },
|
||||||
|
});
|
||||||
|
const resultEvent = JSON.stringify({
|
||||||
|
type: "result",
|
||||||
|
result: "Done",
|
||||||
|
subtype: "stop",
|
||||||
|
total_cost_usd: 0.001,
|
||||||
|
usage: { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 0 },
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson([assistantEvent, resultEvent].join("\n"));
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(false);
|
||||||
|
expect(result.resultJson).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sets llmApiEmptyResponse=false for normal result", () => {
|
||||||
|
const resultEvent = JSON.stringify({
|
||||||
|
type: "result",
|
||||||
|
result: "Done",
|
||||||
|
subtype: "stop",
|
||||||
|
total_cost_usd: 0.005,
|
||||||
|
usage: { input_tokens: 100, output_tokens: 200, cache_read_input_tokens: 50 },
|
||||||
|
});
|
||||||
|
const result = parseClaudeStreamJson(resultEvent);
|
||||||
|
expect(result.llmApiEmptyResponse).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("extractClaudeLoginUrl", () => {
|
describe("extractClaudeLoginUrl", () => {
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ export function parseClaudeStreamJson(stdout: string) {
|
|||||||
// at the line level; this guard only needs to protect against the same
|
// at the line level; this guard only needs to protect against the same
|
||||||
// message block being parsed twice.
|
// message block being parsed twice.
|
||||||
const seenBlocks = new Set<string>();
|
const seenBlocks = new Set<string>();
|
||||||
|
// Set when we see stop_reason:null + output_tokens:0 on an assistant event
|
||||||
|
// with no subsequent result event — indicates the upstream LLM API returned
|
||||||
|
// an empty/malformed response (e.g. MiniMax degraded performance).
|
||||||
|
let llmApiEmptyResponse = false;
|
||||||
|
|
||||||
for (const rawLine of stdout.split(/\r?\n/)) {
|
for (const rawLine of stdout.split(/\r?\n/)) {
|
||||||
const line = rawLine.trim();
|
const line = rawLine.trim();
|
||||||
@@ -34,6 +38,18 @@ export function parseClaudeStreamJson(stdout: string) {
|
|||||||
const message = parseObject(event.message);
|
const message = parseObject(event.message);
|
||||||
const messageId = asString(message.id, "");
|
const messageId = asString(message.id, "");
|
||||||
const content = Array.isArray(message.content) ? message.content : [];
|
const content = Array.isArray(message.content) ? message.content : [];
|
||||||
|
|
||||||
|
// Detect empty LLM API response: stop_reason:null with zero output tokens.
|
||||||
|
// output_tokens may appear directly on message or nested under message.usage.
|
||||||
|
const stopReason = message.stop_reason;
|
||||||
|
const usageObj = parseObject(message.usage as Record<string, unknown>);
|
||||||
|
const outputTokens = typeof message.output_tokens === "number"
|
||||||
|
? message.output_tokens
|
||||||
|
: asNumber(usageObj.output_tokens, -1);
|
||||||
|
if (stopReason === null && outputTokens === 0) {
|
||||||
|
llmApiEmptyResponse = true;
|
||||||
|
}
|
||||||
|
|
||||||
for (let i = 0; i < content.length; i++) {
|
for (let i = 0; i < content.length; i++) {
|
||||||
const entry = content[i];
|
const entry = content[i];
|
||||||
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) continue;
|
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) continue;
|
||||||
@@ -55,6 +71,7 @@ export function parseClaudeStreamJson(stdout: string) {
|
|||||||
|
|
||||||
if (type === "result") {
|
if (type === "result") {
|
||||||
finalResult = event;
|
finalResult = event;
|
||||||
|
llmApiEmptyResponse = false; // result event means Claude completed normally
|
||||||
sessionId = asString(event.session_id, sessionId ?? "") || sessionId;
|
sessionId = asString(event.session_id, sessionId ?? "") || sessionId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -67,6 +84,7 @@ export function parseClaudeStreamJson(stdout: string) {
|
|||||||
usage: null as UsageSummary | null,
|
usage: null as UsageSummary | null,
|
||||||
summary: assistantTexts.join("\n\n").trim(),
|
summary: assistantTexts.join("\n\n").trim(),
|
||||||
resultJson: null as Record<string, unknown> | null,
|
resultJson: null as Record<string, unknown> | null,
|
||||||
|
llmApiEmptyResponse,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -87,6 +105,7 @@ export function parseClaudeStreamJson(stdout: string) {
|
|||||||
usage,
|
usage,
|
||||||
summary,
|
summary,
|
||||||
resultJson: finalResult,
|
resultJson: finalResult,
|
||||||
|
llmApiEmptyResponse: false,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user