From 83b58f9207dae456c554de6af721836e6bcc1118 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Fri, 24 Apr 2026 20:00:42 +0000 Subject: [PATCH] fix: detect stop_reason:null + output_tokens:0 and emit llm_api_error (FAR-30) parseClaudeStreamJson now tracks assistant events with stop_reason:null and output_tokens:0 (the MiniMax degraded-response pattern). When no result event follows, execute() returns errorCode:"llm_api_error" with a descriptive message instead of the generic adapter_failed. Co-Authored-By: Paperclip --- src/server/execute.test.ts | 36 +++++++++++++++++ src/server/parse.test.ts | 81 ++++++++++++++++++++++++++++++++++++++ src/server/parse.ts | 19 +++++++++ 3 files changed, 136 insertions(+) diff --git a/src/server/execute.test.ts b/src/server/execute.test.ts index 247aa9c..5ad5406 100644 --- a/src/server/execute.test.ts +++ b/src/server/execute.test.ts @@ -958,6 +958,42 @@ describe("execute: happy path", () => { expect(result.exitCode).toBeNull(); }); + it("returns llm_api_error when assistant event has stop_reason:null and output_tokens:0 (FAR-30)", async () => { + // Reproduces the MiniMax degradation pattern: init event + assistant event with + // stop_reason:null and output_tokens:0, no result event, Claude exits -1. + const emptyResponseOutput = [ + JSON.stringify({ type: "system", subtype: "init", model: "MiniMax-M2.7", session_id: "sess_mm" }), + JSON.stringify({ + type: "assistant", + session_id: "sess_mm", + message: { + id: "msg_empty", + stop_reason: null, + usage: { input_tokens: 500, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, + content: [], + }, + }), + ].join("\n") + "\n"; + + mockLogFn.mockImplementation( + async (_ns: string, _pod: string, _ctr: string, writable: Writable) => { + writable.write(emptyResponseOutput); + }, + ); + // getPodExitCode: exit code -1 (as reported in the issue) + mockCoreListPods.mockResolvedValue({ + items: [{ metadata: { name: "pod-abc" }, status: { containerStatuses: [{ name: "claude", state: { terminated: { exitCode: -1 } } }] } }], + }); + + const executePromise = execute(makeCtx()); + await vi.advanceTimersByTimeAsync(3_100); + const result = await executePromise; + + expect(result.errorCode).toBe("llm_api_error"); + expect(result.errorMessage).toContain("stop_reason: null"); + expect(result.errorMessage).toContain("output_tokens: 0"); + }); + it("reconnects log stream and logs status when job completion takes > 3s", async () => { // Make waitForJobCompletion take 4s so the 3s stream reconnect fires first. // timeoutSec=4, graceSec=0 → completionTimeoutMs=4000. diff --git a/src/server/parse.test.ts b/src/server/parse.test.ts index f6e6193..195fb9c 100644 --- a/src/server/parse.test.ts +++ b/src/server/parse.test.ts @@ -154,6 +154,87 @@ more raw output`; // Should not be "Hello world\n\nHello world" expect(result.summary.split("Hello world").length).toBe(2); }); + + it("sets llmApiEmptyResponse=true when stop_reason:null and usage.output_tokens:0", () => { + const initLine = JSON.stringify({ type: "system", subtype: "init", model: "MiniMax-M2.7", session_id: "sess_1" }); + const assistantEvent = JSON.stringify({ + type: "assistant", + session_id: "sess_1", + message: { + id: "msg_abc", + stop_reason: null, + usage: { input_tokens: 100, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, + content: [], + }, + }); + const result = parseClaudeStreamJson([initLine, assistantEvent].join("\n")); + expect(result.llmApiEmptyResponse).toBe(true); + expect(result.resultJson).toBeNull(); + }); + + it("sets llmApiEmptyResponse=true when stop_reason:null and message-level output_tokens:0", () => { + const assistantEvent = JSON.stringify({ + type: "assistant", + message: { stop_reason: null, output_tokens: 0, content: [] }, + }); + const result = parseClaudeStreamJson(assistantEvent); + expect(result.llmApiEmptyResponse).toBe(true); + }); + + it("does not set llmApiEmptyResponse when stop_reason is non-null", () => { + const assistantEvent = JSON.stringify({ + type: "assistant", + message: { + stop_reason: "end_turn", + usage: { output_tokens: 0 }, + content: [], + }, + }); + const result = parseClaudeStreamJson(assistantEvent); + expect(result.llmApiEmptyResponse).toBe(false); + }); + + it("does not set llmApiEmptyResponse when output_tokens > 0", () => { + const assistantEvent = JSON.stringify({ + type: "assistant", + message: { + stop_reason: null, + usage: { output_tokens: 5 }, + content: [{ type: "text", text: "hello" }], + }, + }); + const result = parseClaudeStreamJson(assistantEvent); + expect(result.llmApiEmptyResponse).toBe(false); + }); + + it("clears llmApiEmptyResponse when a result event follows the empty assistant event", () => { + const assistantEvent = JSON.stringify({ + type: "assistant", + message: { stop_reason: null, usage: { output_tokens: 0 }, content: [] }, + }); + const resultEvent = JSON.stringify({ + type: "result", + result: "Done", + subtype: "stop", + total_cost_usd: 0.001, + usage: { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 0 }, + }); + const result = parseClaudeStreamJson([assistantEvent, resultEvent].join("\n")); + expect(result.llmApiEmptyResponse).toBe(false); + expect(result.resultJson).not.toBeNull(); + }); + + it("sets llmApiEmptyResponse=false for normal result", () => { + const resultEvent = JSON.stringify({ + type: "result", + result: "Done", + subtype: "stop", + total_cost_usd: 0.005, + usage: { input_tokens: 100, output_tokens: 200, cache_read_input_tokens: 50 }, + }); + const result = parseClaudeStreamJson(resultEvent); + expect(result.llmApiEmptyResponse).toBe(false); + }); }); describe("extractClaudeLoginUrl", () => { diff --git a/src/server/parse.ts b/src/server/parse.ts index bd641ff..5d900b7 100644 --- a/src/server/parse.ts +++ b/src/server/parse.ts @@ -15,6 +15,10 @@ export function parseClaudeStreamJson(stdout: string) { // at the line level; this guard only needs to protect against the same // message block being parsed twice. const seenBlocks = new Set(); + // Set when we see stop_reason:null + output_tokens:0 on an assistant event + // with no subsequent result event — indicates the upstream LLM API returned + // an empty/malformed response (e.g. MiniMax degraded performance). + let llmApiEmptyResponse = false; for (const rawLine of stdout.split(/\r?\n/)) { const line = rawLine.trim(); @@ -34,6 +38,18 @@ export function parseClaudeStreamJson(stdout: string) { const message = parseObject(event.message); const messageId = asString(message.id, ""); const content = Array.isArray(message.content) ? message.content : []; + + // Detect empty LLM API response: stop_reason:null with zero output tokens. + // output_tokens may appear directly on message or nested under message.usage. + const stopReason = message.stop_reason; + const usageObj = parseObject(message.usage as Record); + const outputTokens = typeof message.output_tokens === "number" + ? message.output_tokens + : asNumber(usageObj.output_tokens, -1); + if (stopReason === null && outputTokens === 0) { + llmApiEmptyResponse = true; + } + for (let i = 0; i < content.length; i++) { const entry = content[i]; if (typeof entry !== "object" || entry === null || Array.isArray(entry)) continue; @@ -55,6 +71,7 @@ export function parseClaudeStreamJson(stdout: string) { if (type === "result") { finalResult = event; + llmApiEmptyResponse = false; // result event means Claude completed normally sessionId = asString(event.session_id, sessionId ?? "") || sessionId; } } @@ -67,6 +84,7 @@ export function parseClaudeStreamJson(stdout: string) { usage: null as UsageSummary | null, summary: assistantTexts.join("\n\n").trim(), resultJson: null as Record | null, + llmApiEmptyResponse, }; } @@ -87,6 +105,7 @@ export function parseClaudeStreamJson(stdout: string) { usage, summary, resultJson: finalResult, + llmApiEmptyResponse: false, }; }