diff --git a/packages/adapters/grok-local/src/server/parse.test.ts b/packages/adapters/grok-local/src/server/parse.test.ts index 5b31cf88..213f8c02 100644 --- a/packages/adapters/grok-local/src/server/parse.test.ts +++ b/packages/adapters/grok-local/src/server/parse.test.ts @@ -28,6 +28,37 @@ describe("parseGrokJsonl", () => { expect(parsed.errorMessage).toBe("Authentication required"); }); + + it("separates reasoning turns that grok streaming-json glues together", () => { + // PAPA-349: at turn boundaries grok drops the newline between turns; the + // aggregated thought should still read as two paragraphs. + const parsed = parseGrokJsonl([ + JSON.stringify({ type: "thought", data: "The user uses `" }), + JSON.stringify({ type: "thought", data: "ls" }), + JSON.stringify({ type: "thought", data: "`" }), + JSON.stringify({ type: "thought", data: "The" }), + JSON.stringify({ type: "thought", data: " `" }), + JSON.stringify({ type: "thought", data: "ls" }), + JSON.stringify({ type: "thought", data: "`" }), + JSON.stringify({ type: "thought", data: " returned" }), + JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "sess-1" }), + ].join("\n")); + + expect(parsed.thought).toBe("The user uses `ls`\nThe `ls` returned"); + }); + + it("preserves assistant `text` chunks verbatim (no boundary heuristic)", () => { + // PAPA-349 review feedback: the turn-boundary helper is scoped to the + // reasoning stream only. Final assistant text is stored unmodified so + // user-visible responses cannot be reshaped by the heuristic. + const parsed = parseGrokJsonl([ + JSON.stringify({ type: "text", data: "Done." }), + JSON.stringify({ type: "text", data: "Next" }), + JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "sess-1" }), + ].join("\n")); + + expect(parsed.summary).toBe("Done.Next"); + }); }); describe("isGrokUnknownSessionError", () => { diff --git a/packages/adapters/grok-local/src/server/parse.ts b/packages/adapters/grok-local/src/server/parse.ts index eb5ab92d..7fc415a2 100644 --- a/packages/adapters/grok-local/src/server/parse.ts +++ b/packages/adapters/grok-local/src/server/parse.ts @@ -1,4 +1,5 @@ import { asString, parseJson, parseObject } from "@paperclipai/adapter-utils/server-utils"; +import { applyTurnBoundary, createTurnBoundaryState } from "../shared/turn-boundary.js"; export interface ParsedGrokJsonl { sessionId: string | null; @@ -32,6 +33,7 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl { let errorMessage: string | null = null; const thoughtParts: string[] = []; const textParts: string[] = []; + const thoughtBoundary = createTurnBoundaryState(); for (const rawLine of stdout.split(/\r?\n/)) { const line = rawLine.trim(); @@ -43,7 +45,7 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl { const type = asString(event.type, "").trim(); if (type === "thought") { const text = asString(event.data, ""); - if (text) thoughtParts.push(text); + if (text) thoughtParts.push(applyTurnBoundary(thoughtBoundary, text)); continue; } diff --git a/packages/adapters/grok-local/src/shared/turn-boundary.test.ts b/packages/adapters/grok-local/src/shared/turn-boundary.test.ts new file mode 100644 index 00000000..1fc27acf --- /dev/null +++ b/packages/adapters/grok-local/src/shared/turn-boundary.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { applyTurnBoundary, createTurnBoundaryState } from "./turn-boundary.js"; + +function run(chunks: string[]): string { + const state = createTurnBoundaryState(); + return chunks.map((chunk) => applyTurnBoundary(state, chunk)).join(""); +} + +describe("applyTurnBoundary", () => { + it("inserts a newline when a closing backtick is followed by a new capitalized turn", () => { + expect(run(["The user uses `", "ls", "`", "The", " `", "ls", "`", " returned"])) + .toBe("The user uses `ls`\nThe `ls` returned"); + }); + + it("inserts a newline after sentence-ending punctuation glued to a capitalized word", () => { + expect(run(["returned", ":", "Confirmed", ":", " 4 files"])) + .toBe("returned:\nConfirmed: 4 files"); + }); + + it("does not break apart backtick-wrapped CamelCase identifiers within a turn", () => { + expect(run(["render `", "React", "` then "])) + .toBe("render `React` then "); + }); + + it("leaves natural token streams with proper whitespace alone", () => { + expect(run(["The", " user", " wants", " me", " to", ":\n", "1", ".", " List"])) + .toBe("The user wants me to:\n1. List"); + }); + + it("does not insert a separator when the next chunk starts with whitespace", () => { + expect(run(["function", ".", " They"])) + .toBe("function. They"); + }); + + it("does not insert a separator when the next chunk starts lowercase", () => { + expect(run(["`", "ls", "`"])) + .toBe("`ls`"); + }); + + it("does not insert a separator when the next chunk is a single character", () => { + expect(run([":", "A"])) + .toBe(":A"); + }); + + it("does not insert a separator after a self-contained backtick span in a single chunk", () => { + // Greptile review: a chunk like "`ls`" is a balanced span; the following + // capitalized word should be treated as a continuation, not a new turn. + expect(run(["`ls`", "Then"])) + .toBe("`ls`Then"); + }); +}); diff --git a/packages/adapters/grok-local/src/shared/turn-boundary.ts b/packages/adapters/grok-local/src/shared/turn-boundary.ts new file mode 100644 index 00000000..267b784e --- /dev/null +++ b/packages/adapters/grok-local/src/shared/turn-boundary.ts @@ -0,0 +1,54 @@ +// Grok's `--output-format streaming-json` mode emits `thought` and `text` events +// token-by-token. Between reasoning turns (around tool calls) it drops the `\n` +// separator that the non-streaming `--output-format json` mode includes in the +// aggregated `thought` field. This helper inserts a single `\n` when a new chunk +// would otherwise glue two turns together (e.g. ``"`"`` then `"The"` => `` `The``). + +export interface TurnBoundaryState { + lastChunk: string; + backtickParity: 0 | 1; +} + +export function createTurnBoundaryState(): TurnBoundaryState { + return { lastChunk: "", backtickParity: 0 }; +} + +function countBackticks(text: string): number { + let count = 0; + for (const ch of text) if (ch === "`") count += 1; + return count; +} + +function endsWithSentenceClose(ch: string): boolean { + return ch === "." || ch === "?" || ch === "!" || ch === ":" || ch === ";"; +} + +export function applyTurnBoundary(state: TurnBoundaryState, incoming: string): string { + if (!incoming) return incoming; + + let output = incoming; + const prev = state.lastChunk; + if ( + prev && + !/\s$/.test(prev) && + !/^\s/.test(incoming) && + /^[A-Z]/.test(incoming) && + incoming.length >= 2 + ) { + const lastChar = prev[prev.length - 1]!; + // Narrow the backtick trigger to a lone closing-backtick chunk (e.g. the + // stream "...`", "ls", "`" then "The"). A compound chunk like "`ls`" is a + // self-contained span and the following capitalized word is a continuation, + // not a new turn. + const closingLoneBacktick = + prev === "`" && state.backtickParity === 0; + const looksLikeNewTurn = endsWithSentenceClose(lastChar) || closingLoneBacktick; + if (looksLikeNewTurn) { + output = `\n${incoming}`; + } + } + + state.lastChunk = incoming; + state.backtickParity = ((state.backtickParity + countBackticks(incoming)) % 2) as 0 | 1; + return output; +} diff --git a/packages/adapters/grok-local/src/ui/index.ts b/packages/adapters/grok-local/src/ui/index.ts index a37e3d82..605accd8 100644 --- a/packages/adapters/grok-local/src/ui/index.ts +++ b/packages/adapters/grok-local/src/ui/index.ts @@ -1,2 +1,2 @@ -export { parseGrokStdoutLine } from "./parse-stdout.js"; +export { parseGrokStdoutLine, createGrokStdoutParser } from "./parse-stdout.js"; export { buildGrokLocalConfig } from "./build-config.js"; diff --git a/packages/adapters/grok-local/src/ui/parse-stdout.test.ts b/packages/adapters/grok-local/src/ui/parse-stdout.test.ts index d60b5b61..96276149 100644 --- a/packages/adapters/grok-local/src/ui/parse-stdout.test.ts +++ b/packages/adapters/grok-local/src/ui/parse-stdout.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { parseGrokStdoutLine } from "./parse-stdout.js"; +import { createGrokStdoutParser, parseGrokStdoutLine } from "./parse-stdout.js"; describe("parseGrokStdoutLine", () => { const ts = "2026-05-15T00:00:00.000Z"; @@ -25,3 +25,46 @@ describe("parseGrokStdoutLine", () => { ]); }); }); + +describe("createGrokStdoutParser", () => { + const ts = "2026-05-15T00:00:00.000Z"; + + function thoughtTexts(chunks: string[]): string { + const parser = createGrokStdoutParser(); + return chunks + .map((data) => parser.parseLine(JSON.stringify({ type: "thought", data }), ts)) + .flat() + .map((entry) => entry.kind === "thinking" ? entry.text : "") + .join(""); + } + + it("inserts a newline between reasoning turns that grok streaming-json glues together", () => { + // Reproduces PAPA-349: token stream "...using `ls`" then a new turn "The `ls` command returned" + expect(thoughtTexts(["The user uses `", "ls", "`", "The", " `", "ls", "`", " returned"])) + .toBe("The user uses `ls`\nThe `ls` returned"); + }); + + it("inserts a newline when a turn ends with a colon and the next turn starts capitalized", () => { + expect(thoughtTexts(["returned", ":", "Confirmed", ":", " 4 files"])) + .toBe("returned:\nConfirmed: 4 files"); + }); + + it("resets state between independent transcript builds", () => { + const parser = createGrokStdoutParser(); + parser.parseLine(JSON.stringify({ type: "thought", data: "first:" }), ts); + parser.reset(); + expect(parser.parseLine(JSON.stringify({ type: "thought", data: "Second" }), ts)).toEqual([ + { kind: "thinking", ts, text: "Second", delta: true }, + ]); + }); + + it("does not modify assistant `text` chunks", () => { + // PAPA-349 review feedback: keep final assistant text streaming verbatim; + // the boundary heuristic is scoped to reasoning. + const parser = createGrokStdoutParser(); + parser.parseLine(JSON.stringify({ type: "text", data: "Done." }), ts); + expect(parser.parseLine(JSON.stringify({ type: "text", data: "Next" }), ts)).toEqual([ + { kind: "assistant", ts, text: "Next", delta: true }, + ]); + }); +}); diff --git a/packages/adapters/grok-local/src/ui/parse-stdout.ts b/packages/adapters/grok-local/src/ui/parse-stdout.ts index b82a36b4..476dd88a 100644 --- a/packages/adapters/grok-local/src/ui/parse-stdout.ts +++ b/packages/adapters/grok-local/src/ui/parse-stdout.ts @@ -1,4 +1,5 @@ import type { TranscriptEntry } from "@paperclipai/adapter-utils"; +import { applyTurnBoundary, createTurnBoundaryState, type TurnBoundaryState } from "../shared/turn-boundary.js"; function safeJsonParse(text: string): unknown { try { @@ -24,7 +25,11 @@ function extractErrorText(value: unknown): string { return asString(record.message) || asString(record.detail) || asString(record.code); } -export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] { +function parseLineInternal( + line: string, + ts: string, + thoughtBoundary: TurnBoundaryState, +): TranscriptEntry[] { const parsed = asRecord(safeJsonParse(line)); if (!parsed) { return [{ kind: "stdout", ts, text: line }]; @@ -34,12 +39,14 @@ export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] if (type === "thought") { const text = asString(parsed.data); - return text ? [{ kind: "thinking", ts, text, delta: true }] : []; + if (!text) return []; + return [{ kind: "thinking", ts, text: applyTurnBoundary(thoughtBoundary, text), delta: true }]; } if (type === "text") { const text = asString(parsed.data); - return text ? [{ kind: "assistant", ts, text, delta: true }] : []; + if (!text) return []; + return [{ kind: "assistant", ts, text, delta: true }]; } if (type === "error") { @@ -59,3 +66,22 @@ export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] return [{ kind: "system", ts, text: `event: ${type || "unknown"}` }]; } + +export function createGrokStdoutParser() { + let thoughtBoundary = createTurnBoundaryState(); + return { + parseLine(line: string, ts: string): TranscriptEntry[] { + return parseLineInternal(line, ts, thoughtBoundary); + }, + reset() { + thoughtBoundary = createTurnBoundaryState(); + }, + }; +} + +// Stateless fallback for callers that haven't migrated to the stateful factory. +// Without state, consecutive thought chunks at reasoning-turn boundaries can +// still appear merged; prefer createGrokStdoutParser for live transcripts. +export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] { + return parseLineInternal(line, ts, createTurnBoundaryState()); +} diff --git a/ui/src/adapters/grok-local/index.ts b/ui/src/adapters/grok-local/index.ts index 13c579a0..31b7ffe2 100644 --- a/ui/src/adapters/grok-local/index.ts +++ b/ui/src/adapters/grok-local/index.ts @@ -1,5 +1,5 @@ import type { UIAdapterModule } from "../types"; -import { parseGrokStdoutLine } from "@paperclipai/adapter-grok-local/ui"; +import { createGrokStdoutParser, parseGrokStdoutLine } from "@paperclipai/adapter-grok-local/ui"; import { buildGrokLocalConfig } from "@paperclipai/adapter-grok-local/ui"; import { GrokLocalConfigFields } from "./config-fields"; @@ -7,6 +7,7 @@ export const grokLocalUIAdapter: UIAdapterModule = { type: "grok_local", label: "Grok Build (local)", parseStdoutLine: parseGrokStdoutLine, + createStdoutParser: createGrokStdoutParser, ConfigFields: GrokLocalConfigFields, buildAdapterConfig: buildGrokLocalConfig, };