fix(grok-local): restore turn boundaries in streaming reasoning text (#6142)

## Thinking Path

> - Paperclip orchestrates AI agents for zero-human companies
> - The `grok-local` adapter streams reasoning text to the issue
"Working..." panel as the grok CLI runs
> - The `grok` CLI's `--output-format streaming-json` mode silently
drops the `\n` separator between reasoning turns around tool calls
> - Consecutive `thought` chunks (e.g. `` "`" `` followed by `"The"`)
arrive with no intervening whitespace event, so the UI's `delta: true`
concatenator merged them into run-on text like `"…planningGreat, now I
have the issue descriptionThe only co"`
> - This PR adds a small turn-boundary helper that detects sentence
boundaries in the upstream `thought` stream and inserts a single `\n`
only when the previous chunk ended with sentence punctuation (or a
balanced closing backtick) AND the next chunk begins a new uppercase
sentence
> - The benefit is readable streaming reasoning in the UI without
changing how completed messages are stored

## What Changed

- Added `packages/adapters/grok-local/src/shared/turn-boundary.ts` with
per-stream state (last chunk + backtick parity) and a
`restoreTurnBoundary()` helper that inserts `\n` only between balanced,
sentence-terminated `thought` chunks
- Wired the helper into `parseGrokJsonl` (server) and added a new
`createGrokStdoutParser` factory used by `grokLocalUIAdapter` for the
live "Working..." panel
- Added focused tests in `shared/turn-boundary.test.ts`, plus regression
assertions in `server/parse.test.ts` and `ui/parse-stdout.test.ts`

## Verification

- `pnpm --filter @paperclip/grok-local test` — 23/23 adapter tests pass
- `pnpm --filter @paperclip/grok-local typecheck` and UI typecheck —
clean
- Replayed an actual broken `grok 0.1.210` stream from the report;
previously-merged boundaries (`` `ls`The ``, `returned:Confirmed`) now
render with a separating newline; chunks inside un-closed backtick spans
are left alone

## Risks

- Low risk. Boundary insertion only fires when prev ends with
`.`/`!`/`?`/balanced `` ` `` and next begins with an uppercase ≥2-char
word, with no whitespace on either side. Worst case: a rare missed split
or a misplaced newline inside reasoning — both purely cosmetic and
confined to the live streaming panel.

## Model Used

- Claude Opus 4.7 (claude-opus-4-7), Anthropic, extended thinking + tool
use via Claude Code

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>

packages/adapters/grok-local/src/server/parse.test.ts

@@ -28,6 +28,37 @@ describe("parseGrokJsonl", () => {
 
     expect(parsed.errorMessage).toBe("Authentication required");
   });
+
+  it("separates reasoning turns that grok streaming-json glues together", () => {
+    // PAPA-349: at turn boundaries grok drops the newline between turns; the
+    // aggregated thought should still read as two paragraphs.
+    const parsed = parseGrokJsonl([
+      JSON.stringify({ type: "thought", data: "The user uses `" }),
+      JSON.stringify({ type: "thought", data: "ls" }),
+      JSON.stringify({ type: "thought", data: "`" }),
+      JSON.stringify({ type: "thought", data: "The" }),
+      JSON.stringify({ type: "thought", data: " `" }),
+      JSON.stringify({ type: "thought", data: "ls" }),
+      JSON.stringify({ type: "thought", data: "`" }),
+      JSON.stringify({ type: "thought", data: " returned" }),
+      JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "sess-1" }),
+    ].join("\n"));
+
+    expect(parsed.thought).toBe("The user uses `ls`\nThe `ls` returned");
+  });
+
+  it("preserves assistant `text` chunks verbatim (no boundary heuristic)", () => {
+    // PAPA-349 review feedback: the turn-boundary helper is scoped to the
+    // reasoning stream only. Final assistant text is stored unmodified so
+    // user-visible responses cannot be reshaped by the heuristic.
+    const parsed = parseGrokJsonl([
+      JSON.stringify({ type: "text", data: "Done." }),
+      JSON.stringify({ type: "text", data: "Next" }),
+      JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "sess-1" }),
+    ].join("\n"));
+
+    expect(parsed.summary).toBe("Done.Next");
+  });
 });
 
 describe("isGrokUnknownSessionError", () => {

packages/adapters/grok-local/src/server/parse.ts

@@ -1,4 +1,5 @@
 import { asString, parseJson, parseObject } from "@paperclipai/adapter-utils/server-utils";
+import { applyTurnBoundary, createTurnBoundaryState } from "../shared/turn-boundary.js";
 
 export interface ParsedGrokJsonl {
   sessionId: string | null;
@@ -32,6 +33,7 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl {
   let errorMessage: string | null = null;
   const thoughtParts: string[] = [];
   const textParts: string[] = [];
+  const thoughtBoundary = createTurnBoundaryState();
 
   for (const rawLine of stdout.split(/\r?\n/)) {
     const line = rawLine.trim();
@@ -43,7 +45,7 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl {
     const type = asString(event.type, "").trim();
     if (type === "thought") {
       const text = asString(event.data, "");
-      if (text) thoughtParts.push(text);
+      if (text) thoughtParts.push(applyTurnBoundary(thoughtBoundary, text));
       continue;
     }
 

packages/adapters/grok-local/src/shared/turn-boundary.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+import { applyTurnBoundary, createTurnBoundaryState } from "./turn-boundary.js";
+
+function run(chunks: string[]): string {
+  const state = createTurnBoundaryState();
+  return chunks.map((chunk) => applyTurnBoundary(state, chunk)).join("");
+}
+
+describe("applyTurnBoundary", () => {
+  it("inserts a newline when a closing backtick is followed by a new capitalized turn", () => {
+    expect(run(["The user uses `", "ls", "`", "The", " `", "ls", "`", " returned"]))
+      .toBe("The user uses `ls`\nThe `ls` returned");
+  });
+
+  it("inserts a newline after sentence-ending punctuation glued to a capitalized word", () => {
+    expect(run(["returned", ":", "Confirmed", ":", " 4 files"]))
+      .toBe("returned:\nConfirmed: 4 files");
+  });
+
+  it("does not break apart backtick-wrapped CamelCase identifiers within a turn", () => {
+    expect(run(["render `", "React", "` then "]))
+      .toBe("render `React` then ");
+  });
+
+  it("leaves natural token streams with proper whitespace alone", () => {
+    expect(run(["The", " user", " wants", " me", " to", ":\n", "1", ".", " List"]))
+      .toBe("The user wants me to:\n1. List");
+  });
+
+  it("does not insert a separator when the next chunk starts with whitespace", () => {
+    expect(run(["function", ".", " They"]))
+      .toBe("function. They");
+  });
+
+  it("does not insert a separator when the next chunk starts lowercase", () => {
+    expect(run(["`", "ls", "`"]))
+      .toBe("`ls`");
+  });
+
+  it("does not insert a separator when the next chunk is a single character", () => {
+    expect(run([":", "A"]))
+      .toBe(":A");
+  });
+
+  it("does not insert a separator after a self-contained backtick span in a single chunk", () => {
+    // Greptile review: a chunk like "`ls`" is a balanced span; the following
+    // capitalized word should be treated as a continuation, not a new turn.
+    expect(run(["`ls`", "Then"]))
+      .toBe("`ls`Then");
+  });
+});

packages/adapters/grok-local/src/shared/turn-boundary.ts

@@ -0,0 +1,54 @@
+// Grok's `--output-format streaming-json` mode emits `thought` and `text` events
+// token-by-token. Between reasoning turns (around tool calls) it drops the `\n`
+// separator that the non-streaming `--output-format json` mode includes in the
+// aggregated `thought` field. This helper inserts a single `\n` when a new chunk
+// would otherwise glue two turns together (e.g. ``"`"`` then `"The"` => `` `The``).
+
+export interface TurnBoundaryState {
+  lastChunk: string;
+  backtickParity: 0 | 1;
+}
+
+export function createTurnBoundaryState(): TurnBoundaryState {
+  return { lastChunk: "", backtickParity: 0 };
+}
+
+function countBackticks(text: string): number {
+  let count = 0;
+  for (const ch of text) if (ch === "`") count += 1;
+  return count;
+}
+
+function endsWithSentenceClose(ch: string): boolean {
+  return ch === "." || ch === "?" || ch === "!" || ch === ":" || ch === ";";
+}
+
+export function applyTurnBoundary(state: TurnBoundaryState, incoming: string): string {
+  if (!incoming) return incoming;
+
+  let output = incoming;
+  const prev = state.lastChunk;
+  if (
+    prev &&
+    !/\s$/.test(prev) &&
+    !/^\s/.test(incoming) &&
+    /^[A-Z]/.test(incoming) &&
+    incoming.length >= 2
+  ) {
+    const lastChar = prev[prev.length - 1]!;
+    // Narrow the backtick trigger to a lone closing-backtick chunk (e.g. the
+    // stream "...`", "ls", "`" then "The"). A compound chunk like "`ls`" is a
+    // self-contained span and the following capitalized word is a continuation,
+    // not a new turn.
+    const closingLoneBacktick =
+      prev === "`" && state.backtickParity === 0;
+    const looksLikeNewTurn = endsWithSentenceClose(lastChar) || closingLoneBacktick;
+    if (looksLikeNewTurn) {
+      output = `\n${incoming}`;
+    }
+  }
+
+  state.lastChunk = incoming;
+  state.backtickParity = ((state.backtickParity + countBackticks(incoming)) % 2) as 0 | 1;
+  return output;
+}

packages/adapters/grok-local/src/ui/index.ts

@@ -1,2 +1,2 @@
-export { parseGrokStdoutLine } from "./parse-stdout.js";
+export { parseGrokStdoutLine, createGrokStdoutParser } from "./parse-stdout.js";
 export { buildGrokLocalConfig } from "./build-config.js";

packages/adapters/grok-local/src/ui/parse-stdout.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { parseGrokStdoutLine } from "./parse-stdout.js";
+import { createGrokStdoutParser, parseGrokStdoutLine } from "./parse-stdout.js";
 
 describe("parseGrokStdoutLine", () => {
   const ts = "2026-05-15T00:00:00.000Z";
@@ -25,3 +25,46 @@ describe("parseGrokStdoutLine", () => {
     ]);
   });
 });
+
+describe("createGrokStdoutParser", () => {
+  const ts = "2026-05-15T00:00:00.000Z";
+
+  function thoughtTexts(chunks: string[]): string {
+    const parser = createGrokStdoutParser();
+    return chunks
+      .map((data) => parser.parseLine(JSON.stringify({ type: "thought", data }), ts))
+      .flat()
+      .map((entry) => entry.kind === "thinking" ? entry.text : "")
+      .join("");
+  }
+
+  it("inserts a newline between reasoning turns that grok streaming-json glues together", () => {
+    // Reproduces PAPA-349: token stream "...using `ls`" then a new turn "The `ls` command returned"
+    expect(thoughtTexts(["The user uses `", "ls", "`", "The", " `", "ls", "`", " returned"]))
+      .toBe("The user uses `ls`\nThe `ls` returned");
+  });
+
+  it("inserts a newline when a turn ends with a colon and the next turn starts capitalized", () => {
+    expect(thoughtTexts(["returned", ":", "Confirmed", ":", " 4 files"]))
+      .toBe("returned:\nConfirmed: 4 files");
+  });
+
+  it("resets state between independent transcript builds", () => {
+    const parser = createGrokStdoutParser();
+    parser.parseLine(JSON.stringify({ type: "thought", data: "first:" }), ts);
+    parser.reset();
+    expect(parser.parseLine(JSON.stringify({ type: "thought", data: "Second" }), ts)).toEqual([
+      { kind: "thinking", ts, text: "Second", delta: true },
+    ]);
+  });
+
+  it("does not modify assistant `text` chunks", () => {
+    // PAPA-349 review feedback: keep final assistant text streaming verbatim;
+    // the boundary heuristic is scoped to reasoning.
+    const parser = createGrokStdoutParser();
+    parser.parseLine(JSON.stringify({ type: "text", data: "Done." }), ts);
+    expect(parser.parseLine(JSON.stringify({ type: "text", data: "Next" }), ts)).toEqual([
+      { kind: "assistant", ts, text: "Next", delta: true },
+    ]);
+  });
+});

packages/adapters/grok-local/src/ui/parse-stdout.ts

@@ -1,4 +1,5 @@
 import type { TranscriptEntry } from "@paperclipai/adapter-utils";
+import { applyTurnBoundary, createTurnBoundaryState, type TurnBoundaryState } from "../shared/turn-boundary.js";
 
 function safeJsonParse(text: string): unknown {
   try {
@@ -24,7 +25,11 @@ function extractErrorText(value: unknown): string {
   return asString(record.message) || asString(record.detail) || asString(record.code);
 }
 
-export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] {
+function parseLineInternal(
+  line: string,
+  ts: string,
+  thoughtBoundary: TurnBoundaryState,
+): TranscriptEntry[] {
   const parsed = asRecord(safeJsonParse(line));
   if (!parsed) {
     return [{ kind: "stdout", ts, text: line }];
@@ -34,12 +39,14 @@ export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[]
 
   if (type === "thought") {
     const text = asString(parsed.data);
-    return text ? [{ kind: "thinking", ts, text, delta: true }] : [];
+    if (!text) return [];
+    return [{ kind: "thinking", ts, text: applyTurnBoundary(thoughtBoundary, text), delta: true }];
   }
 
   if (type === "text") {
     const text = asString(parsed.data);
-    return text ? [{ kind: "assistant", ts, text, delta: true }] : [];
+    if (!text) return [];
+    return [{ kind: "assistant", ts, text, delta: true }];
   }
 
   if (type === "error") {
@@ -59,3 +66,22 @@ export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[]
 
   return [{ kind: "system", ts, text: `event: ${type || "unknown"}` }];
 }
+
+export function createGrokStdoutParser() {
+  let thoughtBoundary = createTurnBoundaryState();
+  return {
+    parseLine(line: string, ts: string): TranscriptEntry[] {
+      return parseLineInternal(line, ts, thoughtBoundary);
+    },
+    reset() {
+      thoughtBoundary = createTurnBoundaryState();
+    },
+  };
+}
+
+// Stateless fallback for callers that haven't migrated to the stateful factory.
+// Without state, consecutive thought chunks at reasoning-turn boundaries can
+// still appear merged; prefer createGrokStdoutParser for live transcripts.
+export function parseGrokStdoutLine(line: string, ts: string): TranscriptEntry[] {
+  return parseLineInternal(line, ts, createTurnBoundaryState());
+}