Files
paperclip/packages/adapters/pi-local/src/server/execute.remote.test.ts
T
Devin Foley 09eceb952a Avoid resuming stale remote sessions (Pi adapter) (#5120)
> **Stacked PR (part 7 of 7).** Depends on:
  - PR #5114
  - PR #5115
  - PR #5116
  - PR #5117
  - PR #5118
  - PR #5119
> Diff against `master` includes commits from earlier PRs in the stack —
the new commit in this PR is the topmost one.

## Thinking Path

> - Paperclip orchestrates AI agents for zero-human companies
> - The Pi adapter persists a session jsonl per agent so subsequent runs
resume
>   conversation context instead of starting cold
> - SSH testing reproduced a real failure: a verification issue reached
terminal
>   `done` and the agent claimed success, but the proof artifact
> `manual-qa/environment-matrix/ssh/pi_local.md` was missing from the
realized
>   SSH workspace on the QA target box
> - Root cause: the saved session header recorded a different cwd than
the new
> execution cwd, but the resume eligibility check only compared
session-params
> cwd via local-style `path.resolve` (which doesn't roundtrip on remote
POSIX
> paths). The stale session got resumed and writes landed in the wrong
cwd
> - This PR tightens resume eligibility for remote targets: it adds
remote-aware
> cwd normalisation, reads the first line of the session jsonl over SSH
(`head
>   -n 1`) to verify the saved header cwd, and only resumes when both
> session-params cwd *and* the on-disk header cwd match the realised
execution
>   cwd. Stale sessions are skipped silently and the run starts cold
> - The benefit is that Pi runs across cwd-changing environments stop
> accidentally resuming each other's sessions, and proof artifacts land
where
>   reviewers expect them

## What Changed

- Added `normalizeExecutionCwd`, `executionCwdsMatch`,
`readSessionHeaderCwd`,
  and `readSavedSessionCwd` helpers in `pi-local/src/server/execute.ts`
- `readSavedSessionCwd` reads the first line of the session jsonl —
locally via
`fs.readFile`, remotely via `runAdapterExecutionTargetShellCommand`
(`head -n 1`)
- Resume eligibility now requires:
  1. Saved session id is non-empty
  2. Execution target shape matches (existing check)
  3. Session-params cwd matches the realised execution cwd
4. Session-header cwd (from the on-disk jsonl) matches the realised
execution cwd
- Stale sessions are skipped silently (run starts cold) instead of
resumed
- `execute.remote.test.ts` extended with: matching header → resume;
mismatched
header → start fresh; missing/unreadable header → start fresh; remote
head
  command failure → start fresh

## Verification

- `pnpm --filter @paperclipai/adapter-pi-local test`
- `pnpm test -- pi-local`
- Manual QA: ran a Pi agent twice in two different remote cwds,
confirmed
the second run did not pick up the first run's session and that
subsequent
  runs in the original cwd still resumed correctly

## Risks

- Adds a `head -n 1` shell call per Pi run on remote targets. Negligible
  latency (single read of session jsonl), bounded by 15s timeout.
- If the `head` call fails for unrelated reasons (transient remote
unreachability), the run will start cold instead of resuming. This is
the
safe default but worth noting — operators may see one extra cold run if
a
  remote glitches mid-session.
- No data is deleted or migrated; stale sessions remain on disk for
manual
  inspection if desired.

## Model Used

- OpenAI GPT-5.4 (reasoning effort: high) via Codex CLI
- Provider: OpenAI
- Used to author the code changes in this PR

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots — N/A
- [ ] I have updated relevant documentation to reflect my changes — N/A
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
2026-05-03 13:51:38 -07:00

509 lines
17 KiB
TypeScript

import { mkdir, mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
const {
runChildProcess,
ensureCommandResolvable,
resolveCommandForLogs,
prepareWorkspaceForSshExecution,
restoreWorkspaceFromSshExecution,
runSshCommand,
syncDirectoryToSsh,
startAdapterExecutionTargetPaperclipBridge,
} = vi.hoisted(() => ({
runChildProcess: vi.fn(async () => ({
exitCode: 0,
signal: null,
timedOut: false,
stdout: JSON.stringify({
type: "turn_end",
message: {
role: "assistant",
content: "done",
usage: {
input: 10,
output: 20,
cacheRead: 0,
cost: { total: 0.01 },
},
},
toolResults: [],
}),
stderr: "",
pid: 123,
startedAt: new Date().toISOString(),
})),
ensureCommandResolvable: vi.fn(async () => undefined),
resolveCommandForLogs: vi.fn(async () => "ssh://fixture@127.0.0.1:2222/remote/workspace :: pi"),
prepareWorkspaceForSshExecution: vi.fn(async () => undefined),
restoreWorkspaceFromSshExecution: vi.fn(async () => undefined),
runSshCommand: vi.fn(async () => ({
stdout: "",
stderr: "",
exitCode: 0,
})),
syncDirectoryToSsh: vi.fn(async () => undefined),
startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({
env: {
PAPERCLIP_API_URL: "http://127.0.0.1:4310",
PAPERCLIP_API_KEY: "bridge-token",
PAPERCLIP_API_BRIDGE_MODE: "queue_v1",
},
stop: async () => {},
})),
}));
vi.mock("@paperclipai/adapter-utils/server-utils", async () => {
const actual = await vi.importActual<typeof import("@paperclipai/adapter-utils/server-utils")>(
"@paperclipai/adapter-utils/server-utils",
);
return {
...actual,
ensureCommandResolvable,
resolveCommandForLogs,
runChildProcess,
};
});
vi.mock("@paperclipai/adapter-utils/ssh", async () => {
const actual = await vi.importActual<typeof import("@paperclipai/adapter-utils/ssh")>(
"@paperclipai/adapter-utils/ssh",
);
return {
...actual,
prepareWorkspaceForSshExecution,
restoreWorkspaceFromSshExecution,
runSshCommand,
syncDirectoryToSsh,
};
});
vi.mock("@paperclipai/adapter-utils/execution-target", async () => {
const actual = await vi.importActual<typeof import("@paperclipai/adapter-utils/execution-target")>(
"@paperclipai/adapter-utils/execution-target",
);
return {
...actual,
startAdapterExecutionTargetPaperclipBridge,
};
});
import { execute } from "./execute.js";
describe("pi remote execution", () => {
const cleanupDirs: string[] = [];
afterEach(async () => {
vi.clearAllMocks();
while (cleanupDirs.length > 0) {
const dir = cleanupDirs.pop();
if (!dir) continue;
await rm(dir, { recursive: true, force: true }).catch(() => undefined);
}
});
it("prepares the workspace, syncs Pi skills, and restores workspace changes for remote SSH execution", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-remote-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
const alternateWorkspaceDir = path.join(rootDir, "workspace-other");
await mkdir(workspaceDir, { recursive: true });
await mkdir(alternateWorkspaceDir, { recursive: true });
const result = await execute({
runId: "run-1",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Pi Builder",
adapterType: "pi_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: "pi",
model: "openai/gpt-5.4-mini",
},
context: {
paperclipWorkspace: {
cwd: workspaceDir,
source: "project_primary",
},
paperclipWorkspaces: [
{
workspaceId: "workspace-1",
cwd: workspaceDir,
repoUrl: "https://github.com/paperclipai/paperclip.git",
repoRef: "main",
},
{
workspaceId: "workspace-2",
cwd: alternateWorkspaceDir,
repoUrl: "https://github.com/paperclipai/paperclip.git",
repoRef: "feature/other",
},
],
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
expect(result.sessionParams).toMatchObject({
cwd: "/remote/workspace",
remoteExecution: {
transport: "ssh",
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteCwd: "/remote/workspace",
},
});
expect(String(result.sessionId)).toContain("/remote/workspace/.paperclip-runtime/pi/sessions/");
expect(prepareWorkspaceForSshExecution).toHaveBeenCalledTimes(1);
expect(syncDirectoryToSsh).toHaveBeenCalledTimes(1);
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
remoteDir: "/remote/workspace/.paperclip-runtime/pi/skills",
followSymlinks: true,
}));
expect(runSshCommand).toHaveBeenCalledWith(
expect.anything(),
expect.stringContaining(".paperclip-runtime/pi/sessions"),
expect.anything(),
);
const call = runChildProcess.mock.calls[0] as unknown as
| [string, string, string[], { env: Record<string, string>; remoteExecution?: { remoteCwd: string } | null }]
| undefined;
expect(call?.[2]).toContain("--session");
expect(call?.[2]).toContain("--skill");
expect(call?.[2]).toContain("/remote/workspace/.paperclip-runtime/pi/skills");
expect(call?.[3].env.PAPERCLIP_WORKSPACE_CWD).toBe("/remote/workspace");
expect(JSON.parse(call?.[3].env.PAPERCLIP_WORKSPACES_JSON ?? "[]")).toEqual([
{
workspaceId: "workspace-1",
cwd: "/remote/workspace",
repoUrl: "https://github.com/paperclipai/paperclip.git",
repoRef: "main",
},
{
workspaceId: "workspace-2",
repoUrl: "https://github.com/paperclipai/paperclip.git",
repoRef: "feature/other",
},
]);
expect(call?.[3].env.PAPERCLIP_API_URL).toBe("http://127.0.0.1:4310");
expect(call?.[3].env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1");
expect(call?.[3].remoteExecution?.remoteCwd).toBe("/remote/workspace");
expect(startAdapterExecutionTargetPaperclipBridge).toHaveBeenCalledTimes(1);
expect(restoreWorkspaceFromSshExecution).toHaveBeenCalledTimes(1);
});
it("resumes saved Pi sessions for remote SSH execution only when the identity matches", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-remote-resume-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
await mkdir(workspaceDir, { recursive: true });
runSshCommand.mockImplementation(async (...args: unknown[]) => {
const command = String(args[1] ?? "");
if (command.includes("head -n 1") && command.includes("session-123.jsonl")) {
return {
stdout: `${JSON.stringify({ type: "session", cwd: "/remote/workspace" })}\n`,
stderr: "",
exitCode: 0,
};
}
return {
stdout: "",
stderr: "",
exitCode: 0,
};
});
await execute({
runId: "run-ssh-resume",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Pi Builder",
adapterType: "pi_local",
adapterConfig: {},
},
runtime: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
sessionParams: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
cwd: "/remote/workspace",
remoteExecution: {
transport: "ssh",
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteCwd: "/remote/workspace",
},
},
sessionDisplayId: "session-123",
taskKey: null,
},
config: {
command: "pi",
model: "openai/gpt-5.4-mini",
},
context: {
paperclipWorkspace: {
cwd: workspaceDir,
source: "project_primary",
},
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[]] | undefined;
expect(call?.[2]).toContain("--session");
expect(call?.[2]).toContain("/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl");
});
it("starts a fresh remote Pi session when the saved session header cwd points at a different workspace", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-remote-stale-session-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
await mkdir(workspaceDir, { recursive: true });
runSshCommand.mockImplementation(async (...args: unknown[]) => {
const command = String(args[1] ?? "");
if (command.includes("head -n 1") && command.includes("session-123.jsonl")) {
return {
stdout: `${JSON.stringify({ type: "session", cwd: "/remote/old-workspace" })}\n`,
stderr: "",
exitCode: 0,
};
}
return {
stdout: "",
stderr: "",
exitCode: 0,
};
});
await execute({
runId: "run-ssh-stale-session",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Pi Builder",
adapterType: "pi_local",
adapterConfig: {},
},
runtime: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
sessionParams: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
cwd: "/remote/workspace",
remoteExecution: {
transport: "ssh",
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteCwd: "/remote/workspace",
},
},
sessionDisplayId: "session-123",
taskKey: null,
},
config: {
command: "pi",
model: "openai/gpt-5.4-mini",
},
context: {
paperclipWorkspace: {
cwd: workspaceDir,
source: "project_primary",
},
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[]] | undefined;
const sessionIndex = call?.[2].indexOf("--session") ?? -1;
expect(sessionIndex).toBeGreaterThanOrEqual(0);
const usedSession = sessionIndex >= 0 ? call?.[2][sessionIndex + 1] : null;
expect(usedSession).toContain("/remote/workspace/.paperclip-runtime/pi/sessions/");
expect(usedSession).not.toBe("/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl");
});
it("starts a fresh remote Pi session when the saved session header is empty or unreadable", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-remote-empty-header-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
await mkdir(workspaceDir, { recursive: true });
runSshCommand.mockImplementation(async (...args: unknown[]) => {
const command = String(args[1] ?? "");
if (command.includes("head -n 1") && command.includes("session-123.jsonl")) {
return { stdout: "", stderr: "", exitCode: 0 };
}
return { stdout: "", stderr: "", exitCode: 0 };
});
await execute({
runId: "run-ssh-empty-header",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Pi Builder",
adapterType: "pi_local",
adapterConfig: {},
},
runtime: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
sessionParams: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
cwd: "/remote/workspace",
remoteExecution: {
transport: "ssh",
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteCwd: "/remote/workspace",
},
},
sessionDisplayId: "session-123",
taskKey: null,
},
config: { command: "pi", model: "openai/gpt-5.4-mini" },
context: {
paperclipWorkspace: { cwd: workspaceDir, source: "project_primary" },
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[]] | undefined;
const sessionIndex = call?.[2].indexOf("--session") ?? -1;
expect(sessionIndex).toBeGreaterThanOrEqual(0);
const usedSession = sessionIndex >= 0 ? call?.[2][sessionIndex + 1] : null;
expect(usedSession).not.toBe("/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl");
});
it("starts a fresh remote Pi session when the remote head command fails", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pi-remote-head-failure-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
await mkdir(workspaceDir, { recursive: true });
runSshCommand.mockImplementation(async (...args: unknown[]) => {
const command = String(args[1] ?? "");
if (command.includes("head -n 1") && command.includes("session-123.jsonl")) {
throw Object.assign(new Error("ssh: connect failed"), {
stdout: "",
stderr: "ssh: connect failed",
code: "ENOENT",
});
}
return { stdout: "", stderr: "", exitCode: 0 };
});
await execute({
runId: "run-ssh-head-failure",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Pi Builder",
adapterType: "pi_local",
adapterConfig: {},
},
runtime: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
sessionParams: {
sessionId: "/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl",
cwd: "/remote/workspace",
remoteExecution: {
transport: "ssh",
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteCwd: "/remote/workspace",
},
},
sessionDisplayId: "session-123",
taskKey: null,
},
config: { command: "pi", model: "openai/gpt-5.4-mini" },
context: {
paperclipWorkspace: { cwd: workspaceDir, source: "project_primary" },
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[]] | undefined;
const sessionIndex = call?.[2].indexOf("--session") ?? -1;
expect(sessionIndex).toBeGreaterThanOrEqual(0);
const usedSession = sessionIndex >= 0 ? call?.[2][sessionIndex + 1] : null;
expect(usedSession).not.toBe("/remote/workspace/.paperclip-runtime/pi/sessions/session-123.jsonl");
});
});