diff --git a/packages/adapter-utils/src/command-managed-runtime.test.ts b/packages/adapter-utils/src/command-managed-runtime.test.ts new file mode 100644 index 00000000..9be9c062 --- /dev/null +++ b/packages/adapter-utils/src/command-managed-runtime.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +import { prepareCommandManagedRuntime } from "./command-managed-runtime.js"; +import type { RunProcessResult } from "./server-utils.js"; + +const execFile = promisify(execFileCallback); + +describe("command managed runtime", () => { + const cleanupDirs: string[] = []; + + afterEach(async () => { + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (!dir) continue; + await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + it("keeps the runtime overlay out of sandbox workspace sync by default", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-runtime-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(path.join(localWorkspaceDir, ".paperclip-runtime"), { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "local workspace\n", "utf8"); + await writeFile(path.join(localWorkspaceDir, ".paperclip-runtime", "state.json"), "{\"keep\":true}\n", "utf8"); + + const calls: Array<{ + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + }> = []; + const runner = { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + }): Promise => { + calls.push({ ...input }); + const startedAt = new Date().toISOString(); + const env = { + ...process.env, + ...input.env, + }; + const command = input.command === "sh" ? "/bin/sh" : input.command; + const args = [...(input.args ?? [])]; + if (input.stdin != null && input.command === "sh" && args[0] === "-lc" && typeof args[1] === "string") { + env.PAPERCLIP_TEST_STDIN = input.stdin; + args[1] = `printf '%s' \"$PAPERCLIP_TEST_STDIN\" | (${args[1]})`; + } + try { + const result = await execFile(command, args, { + cwd: input.cwd, + env, + maxBuffer: 32 * 1024 * 1024, + timeout: input.timeoutMs, + }); + return { + exitCode: 0, + signal: null, + timedOut: false, + stdout: result.stdout, + stderr: result.stderr, + pid: null, + startedAt, + }; + } catch (error) { + const err = error as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + code?: string | number | null; + signal?: NodeJS.Signals | null; + killed?: boolean; + }; + return { + exitCode: typeof err.code === "number" ? err.code : null, + signal: err.signal ?? null, + timedOut: Boolean(err.killed && input.timeoutMs), + stdout: err.stdout ?? "", + stderr: err.stderr ?? "", + pid: null, + startedAt, + }; + } + }, + }; + + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "claude", + workspaceLocalDir: localWorkspaceDir, + }); + + await expect(readFile(path.join(remoteWorkspaceDir, "README.md"), "utf8")).resolves.toBe("local workspace\n"); + await expect(readFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "state.json"), "utf8")).rejects + .toMatchObject({ code: "ENOENT" }); + expect(calls.every((call) => call.stdin == null)).toBe(true); + + await mkdir(path.join(remoteWorkspaceDir, ".paperclip-runtime"), { recursive: true }); + await writeFile(path.join(remoteWorkspaceDir, "README.md"), "remote workspace\n", "utf8"); + await writeFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "remote-state.json"), "{\"remote\":true}\n", "utf8"); + await prepared.restoreWorkspace(); + + await expect(readFile(path.join(localWorkspaceDir, "README.md"), "utf8")).resolves.toBe("remote workspace\n"); + await expect(readFile(path.join(localWorkspaceDir, ".paperclip-runtime", "state.json"), "utf8")).resolves + .toBe("{\"keep\":true}\n"); + await expect(readFile(path.join(localWorkspaceDir, ".paperclip-runtime", "remote-state.json"), "utf8")).rejects + .toMatchObject({ code: "ENOENT" }); + expect(calls.every((call) => call.stdin == null)).toBe(true); + }); +}); diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index cb3e2edd..706c3fd7 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -35,6 +35,12 @@ function shellQuote(value: string) { return `'${value.replace(/'/g, `'"'"'`)}'`; } +function mergeRuntimeExcludes(entries: string[] | undefined): string[] { + return [...new Set([".paperclip-runtime", ...(entries ?? [])])]; +} + +const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024; + function toBuffer(bytes: Buffer | Uint8Array | ArrayBuffer): Buffer { if (Buffer.isBuffer(bytes)) return bytes; if (bytes instanceof ArrayBuffer) return Buffer.from(bytes); @@ -48,7 +54,7 @@ function requireSuccessfulResult(result: RunProcessResult, action: string): void throw new Error(`${action} failed with exit code ${result.exitCode ?? "null"}${detail}`); } -function createCommandManagedRuntimeClient(input: { +export function createCommandManagedRuntimeClient(input: { runner: CommandManagedRuntimeRunner; remoteCwd: string; timeoutMs: number; @@ -71,15 +77,39 @@ function createCommandManagedRuntimeClient(input: { }, writeFile: async (remotePath, bytes) => { const body = toBuffer(bytes).toString("base64"); + const remoteDir = path.posix.dirname(remotePath); + const remoteTempPath = `${remotePath}.paperclip-upload.b64`; + await runShell( - `mkdir -p ${shellQuote(path.posix.dirname(remotePath))} && base64 -d > ${shellQuote(remotePath)}`, - { stdin: body }, + `mkdir -p ${shellQuote(remoteDir)} && rm -f ${shellQuote(remoteTempPath)} && : > ${shellQuote(remoteTempPath)}`, + ); + for (let offset = 0; offset < body.length; offset += REMOTE_WRITE_BASE64_CHUNK_SIZE) { + const chunk = body.slice(offset, offset + REMOTE_WRITE_BASE64_CHUNK_SIZE); + await runShell(`printf '%s' ${shellQuote(chunk)} >> ${shellQuote(remoteTempPath)}`); + } + await runShell( + `base64 -d < ${shellQuote(remoteTempPath)} > ${shellQuote(remotePath)} && rm -f ${shellQuote(remoteTempPath)}`, ); }, readFile: async (remotePath) => { const result = await runShell(`base64 < ${shellQuote(remotePath)}`); return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64"); }, + listFiles: async (remotePath) => { + const result = await runShell( + `if [ -d ${shellQuote(remotePath)} ]; then ` + + `for entry in ${shellQuote(remotePath)}/*; do ` + + `[ -f "$entry" ] || continue; ` + + `basename "$entry"; ` + + `done; ` + + `fi`, + ); + return result.stdout + .split(/\r?\n/) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .sort((left, right) => left.localeCompare(right)); + }, remove: async (remotePath) => { const result = await input.runner.execute({ command: "sh", @@ -145,7 +175,7 @@ export async function prepareCommandManagedRuntime(input: { adapterKey: input.adapterKey, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir, - workspaceExclude: input.workspaceExclude, + workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, }); diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index b5c3f011..cda63354 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -1,14 +1,59 @@ -import { describe, expect, it, vi } from "vitest"; +import { createServer } from "node:http"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetToRemoteSpec, runAdapterExecutionTargetProcess, runAdapterExecutionTargetShellCommand, + startAdapterExecutionTargetPaperclipBridge, type AdapterSandboxExecutionTarget, } from "./execution-target.js"; +import { runChildProcess } from "./server-utils.js"; describe("sandbox adapter execution targets", () => { + const cleanupDirs: string[] = []; + + afterEach(async () => { + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (!dir) continue; + await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + function createLocalSandboxRunner() { + let counter = 0; + return { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + onSpawn?: (meta: { pid: number; startedAt: string }) => Promise; + }) => { + counter += 1; + return runChildProcess(`sandbox-run-${counter}`, input.command, input.args ?? [], { + cwd: input.cwd ?? process.cwd(), + env: input.env ?? {}, + stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), + graceSec: 5, + onLog: input.onLog ?? (async () => {}), + onSpawn: input.onSpawn + ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt }) + : undefined, + }); + }, + }; + } + it("executes through the provider-neutral runner without a remote spec", async () => { const runner = { execute: vi.fn(async () => ({ @@ -58,6 +103,7 @@ describe("sandbox adapter execution targets", () => { environmentId: "env-1", leaseId: "lease-1", remoteCwd: "/workspace", + paperclipTransport: "bridge", }); }); @@ -93,4 +139,154 @@ describe("sandbox adapter execution targets", () => { timeoutMs: 7000, })); }); + + it("starts a localhost Paperclip bridge for sandbox targets in bridge mode", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + const requests: Array<{ method: string; url: string; auth: string | null; runId: string | null }> = []; + const apiServer = createServer((req, res) => { + requests.push({ + method: req.method ?? "GET", + url: req.url ?? "/", + auth: req.headers.authorization ?? null, + runId: typeof req.headers["x-paperclip-run-id"] === "string" ? req.headers["x-paperclip-run-id"] : null, + }); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + paperclipTransport: "bridge", + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + }); + try { + expect(bridge).not.toBeNull(); + expect(bridge?.env.PAPERCLIP_API_URL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(bridge?.env.PAPERCLIP_API_KEY).not.toBe("real-run-jwt"); + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); + + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/agents/me`, { + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + accept: "application/json", + }, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(requests).toEqual([{ + method: "GET", + url: "/api/agents/me", + auth: "Bearer real-run-jwt", + runId: "run-bridge", + }]); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); + + it("fails oversized host responses with a 502 before returning them to the sandbox client", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-limit-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + const requests: Array<{ method: string; url: string; auth: string | null; runId: string | null }> = []; + const largeBody = "x".repeat(64); + const apiServer = createServer((req, res) => { + requests.push({ + method: req.method ?? "GET", + url: req.url ?? "/", + auth: req.headers.authorization ?? null, + runId: typeof req.headers["x-paperclip-run-id"] === "string" ? req.headers["x-paperclip-run-id"] : null, + }); + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(largeBody, "utf8")), + }); + res.end(largeBody); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + paperclipTransport: "bridge", + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge-limit", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + maxBodyBytes: 32, + }); + try { + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/agents/me`, { + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + accept: "application/json", + }, + }); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ + error: "Bridge response body exceeded the configured size limit of 32 bytes.", + }); + expect(requests).toEqual([{ + method: "GET", + url: "/api/agents/me", + auth: "Bearer real-run-jwt", + runId: "run-bridge-limit", + }]); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); }); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 861fb025..58c128e8 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -10,6 +10,14 @@ import { remoteExecutionSessionMatches, type RemoteManagedRuntimeAsset, } from "./remote-managed-runtime.js"; +import { + createCommandManagedSandboxCallbackBridgeQueueClient, + createSandboxCallbackBridgeAsset, + createSandboxCallbackBridgeToken, + DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, + startSandboxCallbackBridgeServer, + startSandboxCallbackBridgeWorker, +} from "./sandbox-callback-bridge.js"; import { parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js"; import { ensureCommandResolvable, @@ -43,6 +51,7 @@ export interface AdapterSandboxExecutionTarget { leaseId?: string | null; remoteCwd: string; paperclipApiUrl?: string | null; + paperclipTransport?: "direct" | "bridge"; timeoutMs?: number | null; runner?: CommandManagedRuntimeRunner; } @@ -82,6 +91,11 @@ export interface AdapterExecutionTargetShellOptions { onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; } +export interface AdapterExecutionTargetPaperclipBridgeHandle { + env: Record; + stop(): Promise; +} + function parseObject(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) @@ -96,6 +110,31 @@ function readStringMeta(parsed: Record, key: string): string | return readString(parsed[key]); } +function resolveHostForUrl(rawHost: string): string { + const host = rawHost.trim(); + if (!host || host === "0.0.0.0" || host === "::") return "localhost"; + if (host.includes(":") && !host.startsWith("[") && !host.endsWith("]")) return `[${host}]`; + return host; +} + +function resolveDefaultPaperclipApiUrl(): string { + const runtimeHost = resolveHostForUrl( + process.env.PAPERCLIP_LISTEN_HOST ?? process.env.HOST ?? "localhost", + ); + // 3100 matches the default Paperclip dev server port when the runtime does not provide one. + const runtimePort = process.env.PAPERCLIP_LISTEN_PORT ?? process.env.PORT ?? "3100"; + return `http://${runtimeHost}:${runtimePort}`; +} + +function resolveSandboxPaperclipTransport( + target: Pick, +): "direct" | "bridge" { + if (target.paperclipTransport === "direct" || target.paperclipTransport === "bridge") { + return target.paperclipTransport; + } + return target.paperclipApiUrl ? "direct" : "bridge"; +} + function isAdapterExecutionTargetInstance(value: unknown): value is AdapterExecutionTarget { const parsed = parseObject(value); if (parsed.kind === "local") return true; @@ -146,9 +185,18 @@ export function adapterExecutionTargetPaperclipApiUrl( ): string | null { if (target?.kind !== "remote") return null; if (target.transport === "ssh") return target.paperclipApiUrl ?? target.spec.paperclipApiUrl ?? null; + if (resolveSandboxPaperclipTransport(target) === "bridge") return null; return target.paperclipApiUrl ?? null; } +export function adapterExecutionTargetUsesPaperclipBridge( + target: AdapterExecutionTarget | null | undefined, +): boolean { + return target?.kind === "remote" && + target.transport === "sandbox" && + resolveSandboxPaperclipTransport(target) === "bridge"; +} + export function describeAdapterExecutionTarget( target: AdapterExecutionTarget | null | undefined, ): string { @@ -410,13 +458,15 @@ export function adapterExecutionTargetSessionIdentity( ): Record | null { if (!target || target.kind === "local") return null; if (target.transport === "ssh") return buildRemoteExecutionSessionIdentity(target.spec); + const paperclipTransport = resolveSandboxPaperclipTransport(target); return { transport: "sandbox", providerKey: target.providerKey ?? null, environmentId: target.environmentId ?? null, leaseId: target.leaseId ?? null, remoteCwd: target.remoteCwd, - ...(target.paperclipApiUrl ? { paperclipApiUrl: target.paperclipApiUrl } : {}), + paperclipTransport, + ...(paperclipTransport === "direct" && target.paperclipApiUrl ? { paperclipApiUrl: target.paperclipApiUrl } : {}), }; } @@ -436,6 +486,7 @@ export function adapterExecutionTargetSessionMatches( readStringMeta(parsedSaved, "environmentId") === current?.environmentId && readStringMeta(parsedSaved, "leaseId") === current?.leaseId && readStringMeta(parsedSaved, "remoteCwd") === current?.remoteCwd && + readStringMeta(parsedSaved, "paperclipTransport") === (current?.paperclipTransport ?? null) && readStringMeta(parsedSaved, "paperclipApiUrl") === (current?.paperclipApiUrl ?? null) ); } @@ -468,6 +519,7 @@ export function parseAdapterExecutionTarget(value: unknown): AdapterExecutionTar if (kind === "remote" && readStringMeta(parsed, "transport") === "sandbox") { const remoteCwd = readStringMeta(parsed, "remoteCwd"); + const paperclipTransport = readStringMeta(parsed, "paperclipTransport"); if (!remoteCwd) return null; return { kind: "remote", @@ -477,6 +529,10 @@ export function parseAdapterExecutionTarget(value: unknown): AdapterExecutionTar leaseId: readStringMeta(parsed, "leaseId"), remoteCwd, paperclipApiUrl: readStringMeta(parsed, "paperclipApiUrl"), + paperclipTransport: + paperclipTransport === "direct" || paperclipTransport === "bridge" + ? paperclipTransport + : undefined, timeoutMs: typeof parsed.timeoutMs === "number" ? parsed.timeoutMs : null, }; } @@ -583,3 +639,172 @@ export function runtimeAssetDir( ): string { return prepared.assetDirs[key] ?? path.posix.join(fallbackRemoteCwd, ".paperclip-runtime", key); } + +function buildBridgeResponseHeaders(response: Response): Record { + const out: Record = {}; + for (const key of ["content-type", "etag", "last-modified"]) { + const value = response.headers.get(key); + if (value && value.trim().length > 0) out[key] = value.trim(); + } + return out; +} + +function buildBridgeForwardUrl(baseUrl: string, request: { path: string; query: string }): URL { + const url = new URL(request.path, baseUrl); + const query = request.query.trim(); + url.search = query.startsWith("?") ? query.slice(1) : query; + return url; +} + +function bridgeResponseBodyLimitError(maxBodyBytes: number): Error { + return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`); +} + +async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: number): Promise { + const rawContentLength = response.headers.get("content-length"); + if (rawContentLength) { + const contentLength = Number.parseInt(rawContentLength, 10); + if (Number.isFinite(contentLength) && contentLength > maxBodyBytes) { + throw bridgeResponseBodyLimitError(maxBodyBytes); + } + } + + if (!response.body) { + return ""; + } + + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let totalBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + totalBytes += value.byteLength; + if (totalBytes > maxBodyBytes) { + await reader.cancel().catch(() => undefined); + throw bridgeResponseBodyLimitError(maxBodyBytes); + } + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks, totalBytes).toString("utf8"); +} + +export async function startAdapterExecutionTargetPaperclipBridge(input: { + runId: string; + target: AdapterExecutionTarget | null | undefined; + runtimeRootDir: string | null | undefined; + adapterKey: string; + hostApiToken: string | null | undefined; + hostApiUrl?: string | null; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + maxBodyBytes?: number | null; +}): Promise { + if (!adapterExecutionTargetUsesPaperclipBridge(input.target)) { + return null; + } + if (!input.target || input.target.kind !== "remote" || input.target.transport !== "sandbox") { + return null; + } + + const target = input.target; + const onLog = input.onLog ?? (async () => {}); + const hostApiToken = input.hostApiToken?.trim() ?? ""; + if (hostApiToken.length === 0) { + throw new Error("Sandbox bridge mode requires a host-side Paperclip API token."); + } + + const runtimeRootDir = + input.runtimeRootDir?.trim().length + ? input.runtimeRootDir.trim() + : path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey); + const bridgeRuntimeDir = path.posix.join(runtimeRootDir, "paperclip-bridge"); + const queueDir = path.posix.join(bridgeRuntimeDir, "queue"); + const assetRemoteDir = path.posix.join(bridgeRuntimeDir, "server"); + const bridgeToken = createSandboxCallbackBridgeToken(); + const maxBodyBytes = + typeof input.maxBodyBytes === "number" && Number.isFinite(input.maxBodyBytes) && input.maxBodyBytes > 0 + ? Math.trunc(input.maxBodyBytes) + : DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES; + const hostApiUrl = + input.hostApiUrl?.trim() || + process.env.PAPERCLIP_RUNTIME_API_URL?.trim() || + process.env.PAPERCLIP_API_URL?.trim() || + resolveDefaultPaperclipApiUrl(); + + await onLog( + "stdout", + `[paperclip] Starting sandbox callback bridge for ${input.adapterKey} in ${bridgeRuntimeDir}.\n`, + ); + + const bridgeAsset = await createSandboxCallbackBridgeAsset(); + let server: Awaited> | null = null; + let worker: Awaited> | null = null; + try { + const client = createCommandManagedSandboxCallbackBridgeQueueClient({ + runner: requireSandboxRunner(target), + remoteCwd: target.remoteCwd, + timeoutMs: target.timeoutMs, + }); + worker = await startSandboxCallbackBridgeWorker({ + client, + queueDir, + maxBodyBytes, + handleRequest: async (request) => { + const headers = new Headers(); + for (const [key, value] of Object.entries(request.headers)) { + if (value.trim().length === 0) continue; + headers.set(key, value); + } + headers.set("authorization", `Bearer ${hostApiToken}`); + headers.set("x-paperclip-run-id", input.runId); + const method = request.method.trim().toUpperCase() || "GET"; + const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), { + method, + headers, + ...(method === "GET" || method === "HEAD" ? {} : { body: request.body }), + signal: AbortSignal.timeout(30_000), + }); + return { + status: response.status, + headers: buildBridgeResponseHeaders(response), + body: await readBridgeForwardResponseBody(response, maxBodyBytes), + }; + }, + }); + server = await startSandboxCallbackBridgeServer({ + runner: requireSandboxRunner(target), + remoteCwd: target.remoteCwd, + assetRemoteDir, + queueDir, + bridgeToken, + bridgeAsset, + timeoutMs: target.timeoutMs, + maxBodyBytes, + }); + } catch (error) { + await Promise.allSettled([ + server?.stop(), + worker?.stop(), + bridgeAsset.cleanup(), + ]); + throw error; + } + + return { + env: { + PAPERCLIP_API_URL: server.baseUrl, + PAPERCLIP_API_KEY: bridgeToken, + PAPERCLIP_API_BRIDGE_MODE: "queue_v1", + }, + stop: async () => { + await Promise.allSettled([ + server?.stop(), + ]); + await Promise.allSettled([ + worker?.stop(), + bridgeAsset.cleanup(), + ]); + }, + }; +} diff --git a/packages/adapter-utils/src/index.ts b/packages/adapter-utils/src/index.ts index 6770ae51..9dcf068b 100644 --- a/packages/adapter-utils/src/index.ts +++ b/packages/adapter-utils/src/index.ts @@ -54,3 +54,15 @@ export { redactTranscriptEntryPaths, } from "./log-redaction.js"; export { inferOpenAiCompatibleBiller } from "./billing.js"; +// Keep the root adapter-utils entry browser-safe because the UI imports it. +// The sandbox callback bridge stays available via its dedicated subpath export. +export type { + SandboxCallbackBridgeRequest, + SandboxCallbackBridgeResponse, + SandboxCallbackBridgeAsset, + SandboxCallbackBridgeDirectories, + SandboxCallbackBridgeRouteRule, + SandboxCallbackBridgeQueueClient, + SandboxCallbackBridgeWorkerHandle, + StartedSandboxCallbackBridgeServer, +} from "./sandbox-callback-bridge.js"; diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts new file mode 100644 index 00000000..d036771d --- /dev/null +++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts @@ -0,0 +1,610 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +import { prepareCommandManagedRuntime } from "./command-managed-runtime.js"; +import { + createFileSystemSandboxCallbackBridgeQueueClient, + createSandboxCallbackBridgeAsset, + createSandboxCallbackBridgeToken, + sandboxCallbackBridgeDirectories, + startSandboxCallbackBridgeServer, + startSandboxCallbackBridgeWorker, +} from "./sandbox-callback-bridge.js"; +import type { RunProcessResult } from "./server-utils.js"; + +const execFile = promisify(execFileCallback); + +describe("sandbox callback bridge", () => { + const cleanupDirs: string[] = []; + const cleanupFns: Array<() => Promise> = []; + + function createExecRunner() { + return { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + }): Promise => { + const startedAt = new Date().toISOString(); + const env = { + ...process.env, + ...input.env, + }; + const command = input.command === "sh" ? "/bin/sh" : input.command; + const args = [...(input.args ?? [])]; + if (input.stdin != null && input.command === "sh" && args[0] === "-lc" && typeof args[1] === "string") { + env.PAPERCLIP_TEST_STDIN = input.stdin; + args[1] = `printf '%s' \"$PAPERCLIP_TEST_STDIN\" | (${args[1]})`; + } + try { + const result = await execFile(command, args, { + cwd: input.cwd, + env, + maxBuffer: 32 * 1024 * 1024, + timeout: input.timeoutMs, + }); + return { + exitCode: 0, + signal: null, + timedOut: false, + stdout: result.stdout, + stderr: result.stderr, + pid: null, + startedAt, + }; + } catch (error) { + const err = error as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + code?: string | number | null; + signal?: NodeJS.Signals | null; + killed?: boolean; + }; + return { + exitCode: typeof err.code === "number" ? err.code : null, + signal: err.signal ?? null, + timedOut: Boolean(err.killed && input.timeoutMs), + stdout: err.stdout ?? "", + stderr: err.stderr ?? "", + pid: null, + startedAt, + }; + } + }, + }; + } + + async function waitForJsonFile(directory: string, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const entries = await readdir(directory).catch(() => []); + const match = entries.find((entry) => entry.endsWith(".json")); + if (match) return match; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for a JSON file in ${directory}.`); + } + + afterEach(async () => { + while (cleanupFns.length > 0) { + const cleanup = cleanupFns.pop(); + if (!cleanup) continue; + await cleanup().catch(() => undefined); + } + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (!dir) continue; + await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + it("round-trips localhost bridge requests over the sandbox queue without forwarding the bridge token", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-runtime-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge test\n", "utf8"); + + const runner = createExecRunner(); + + const bridgeAsset = await createSandboxCallbackBridgeAsset(); + cleanupFns.push(bridgeAsset.cleanup); + + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "codex", + workspaceLocalDir: localWorkspaceDir, + assets: [ + { + key: "bridge", + localDir: bridgeAsset.localDir, + }, + ], + }); + + const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const bridgeToken = createSandboxCallbackBridgeToken(); + const seenRequests: Array<{ + method: string; + path: string; + query: string; + headers: Record; + body: string; + }> = []; + + const worker = await startSandboxCallbackBridgeWorker({ + client: createFileSystemSandboxCallbackBridgeQueueClient(), + queueDir, + authorizeRequest: async (request) => + request.path === "/api/agents/me" ? null : `Route not allowed: ${request.method} ${request.path}`, + handleRequest: async (request) => { + seenRequests.push({ + method: request.method, + path: request.path, + query: request.query, + headers: request.headers, + body: request.body, + }); + return { + status: 200, + headers: { + "content-type": "application/json", + etag: '"bridge-rev-1"', + "last-modified": "Tue, 01 Apr 2025 00:00:00 GMT", + }, + body: JSON.stringify({ + ok: true, + method: request.method, + path: request.path, + }), + }; + }, + }); + cleanupFns.push(async () => { + await worker.stop(); + }); + + const bridge = await startSandboxCallbackBridgeServer({ + runner, + remoteCwd: remoteWorkspaceDir, + assetRemoteDir: prepared.assetDirs.bridge, + queueDir, + bridgeToken, + timeoutMs: 30_000, + }); + cleanupFns.push(async () => { + await bridge.stop(); + }); + + const okResponse = await fetch(`${bridge.baseUrl}/api/agents/me?view=compact`, { + headers: { + authorization: `Bearer ${bridgeToken}`, + accept: "application/json", + "if-none-match": '"client-cache-key"', + "x-paperclip-run-id": "run-bridge-1", + "x-bridge-debug": "drop-me", + }, + }); + expect(okResponse.status).toBe(200); + expect(okResponse.headers.get("content-type")).toContain("application/json"); + expect(okResponse.headers.get("etag")).toBe('"bridge-rev-1"'); + expect(okResponse.headers.get("last-modified")).toBe("Tue, 01 Apr 2025 00:00:00 GMT"); + await expect(okResponse.json()).resolves.toMatchObject({ + ok: true, + method: "GET", + path: "/api/agents/me", + }); + + const deniedResponse = await fetch(`${bridge.baseUrl}/api/issues/issue-1`, { + method: "PATCH", + headers: { + authorization: `Bearer ${bridgeToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ status: "in_progress" }), + }); + expect(deniedResponse.status).toBe(403); + await expect(deniedResponse.json()).resolves.toMatchObject({ + error: "Route not allowed: PATCH /api/issues/issue-1", + }); + + const unauthorizedResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, { + headers: { + authorization: "Bearer wrong-token", + }, + }); + expect(unauthorizedResponse.status).toBe(401); + await expect(unauthorizedResponse.json()).resolves.toMatchObject({ + error: "Invalid bridge token.", + }); + + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]).toMatchObject({ + method: "GET", + path: "/api/agents/me", + query: "?view=compact", + body: "", + headers: { + accept: "application/json", + "if-none-match": '"client-cache-key"', + }, + }); + expect(seenRequests[0]?.headers.authorization).toBeUndefined(); + expect(seenRequests[0]?.headers["x-paperclip-run-id"]).toBeUndefined(); + + }); + + it("denies non-allowlisted requests by default", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-default-policy-")); + cleanupDirs.push(rootDir); + + const queueDir = path.posix.join(rootDir, "queue"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + let handled = 0; + + const worker = await startSandboxCallbackBridgeWorker({ + client: createFileSystemSandboxCallbackBridgeQueueClient(), + queueDir, + handleRequest: async () => { + handled += 1; + return { + status: 200, + body: "should not happen", + }; + }, + }); + + await writeFile( + path.posix.join(directories.requestsDir, "req-1.json"), + `${JSON.stringify({ + id: "req-1", + method: "DELETE", + path: "/api/secrets", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + + await worker.stop({ drainTimeoutMs: 1_000 }); + + const response = JSON.parse( + await readFile(path.posix.join(directories.responsesDir, "req-1.json"), "utf8"), + ) as { status: number; body: string }; + expect(handled).toBe(0); + expect(response.status).toBe(403); + expect(JSON.parse(response.body)).toEqual({ + error: "Route not allowed: DELETE /api/secrets", + }); + }); + + it("drains already-queued requests on stop", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-drain-")); + cleanupDirs.push(rootDir); + + const queueDir = path.posix.join(rootDir, "queue"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const processed: string[] = []; + + const worker = await startSandboxCallbackBridgeWorker({ + client: createFileSystemSandboxCallbackBridgeQueueClient(), + queueDir, + authorizeRequest: async () => null, + handleRequest: async (request) => { + processed.push(request.id); + await new Promise((resolve) => setTimeout(resolve, 25)); + return { + status: 200, + body: request.id, + }; + }, + }); + + await writeFile( + path.posix.join(directories.requestsDir, "req-a.json"), + `${JSON.stringify({ + id: "req-a", + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + await writeFile( + path.posix.join(directories.requestsDir, "req-b.json"), + `${JSON.stringify({ + id: "req-b", + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + + await worker.stop({ drainTimeoutMs: 1_000 }); + + expect(processed).toEqual(["req-a", "req-b"]); + await expect(readFile(path.posix.join(directories.responsesDir, "req-a.json"), "utf8")).resolves.toContain("\"req-a\""); + await expect(readFile(path.posix.join(directories.responsesDir, "req-b.json"), "utf8")).resolves.toContain("\"req-b\""); + }); + + it("writes fast 503 responses for queued requests that miss the drain deadline", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-drain-timeout-")); + cleanupDirs.push(rootDir); + + const queueDir = path.posix.join(rootDir, "queue"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const processed: string[] = []; + + const worker = await startSandboxCallbackBridgeWorker({ + client: createFileSystemSandboxCallbackBridgeQueueClient(), + queueDir, + authorizeRequest: async () => null, + handleRequest: async (request) => { + processed.push(request.id); + await new Promise((resolve) => setTimeout(resolve, 100)); + return { + status: 200, + body: request.id, + }; + }, + }); + + await writeFile( + path.posix.join(directories.requestsDir, "req-a.json"), + `${JSON.stringify({ + id: "req-a", + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + await writeFile( + path.posix.join(directories.requestsDir, "req-b.json"), + `${JSON.stringify({ + id: "req-b", + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + + for (let attempt = 0; attempt < 50 && processed.length === 0; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + + await worker.stop({ drainTimeoutMs: 10 }); + + expect(processed).toEqual(["req-a"]); + await expect(readFile(path.posix.join(directories.responsesDir, "req-a.json"), "utf8")).resolves.toContain("\"req-a\""); + await expect(readFile(path.posix.join(directories.responsesDir, "req-b.json"), "utf8")).resolves.toContain( + "Bridge worker stopped before request could be handled.", + ); + }); + + it("rejects non-JSON request bodies and full queues at the bridge server", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-server-guards-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge guard test\n", "utf8"); + + const runner = createExecRunner(); + + const bridgeAsset = await createSandboxCallbackBridgeAsset(); + cleanupFns.push(bridgeAsset.cleanup); + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "codex", + workspaceLocalDir: localWorkspaceDir, + assets: [{ key: "bridge", localDir: bridgeAsset.localDir }], + }); + + const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const bridgeToken = createSandboxCallbackBridgeToken(); + + const bridge = await startSandboxCallbackBridgeServer({ + runner, + remoteCwd: remoteWorkspaceDir, + assetRemoteDir: prepared.assetDirs.bridge, + queueDir, + bridgeToken, + timeoutMs: 30_000, + maxQueueDepth: 1, + }); + cleanupFns.push(async () => { + await bridge.stop(); + }); + + await writeFile( + path.posix.join(directories.requestsDir, "existing.json"), + `${JSON.stringify({ + id: "existing", + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + createdAt: new Date().toISOString(), + })}\n`, + "utf8", + ); + + const queueFullResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, { + headers: { + authorization: `Bearer ${bridgeToken}`, + }, + }); + expect(queueFullResponse.status).toBe(503); + await expect(queueFullResponse.json()).resolves.toEqual({ + error: "Bridge request queue is full.", + }); + + await rm(path.posix.join(directories.requestsDir, "existing.json"), { force: true }); + + const nonJsonResponse = await fetch(`${bridge.baseUrl}/api/issues/issue-1/comments`, { + method: "POST", + headers: { + authorization: `Bearer ${bridgeToken}`, + "content-type": "text/plain", + }, + body: "not json", + }); + expect(nonJsonResponse.status).toBe(415); + await expect(nonJsonResponse.json()).resolves.toEqual({ + error: "Bridge only accepts JSON request bodies.", + }); + }); + + it("returns a 502 when the host response times out", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-timeout-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge timeout test\n", "utf8"); + + const runner = createExecRunner(); + const bridgeAsset = await createSandboxCallbackBridgeAsset(); + cleanupFns.push(bridgeAsset.cleanup); + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "codex", + workspaceLocalDir: localWorkspaceDir, + assets: [{ key: "bridge", localDir: bridgeAsset.localDir }], + }); + + const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge"); + const bridgeToken = createSandboxCallbackBridgeToken(); + const bridge = await startSandboxCallbackBridgeServer({ + runner, + remoteCwd: remoteWorkspaceDir, + assetRemoteDir: prepared.assetDirs.bridge, + queueDir, + bridgeToken, + timeoutMs: 30_000, + pollIntervalMs: 10, + responseTimeoutMs: 75, + }); + cleanupFns.push(async () => { + await bridge.stop(); + }); + + const response = await fetch(`${bridge.baseUrl}/api/agents/me`, { + headers: { + authorization: `Bearer ${bridgeToken}`, + }, + }); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ + error: "Timed out waiting for host bridge response.", + }); + }); + + it("returns a 502 for malformed host response files", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-malformed-response-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge malformed response test\n", "utf8"); + + const runner = createExecRunner(); + const bridgeAsset = await createSandboxCallbackBridgeAsset(); + cleanupFns.push(bridgeAsset.cleanup); + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "codex", + workspaceLocalDir: localWorkspaceDir, + assets: [{ key: "bridge", localDir: bridgeAsset.localDir }], + }); + + const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const bridgeToken = createSandboxCallbackBridgeToken(); + const bridge = await startSandboxCallbackBridgeServer({ + runner, + remoteCwd: remoteWorkspaceDir, + assetRemoteDir: prepared.assetDirs.bridge, + queueDir, + bridgeToken, + timeoutMs: 30_000, + pollIntervalMs: 10, + responseTimeoutMs: 1_000, + }); + cleanupFns.push(async () => { + await bridge.stop(); + }); + + const responsePromise = fetch(`${bridge.baseUrl}/api/agents/me`, { + headers: { + authorization: `Bearer ${bridgeToken}`, + }, + }); + + const requestFile = await waitForJsonFile(directories.requestsDir); + await writeFile( + path.posix.join(directories.responsesDir, requestFile), + '{"status":200,"headers":{"content-type":"application/json"},"body"', + "utf8", + ); + + const response = await responsePromise; + expect(response.status).toBe(502); + await expect(response.json()).resolves.toMatchObject({ + error: expect.stringMatching(/JSON|Unexpected|Unterminated/i), + }); + }); +}); diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts new file mode 100644 index 00000000..013a3bbb --- /dev/null +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -0,0 +1,822 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js"; +import type { RunProcessResult } from "./server-utils.js"; + +const DEFAULT_BRIDGE_TOKEN_BYTES = 24; +const DEFAULT_BRIDGE_POLL_INTERVAL_MS = 100; +const DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS = 30_000; +const DEFAULT_BRIDGE_STOP_TIMEOUT_MS = 2_000; +const DEFAULT_BRIDGE_MAX_QUEUE_DEPTH = 64; +const DEFAULT_BRIDGE_MAX_BODY_BYTES = 256 * 1024; +const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024; +const SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT = "paperclip-bridge-server.mjs"; + +export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES = DEFAULT_BRIDGE_MAX_BODY_BYTES; + +export interface SandboxCallbackBridgeRouteRule { + method: string; + path: RegExp; +} + +export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST: readonly SandboxCallbackBridgeRouteRule[] = [ + { method: "GET", path: /^\/api\/agents\/me$/ }, + { method: "GET", path: /^\/api\/issues\/[^/]+\/heartbeat-context$/ }, + { method: "GET", path: /^\/api\/issues\/[^/]+\/comments(?:\/[^/]+)?$/ }, + { method: "GET", path: /^\/api\/issues\/[^/]+\/documents(?:\/[^/]+)?$/ }, + { method: "POST", path: /^\/api\/issues\/[^/]+\/checkout$/ }, + { method: "POST", path: /^\/api\/issues\/[^/]+\/comments$/ }, + { method: "POST", path: /^\/api\/issues\/[^/]+\/interactions(?:\/[^/]+)?$/ }, + { method: "PATCH", path: /^\/api\/issues\/[^/]+$/ }, +] as const; + +export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST = [ + "accept", + "content-type", + "if-match", + "if-none-match", +] as const; + +export interface SandboxCallbackBridgeRequest { + id: string; + method: string; + path: string; + query: string; + headers: Record; + /** + * UTF-8 body contents. The bridge rejects non-JSON request bodies; binary + * payloads are intentionally out of scope for this queue protocol. + */ + body: string; + createdAt: string; +} + +export interface SandboxCallbackBridgeResponse { + id: string; + status: number; + headers: Record; + body: string; + completedAt: string; +} + +export interface SandboxCallbackBridgeAsset { + localDir: string; + entrypoint: string; + cleanup(): Promise; +} + +export interface SandboxCallbackBridgeDirectories { + rootDir: string; + requestsDir: string; + responsesDir: string; + logsDir: string; + readyFile: string; + pidFile: string; + logFile: string; +} + +export interface SandboxCallbackBridgeQueueClient { + makeDir(remotePath: string): Promise; + listJsonFiles(remotePath: string): Promise; + readTextFile(remotePath: string): Promise; + writeTextFile(remotePath: string, body: string): Promise; + rename(fromPath: string, toPath: string): Promise; + remove(remotePath: string): Promise; +} + +export interface SandboxCallbackBridgeWorkerHandle { + stop(options?: { drainTimeoutMs?: number }): Promise; +} + +export interface StartedSandboxCallbackBridgeServer { + baseUrl: string; + host: string; + port: number; + pid: number; + directories: SandboxCallbackBridgeDirectories; + stop(): Promise; +} + +function shellQuote(value: string) { + return `'${value.replace(/'/g, `'"'"'`)}'`; +} + +function normalizeMethod(value: string | null | undefined): string { + return typeof value === "string" && value.trim().length > 0 ? value.trim().toUpperCase() : "GET"; +} + +function normalizeTimeoutMs(value: number | null | undefined, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : fallback; +} + +function toBuffer(bytes: Buffer | Uint8Array | ArrayBuffer): Buffer { + if (Buffer.isBuffer(bytes)) return bytes; + if (bytes instanceof ArrayBuffer) return Buffer.from(bytes); + return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); +} + +function buildRunnerFailureMessage(action: string, result: RunProcessResult): string { + const stderr = result.stderr.trim(); + const stdout = result.stdout.trim(); + const detail = stderr || stdout; + if (result.timedOut) { + return `${action} timed out${detail ? `: ${detail}` : ""}`; + } + return `${action} failed with exit code ${result.exitCode ?? "null"}${detail ? `: ${detail}` : ""}`; +} + +async function runShell( + runner: CommandManagedRuntimeRunner, + cwd: string, + script: string, + timeoutMs: number, +): Promise { + return await runner.execute({ + command: "sh", + args: ["-lc", script], + cwd, + timeoutMs, + }); +} + +function requireSuccessfulResult(action: string, result: RunProcessResult): RunProcessResult { + if (!result.timedOut && result.exitCode === 0) return result; + throw new Error(buildRunnerFailureMessage(action, result)); +} + +function base64Chunks(body: string): string[] { + const out: string[] = []; + for (let offset = 0; offset < body.length; offset += REMOTE_WRITE_BASE64_CHUNK_SIZE) { + out.push(body.slice(offset, offset + REMOTE_WRITE_BASE64_CHUNK_SIZE)); + } + return out; +} + +export function createSandboxCallbackBridgeToken(bytes = DEFAULT_BRIDGE_TOKEN_BYTES): string { + return randomBytes(bytes).toString("base64url"); +} + +export function authorizeSandboxCallbackBridgeRequestWithRoutes( + request: Pick, + routes: readonly SandboxCallbackBridgeRouteRule[] = DEFAULT_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST, +): string | null { + const method = normalizeMethod(request.method); + return routes.some((route) => route.method === method && route.path.test(request.path)) + ? null + : `Route not allowed: ${method} ${request.path}`; +} + +export function sanitizeSandboxCallbackBridgeHeaders( + headers: Record, + allowlist: readonly string[] = DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST, +): Record { + const allowed = new Set(allowlist.map((header) => header.toLowerCase())); + return Object.fromEntries( + Object.entries(headers).filter(([key]) => allowed.has(key.toLowerCase())), + ); +} + +export function sandboxCallbackBridgeDirectories(rootDir: string): SandboxCallbackBridgeDirectories { + return { + rootDir, + requestsDir: path.posix.join(rootDir, "requests"), + responsesDir: path.posix.join(rootDir, "responses"), + logsDir: path.posix.join(rootDir, "logs"), + readyFile: path.posix.join(rootDir, "ready.json"), + pidFile: path.posix.join(rootDir, "server.pid"), + logFile: path.posix.join(rootDir, "logs", "bridge.log"), + }; +} + +export function buildSandboxCallbackBridgeEnv(input: { + queueDir: string; + bridgeToken: string; + host?: string; + port?: number | null; + pollIntervalMs?: number | null; + responseTimeoutMs?: number | null; + maxQueueDepth?: number | null; + maxBodyBytes?: number | null; +}): Record { + return { + PAPERCLIP_API_BRIDGE_MODE: "queue_v1", + PAPERCLIP_BRIDGE_QUEUE_DIR: input.queueDir, + PAPERCLIP_BRIDGE_TOKEN: input.bridgeToken, + PAPERCLIP_BRIDGE_HOST: input.host?.trim() || "127.0.0.1", + PAPERCLIP_BRIDGE_PORT: String(input.port && input.port > 0 ? Math.trunc(input.port) : 0), + PAPERCLIP_BRIDGE_POLL_INTERVAL_MS: String( + normalizeTimeoutMs(input.pollIntervalMs, DEFAULT_BRIDGE_POLL_INTERVAL_MS), + ), + PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: String( + normalizeTimeoutMs(input.responseTimeoutMs, DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS), + ), + PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH: String( + normalizeTimeoutMs(input.maxQueueDepth, DEFAULT_BRIDGE_MAX_QUEUE_DEPTH), + ), + PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String( + normalizeTimeoutMs(input.maxBodyBytes, DEFAULT_BRIDGE_MAX_BODY_BYTES), + ), + }; +} + +export async function createSandboxCallbackBridgeAsset(): Promise { + const localDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-asset-")); + const entrypoint = path.join(localDir, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT); + await fs.writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); + return { + localDir, + entrypoint, + cleanup: async () => { + await fs.rm(localDir, { recursive: true, force: true }).catch(() => undefined); + }, + }; +} + +export function createFileSystemSandboxCallbackBridgeQueueClient(): SandboxCallbackBridgeQueueClient { + return { + makeDir: async (remotePath) => { + await fs.mkdir(remotePath, { recursive: true }); + }, + listJsonFiles: async (remotePath) => { + const entries = await fs.readdir(remotePath, { withFileTypes: true }).catch(() => []); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => entry.name) + .sort((left, right) => left.localeCompare(right)); + }, + readTextFile: async (remotePath) => await fs.readFile(remotePath, "utf8"), + writeTextFile: async (remotePath, body) => { + await fs.mkdir(path.posix.dirname(remotePath), { recursive: true }); + await fs.writeFile(remotePath, body, "utf8"); + }, + rename: async (fromPath, toPath) => { + await fs.mkdir(path.posix.dirname(toPath), { recursive: true }); + await fs.rename(fromPath, toPath); + }, + remove: async (remotePath) => { + await fs.rm(remotePath, { recursive: true, force: true }).catch(() => undefined); + }, + }; +} + +export function createCommandManagedSandboxCallbackBridgeQueueClient(input: { + runner: CommandManagedRuntimeRunner; + remoteCwd: string; + timeoutMs?: number | null; +}): SandboxCallbackBridgeQueueClient { + const timeoutMs = normalizeTimeoutMs(input.timeoutMs, DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS); + const runChecked = async (action: string, script: string) => + requireSuccessfulResult(action, await runShell(input.runner, input.remoteCwd, script, timeoutMs)); + + return { + makeDir: async (remotePath) => { + await runChecked(`mkdir ${remotePath}`, `mkdir -p ${shellQuote(remotePath)}`); + }, + listJsonFiles: async (remotePath) => { + const result = await runShell( + input.runner, + input.remoteCwd, + [ + `if [ -d ${shellQuote(remotePath)} ]; then`, + ` for file in ${shellQuote(remotePath)}/*.json; do`, + ` [ -f "$file" ] || continue`, + " basename \"$file\"", + " done", + "fi", + ].join("\n"), + timeoutMs, + ); + requireSuccessfulResult(`list ${remotePath}`, result); + return result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .sort((left, right) => left.localeCompare(right)); + }, + readTextFile: async (remotePath) => { + const result = await runChecked(`read ${remotePath}`, `base64 < ${shellQuote(remotePath)}`); + return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"); + }, + writeTextFile: async (remotePath, body) => { + const remoteDir = path.posix.dirname(remotePath); + const tempPath = `${remotePath}.paperclip-upload.b64`; + await runChecked( + `prepare upload ${remotePath}`, + `mkdir -p ${shellQuote(remoteDir)} && rm -f ${shellQuote(tempPath)} && : > ${shellQuote(tempPath)}`, + ); + const base64Body = toBuffer(Buffer.from(body, "utf8")).toString("base64"); + for (const chunk of base64Chunks(base64Body)) { + await runChecked( + `append upload chunk ${remotePath}`, + `printf '%s' ${shellQuote(chunk)} >> ${shellQuote(tempPath)}`, + ); + } + await runChecked( + `finalize upload ${remotePath}`, + `base64 -d < ${shellQuote(tempPath)} > ${shellQuote(remotePath)} && rm -f ${shellQuote(tempPath)}`, + ); + }, + rename: async (fromPath, toPath) => { + await runChecked( + `rename ${fromPath}`, + `mkdir -p ${shellQuote(path.posix.dirname(toPath))} && mv ${shellQuote(fromPath)} ${shellQuote(toPath)}`, + ); + }, + remove: async (remotePath) => { + await runChecked(`remove ${remotePath}`, `rm -rf ${shellQuote(remotePath)}`); + }, + }; +} + +async function writeBridgeResponse( + client: SandboxCallbackBridgeQueueClient, + responsePath: string, + response: SandboxCallbackBridgeResponse, +) { + const tempPath = `${responsePath}.tmp`; + await client.writeTextFile(tempPath, `${JSON.stringify(response)}\n`); + await client.rename(tempPath, responsePath); +} + +export async function startSandboxCallbackBridgeWorker(input: { + client: SandboxCallbackBridgeQueueClient; + queueDir: string; + pollIntervalMs?: number | null; + authorizeRequest?: (request: SandboxCallbackBridgeRequest) => string | null | Promise; + handleRequest: (request: SandboxCallbackBridgeRequest) => Promise<{ + status: number; + headers?: Record; + body?: string; + }>; + maxBodyBytes?: number | null; +}): Promise { + const pollIntervalMs = normalizeTimeoutMs(input.pollIntervalMs, DEFAULT_BRIDGE_POLL_INTERVAL_MS); + const maxBodyBytes = normalizeTimeoutMs(input.maxBodyBytes, DEFAULT_BRIDGE_MAX_BODY_BYTES); + const directories = sandboxCallbackBridgeDirectories(input.queueDir); + await input.client.makeDir(directories.rootDir); + await input.client.makeDir(directories.requestsDir); + await input.client.makeDir(directories.responsesDir); + await input.client.makeDir(directories.logsDir); + + let stopping = false; + let inFlight = 0; + let settled = false; + let stopDeadline = Number.POSITIVE_INFINITY; + let settleResolve: (() => void) | null = null; + const settledPromise = new Promise((resolve) => { + settleResolve = resolve; + }); + const authorizeRequest = input.authorizeRequest ?? + ((request: SandboxCallbackBridgeRequest) => authorizeSandboxCallbackBridgeRequestWithRoutes(request)); + + const processRequestFile = async (fileName: string) => { + const requestPath = path.posix.join(directories.requestsDir, fileName); + const responsePath = path.posix.join(directories.responsesDir, fileName); + const raw = await input.client.readTextFile(requestPath); + let request: SandboxCallbackBridgeRequest; + try { + request = JSON.parse(raw) as SandboxCallbackBridgeRequest; + } catch { + const requestId = fileName.replace(/\.json$/i, "") || randomUUID(); + await writeBridgeResponse(input.client, responsePath, { + id: requestId, + status: 400, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: "Invalid bridge request payload." }), + completedAt: new Date().toISOString(), + }); + await input.client.remove(requestPath); + return; + } + + const denialReason = await authorizeRequest(request); + if (denialReason) { + await writeBridgeResponse(input.client, responsePath, { + id: request.id, + status: 403, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: denialReason }), + completedAt: new Date().toISOString(), + }); + await input.client.remove(requestPath); + return; + } + + try { + const result = await input.handleRequest(request); + const responseBody = result.body ?? ""; + if (Buffer.byteLength(responseBody, "utf8") > maxBodyBytes) { + throw new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`); + } + await writeBridgeResponse(input.client, responsePath, { + id: request.id, + status: result.status, + headers: result.headers ?? {}, + body: responseBody, + completedAt: new Date().toISOString(), + }); + } catch (error) { + console.warn( + `[paperclip] sandbox callback bridge handler failed for ${request.id}: ${error instanceof Error ? error.message : String(error)}`, + ); + await writeBridgeResponse(input.client, responsePath, { + id: request.id, + status: 502, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + error: error instanceof Error ? error.message : String(error), + }), + completedAt: new Date().toISOString(), + }); + } finally { + await input.client.remove(requestPath); + } + }; + + const failPendingRequests = async (message: string) => { + const fileNames = await input.client.listJsonFiles(directories.requestsDir).catch(() => []); + for (const fileName of fileNames) { + const requestPath = path.posix.join(directories.requestsDir, fileName); + const responsePath = path.posix.join(directories.responsesDir, fileName); + const requestId = fileName.replace(/\.json$/i, "") || randomUUID(); + try { + const raw = await input.client.readTextFile(requestPath); + const parsed = JSON.parse(raw) as Partial; + await writeBridgeResponse(input.client, responsePath, { + id: typeof parsed.id === "string" && parsed.id.length > 0 ? parsed.id : requestId, + status: 503, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: message }), + completedAt: new Date().toISOString(), + }); + } catch (error) { + console.warn( + `[paperclip] sandbox callback bridge failed to abort pending request ${requestId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + await input.client.remove(requestPath).catch(() => undefined); + } + } + }; + + const loop = (async () => { + try { + while (true) { + const fileNames = await input.client.listJsonFiles(directories.requestsDir); + if (fileNames.length === 0) { + if (stopping) { + break; + } + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + continue; + } + for (const fileName of fileNames) { + if (stopping && Date.now() >= stopDeadline) break; + inFlight += 1; + try { + await processRequestFile(fileName); + } finally { + inFlight -= 1; + } + } + if (stopping && Date.now() >= stopDeadline) { + break; + } + } + } finally { + settled = true; + if (settleResolve) { + settleResolve(); + } + } + })(); + + void loop; + + return { + stop: async (options = {}) => { + stopping = true; + const drainMs = normalizeTimeoutMs(options.drainTimeoutMs, DEFAULT_BRIDGE_STOP_TIMEOUT_MS); + stopDeadline = Date.now() + drainMs; + if (!settled) { + await Promise.race([ + settledPromise, + new Promise((resolve) => setTimeout(resolve, drainMs)), + ]); + } + await failPendingRequests("Bridge worker stopped before request could be handled."); + }, + }; +} + +export async function startSandboxCallbackBridgeServer(input: { + runner: CommandManagedRuntimeRunner; + remoteCwd: string; + assetRemoteDir: string; + queueDir: string; + bridgeToken: string; + bridgeAsset?: SandboxCallbackBridgeAsset | null; + host?: string; + port?: number | null; + pollIntervalMs?: number | null; + responseTimeoutMs?: number | null; + timeoutMs?: number | null; + nodeCommand?: string; + maxQueueDepth?: number | null; + maxBodyBytes?: number | null; +}): Promise { + const timeoutMs = normalizeTimeoutMs(input.timeoutMs, DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS); + const directories = sandboxCallbackBridgeDirectories(input.queueDir); + const remoteEntrypoint = path.posix.join(input.assetRemoteDir, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT); + if (input.bridgeAsset) { + const assetClient = createCommandManagedSandboxCallbackBridgeQueueClient({ + runner: input.runner, + remoteCwd: input.remoteCwd, + timeoutMs, + }); + await assetClient.makeDir(input.assetRemoteDir); + const entrypointSource = await fs.readFile(input.bridgeAsset.entrypoint, "utf8"); + await assetClient.writeTextFile(remoteEntrypoint, entrypointSource); + } + const env = buildSandboxCallbackBridgeEnv({ + queueDir: input.queueDir, + bridgeToken: input.bridgeToken, + host: input.host, + port: input.port, + pollIntervalMs: input.pollIntervalMs, + responseTimeoutMs: input.responseTimeoutMs, + maxQueueDepth: input.maxQueueDepth, + maxBodyBytes: input.maxBodyBytes, + }); + const nodeCommand = input.nodeCommand?.trim() || "node"; + const startResult = await input.runner.execute({ + command: "sh", + args: [ + "-lc", + [ + `mkdir -p ${shellQuote(directories.requestsDir)} ${shellQuote(directories.responsesDir)} ${shellQuote(directories.logsDir)}`, + `rm -f ${shellQuote(directories.readyFile)} ${shellQuote(directories.pidFile)}`, + `nohup env ${Object.entries(env).map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ")} ` + + `${shellQuote(nodeCommand)} ${shellQuote(remoteEntrypoint)} ` + + `>> ${shellQuote(directories.logFile)} 2>&1 < /dev/null &`, + "pid=$!", + `printf '%s\\n' \"$pid\" > ${shellQuote(directories.pidFile)}`, + "printf '{\"pid\":%s}\\n' \"$pid\"", + ].join("\n"), + ], + cwd: input.remoteCwd, + timeoutMs, + }); + requireSuccessfulResult("start sandbox callback bridge", startResult); + + const readyResult = await runShell( + input.runner, + input.remoteCwd, + [ + "i=0", + `while [ \"$i\" -lt 200 ]; do`, + ` if [ -s ${shellQuote(directories.readyFile)} ]; then`, + ` cat ${shellQuote(directories.readyFile)}`, + " exit 0", + " fi", + ` if [ -s ${shellQuote(directories.logFile)} ] && ! kill -0 \"$(cat ${shellQuote(directories.pidFile)} 2>/dev/null)\" 2>/dev/null; then`, + ` cat ${shellQuote(directories.logFile)} >&2`, + " exit 1", + " fi", + " i=$((i + 1))", + " sleep 0.05", + "done", + `echo "Timed out waiting for bridge readiness." >&2`, + `if [ -s ${shellQuote(directories.logFile)} ]; then cat ${shellQuote(directories.logFile)} >&2; fi`, + "exit 1", + ].join("\n"), + timeoutMs, + ); + requireSuccessfulResult("wait for sandbox callback bridge readiness", readyResult); + + let readyData: { host?: string; port?: number; baseUrl?: string; pid?: number }; + try { + readyData = JSON.parse(readyResult.stdout.trim()) as { host?: string; port?: number; baseUrl?: string; pid?: number }; + } catch (error) { + throw new Error( + `Sandbox callback bridge wrote invalid readiness JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + const host = typeof readyData.host === "string" && readyData.host.trim().length > 0 + ? readyData.host.trim() + : "127.0.0.1"; + const port = typeof readyData.port === "number" && Number.isFinite(readyData.port) ? readyData.port : 0; + if (!port) { + throw new Error("Sandbox callback bridge did not report a listening port."); + } + const baseUrl = + typeof readyData.baseUrl === "string" && readyData.baseUrl.trim().length > 0 + ? readyData.baseUrl.trim() + : `http://${host}:${port}`; + + return { + baseUrl, + host, + port, + pid: typeof readyData.pid === "number" && Number.isFinite(readyData.pid) ? readyData.pid : 0, + directories, + stop: async () => { + const stopResult = await input.runner.execute({ + command: "sh", + args: [ + "-lc", + [ + `if [ -s ${shellQuote(directories.pidFile)} ]; then`, + ` pid="$(cat ${shellQuote(directories.pidFile)})"`, + " kill \"$pid\" 2>/dev/null || true", + " i=0", + " while kill -0 \"$pid\" 2>/dev/null && [ \"$i\" -lt 40 ]; do", + " i=$((i + 1))", + " sleep 0.05", + " done", + "fi", + `rm -f ${shellQuote(directories.pidFile)} ${shellQuote(directories.readyFile)}`, + ].join("\n"), + ], + cwd: input.remoteCwd, + timeoutMs, + }); + if (stopResult.timedOut) { + throw new Error(buildRunnerFailureMessage("stop sandbox callback bridge", stopResult)); + } + }, + }; +} + +function getSandboxCallbackBridgeServerSource(): string { + return `import { randomUUID, timingSafeEqual } from "node:crypto"; +import { createServer } from "node:http"; +import { promises as fs } from "node:fs"; +import path from "node:path"; + +const queueDir = process.env.PAPERCLIP_BRIDGE_QUEUE_DIR; +const bridgeToken = process.env.PAPERCLIP_BRIDGE_TOKEN; +const host = process.env.PAPERCLIP_BRIDGE_HOST || "127.0.0.1"; +const port = Number(process.env.PAPERCLIP_BRIDGE_PORT || "0"); +const pollIntervalMs = Number(process.env.PAPERCLIP_BRIDGE_POLL_INTERVAL_MS || "100"); +const responseTimeoutMs = Number(process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || "30000"); +const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}"); +const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}"); +const allowedHeaders = new Set(${JSON.stringify([...DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST])}); + +if (!queueDir || !bridgeToken) { + throw new Error("PAPERCLIP_BRIDGE_QUEUE_DIR and PAPERCLIP_BRIDGE_TOKEN are required."); +} + +const requestsDir = path.posix.join(queueDir, "requests"); +const responsesDir = path.posix.join(queueDir, "responses"); +const logsDir = path.posix.join(queueDir, "logs"); +const readyFile = path.posix.join(queueDir, "ready.json"); + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function normalizeHeaders(headers) { + const out = {}; + for (const [key, value] of Object.entries(headers)) { + if (value == null) continue; + const normalizedKey = key.toLowerCase(); + if (!allowedHeaders.has(normalizedKey)) { + continue; + } + out[normalizedKey] = Array.isArray(value) ? value.join(", ") : String(value); + } + return out; +} + +async function readBody(req) { + const chunks = []; + let totalBytes = 0; + for await (const chunk of req) { + const nextChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + chunks.push(nextChunk); + totalBytes += nextChunk.byteLength; + if (totalBytes > maxBodyBytes) { + throw new Error("Bridge request body exceeded the configured size limit."); + } + } + return Buffer.concat(chunks).toString("utf8"); +} + +async function queueDepth() { + const entries = await fs.readdir(requestsDir, { withFileTypes: true }).catch(() => []); + return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json")).length; +} + +function tokensMatch(received) { + const expected = Buffer.from(bridgeToken, "utf8"); + const actual = Buffer.from(typeof received === "string" ? received : "", "utf8"); + if (expected.length !== actual.length) return false; + return timingSafeEqual(expected, actual); +} + +async function waitForResponse(requestId) { + const responsePath = path.posix.join(responsesDir, \`\${requestId}.json\`); + const deadline = Date.now() + responseTimeoutMs; + while (Date.now() < deadline) { + const body = await fs.readFile(responsePath, "utf8").catch(() => null); + if (body != null) { + await fs.rm(responsePath, { force: true }).catch(() => undefined); + return JSON.parse(body); + } + await sleep(pollIntervalMs); + } + throw new Error("Timed out waiting for host bridge response."); +} + +const server = createServer(async (req, res) => { + try { + const auth = req.headers.authorization || ""; + const receivedToken = auth.startsWith("Bearer ") ? auth.slice("Bearer ".length) : ""; + if (!tokensMatch(receivedToken)) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "Invalid bridge token." })); + return; + } + + if (await queueDepth() >= maxQueueDepth) { + res.statusCode = 503; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "Bridge request queue is full." })); + return; + } + + const url = new URL(req.url || "/", "http://127.0.0.1"); + const contentType = typeof req.headers["content-type"] === "string" ? req.headers["content-type"] : ""; + if (req.method && req.method !== "GET" && req.method !== "HEAD" && !/json/i.test(contentType)) { + res.statusCode = 415; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "Bridge only accepts JSON request bodies." })); + return; + } + const requestId = randomUUID(); + const requestBody = await readBody(req); + const payload = { + id: requestId, + method: req.method || "GET", + path: url.pathname, + query: url.search, + headers: normalizeHeaders(req.headers), + body: requestBody, + createdAt: new Date().toISOString(), + }; + const requestPath = path.posix.join(requestsDir, \`\${requestId}.json\`); + const tempPath = \`\${requestPath}.tmp\`; + await fs.writeFile(tempPath, \`\${JSON.stringify(payload)}\\n\`, "utf8"); + await fs.rename(tempPath, requestPath); + + const response = await waitForResponse(requestId); + res.statusCode = typeof response.status === "number" ? response.status : 200; + for (const [key, value] of Object.entries(response.headers || {})) { + if (typeof value !== "string" || key.toLowerCase() === "content-length") continue; + res.setHeader(key, value); + } + res.end(typeof response.body === "string" ? response.body : ""); + } catch (error) { + res.statusCode = 502; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) })); + } +}); + +async function shutdown() { + server.close(() => { + process.exit(0); + }); +} + +process.on("SIGINT", () => void shutdown()); +process.on("SIGTERM", () => void shutdown()); + +await fs.mkdir(requestsDir, { recursive: true }); +await fs.mkdir(responsesDir, { recursive: true }); +await fs.mkdir(logsDir, { recursive: true }); + +server.listen(port, host, async () => { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Bridge server did not expose a TCP address."); + } + const ready = { + pid: process.pid, + host, + port: address.port, + baseUrl: \`http://\${host}:\${address.port}\`, + startedAt: new Date().toISOString(), + }; + const tempReadyFile = \`\${readyFile}.tmp\`; + await fs.writeFile(tempReadyFile, JSON.stringify(ready), "utf8"); + await fs.rename(tempReadyFile, readyFile); +});`; +} diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index 5da695af..4f8532c1 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -1,4 +1,4 @@ -import { lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; @@ -73,6 +73,13 @@ describe("sandbox managed runtime", () => { await writeFile(remotePath, Buffer.from(bytes)); }, readFile: async (remotePath) => await readFile(remotePath), + listFiles: async (remotePath) => { + const entries = await readdir(remotePath, { withFileTypes: true }).catch(() => []); + return entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .sort((left, right) => left.localeCompare(right)); + }, remove: async (remotePath) => { await rm(remotePath, { recursive: true, force: true }); }, diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index e68c4363..518cc80a 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -27,6 +27,7 @@ export interface SandboxManagedRuntimeClient { makeDir(remotePath: string): Promise; writeFile(remotePath: string, bytes: ArrayBuffer): Promise; readFile(remotePath: string): Promise; + listFiles(remotePath: string): Promise; remove(remotePath: string): Promise; run(command: string, options: { timeoutMs: number }): Promise; } diff --git a/packages/adapters/claude-local/src/server/claude-config.test.ts b/packages/adapters/claude-local/src/server/claude-config.test.ts new file mode 100644 index 00000000..caeed9d6 --- /dev/null +++ b/packages/adapters/claude-local/src/server/claude-config.test.ts @@ -0,0 +1,66 @@ +import * as fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { prepareClaudeConfigSeed } from "./claude-config.js"; + +describe("prepareClaudeConfigSeed", () => { + const cleanupDirs: string[] = []; + + afterEach(async () => { + vi.restoreAllMocks(); + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (!dir) continue; + await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + function createEnv(root: string, sourceDir: string): NodeJS.ProcessEnv { + return { + HOME: root, + PAPERCLIP_HOME: path.join(root, "paperclip-home"), + PAPERCLIP_INSTANCE_ID: "test-instance", + CLAUDE_CONFIG_DIR: sourceDir, + }; + } + + it("reuses the same snapshot path when the seeded files are unchanged", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-config-seed-")); + cleanupDirs.push(root); + const sourceDir = path.join(root, "claude-source"); + await fs.mkdir(sourceDir, { recursive: true }); + await fs.writeFile(path.join(sourceDir, "settings.json"), JSON.stringify({ theme: "light" }), "utf8"); + + const onLog = vi.fn(async () => {}); + const env = createEnv(root, sourceDir); + + const first = await prepareClaudeConfigSeed(env, onLog, "company-1"); + const second = await prepareClaudeConfigSeed(env, onLog, "company-1"); + + expect(first).toBe(second); + await expect(fs.readFile(path.join(first, "settings.json"), "utf8")) + .resolves.toBe(JSON.stringify({ theme: "light" })); + }); + + it("keeps an existing snapshot intact when the seeded files change", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-config-race-")); + cleanupDirs.push(root); + const sourceDir = path.join(root, "claude-source"); + await fs.mkdir(sourceDir, { recursive: true }); + await fs.writeFile(path.join(sourceDir, "settings.json"), JSON.stringify({ theme: "light" }), "utf8"); + + const onLog = vi.fn(async () => {}); + const env = createEnv(root, sourceDir); + const first = await prepareClaudeConfigSeed(env, onLog, "company-1"); + + await fs.writeFile(path.join(sourceDir, "settings.json"), JSON.stringify({ theme: "dark" }), "utf8"); + const second = await prepareClaudeConfigSeed(env, onLog, "company-1"); + + expect(second).not.toBe(first); + await expect(fs.readFile(path.join(first, "settings.json"), "utf8")) + .resolves.toBe(JSON.stringify({ theme: "light" })); + await expect(fs.readFile(path.join(second, "settings.json"), "utf8")) + .resolves.toBe(JSON.stringify({ theme: "dark" })); + }); +}); diff --git a/packages/adapters/claude-local/src/server/claude-config.ts b/packages/adapters/claude-local/src/server/claude-config.ts new file mode 100644 index 00000000..5e4f2278 --- /dev/null +++ b/packages/adapters/claude-local/src/server/claude-config.ts @@ -0,0 +1,135 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; + +const DEFAULT_PAPERCLIP_INSTANCE_ID = "default"; +const SEEDED_SHARED_FILES = [ + ".credentials.json", + "credentials.json", + "settings.json", + "settings.local.json", + "CLAUDE.md", +] as const; + +function nonEmpty(value: string | undefined): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +async function pathExists(candidate: string): Promise { + return fs.access(candidate).then(() => true).catch(() => false); +} + +function isAlreadyExistsError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const code = "code" in error ? error.code : null; + return code === "EEXIST" || code === "ENOTEMPTY"; +} + +async function collectSeedFiles(sourceDir: string): Promise> { + const files: Array<{ name: string; sourcePath: string }> = []; + for (const name of SEEDED_SHARED_FILES) { + const sourcePath = path.join(sourceDir, name); + if (!(await pathExists(sourcePath))) continue; + files.push({ name, sourcePath }); + } + return files; +} + +async function buildSeedSnapshotKey(files: Array<{ name: string; sourcePath: string }>): Promise { + if (files.length === 0) return "empty"; + const hash = createHash("sha256"); + for (const file of files) { + hash.update(file.name); + hash.update("\0"); + hash.update(await fs.readFile(file.sourcePath)); + hash.update("\0"); + } + return hash.digest("hex").slice(0, 16); +} + +async function materializeSeedSnapshot(input: { + rootDir: string; + snapshotKey: string; + files: Array<{ name: string; sourcePath: string }>; +}): Promise { + const targetDir = path.join(input.rootDir, input.snapshotKey); + if (await pathExists(targetDir)) { + return targetDir; + } + + await fs.mkdir(input.rootDir, { recursive: true }); + const stagingDir = await fs.mkdtemp(path.join(input.rootDir, ".tmp-")); + try { + for (const file of input.files) { + await fs.copyFile(file.sourcePath, path.join(stagingDir, file.name)); + } + try { + await fs.rename(stagingDir, targetDir); + } catch (error) { + if (!isAlreadyExistsError(error)) { + throw error; + } + await fs.rm(stagingDir, { recursive: true, force: true }); + } + } catch (error) { + await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } + + return targetDir; +} + +export function resolveSharedClaudeConfigDir( + env: NodeJS.ProcessEnv = process.env, +): string { + const fromEnv = nonEmpty(env.CLAUDE_CONFIG_DIR); + return fromEnv ? path.resolve(fromEnv) : path.join(os.homedir(), ".claude"); +} + +export function resolveManagedClaudeConfigSeedDir( + env: NodeJS.ProcessEnv, + companyId?: string, +): string { + const paperclipHome = nonEmpty(env.PAPERCLIP_HOME) ?? path.resolve(os.homedir(), ".paperclip"); + const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? DEFAULT_PAPERCLIP_INSTANCE_ID; + return companyId + ? path.resolve(paperclipHome, "instances", instanceId, "companies", companyId, "claude-config-seed") + : path.resolve(paperclipHome, "instances", instanceId, "claude-config-seed"); +} + +export async function prepareClaudeConfigSeed( + env: NodeJS.ProcessEnv, + onLog: AdapterExecutionContext["onLog"], + companyId?: string, +): Promise { + const sourceDir = resolveSharedClaudeConfigDir(env); + const targetRootDir = resolveManagedClaudeConfigSeedDir(env, companyId); + + if (path.resolve(sourceDir) === path.resolve(targetRootDir)) { + return targetRootDir; + } + + const copiedFiles = await collectSeedFiles(sourceDir); + const snapshotKey = await buildSeedSnapshotKey(copiedFiles); + const targetDir = await materializeSeedSnapshot({ + rootDir: targetRootDir, + snapshotKey, + files: copiedFiles, + }); + + if (copiedFiles.length > 0) { + await onLog( + "stdout", + `[paperclip] Prepared Claude config seed "${targetDir}" from "${sourceDir}" (${copiedFiles.map((file) => file.name).join(", ")}).\n`, + ); + } else { + await onLog( + "stdout", + `[paperclip] No local Claude config seed files were found in "${sourceDir}". Remote Claude auth may still require login.\n`, + ); + } + + return targetDir; +} diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 365295b6..f047383a 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -10,12 +10,15 @@ import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetSessionMatches, adapterExecutionTargetUsesManagedHome, + adapterExecutionTargetUsesPaperclipBridge, describeAdapterExecutionTarget, ensureAdapterExecutionTargetCommandResolvable, prepareAdapterExecutionTargetRuntime, readAdapterExecutionTarget, resolveAdapterExecutionTargetCommandForLogs, runAdapterExecutionTargetProcess, + runAdapterExecutionTargetShellCommand, + startAdapterExecutionTargetPaperclipBridge, } from "@paperclipai/adapter-utils/execution-target"; import { asString, @@ -36,6 +39,7 @@ import { stringifyPaperclipWakePayload, DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE, } from "@paperclipai/adapter-utils/server-utils"; +import { shellQuote } from "@paperclipai/adapter-utils/ssh"; import { parseClaudeStreamJson, describeClaudeFailure, @@ -45,6 +49,7 @@ import { isClaudeTransientUpstreamError, isClaudeUnknownSessionError, } from "./parse.js"; +import { prepareClaudeConfigSeed } from "./claude-config.js"; import { resolveClaudeDesiredSkillNames } from "./skills.js"; import { isBedrockModelId } from "./models.js"; import { prepareClaudePromptBundle } from "./prompt-cache.js"; @@ -316,6 +321,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise 0; const instructionsFilePath = asString(config.instructionsFilePath, "").trim(); const instructionsFileDir = instructionsFilePath ? `${path.dirname(instructionsFilePath)}/` : ""; const runtimeConfig = await buildClaudeRuntimeConfig({ @@ -334,11 +342,12 @@ export async function execute(ctx: AdapterExecutionContext): Promise { await onLog( @@ -395,6 +411,13 @@ export async function execute(ctx: AdapterExecutionContext): Promise> = null; + if (executionTargetIsRemote && adapterExecutionTargetUsesPaperclipBridge(executionTarget)) { + paperclipBridge = await startAdapterExecutionTargetPaperclipBridge({ + runId, + target: executionTarget, + runtimeRootDir: preparedExecutionTargetRuntime?.runtimeRootDir, + adapterKey: "claude", + hostApiToken: env.PAPERCLIP_API_KEY, + onLog, + }); + if (paperclipBridge) { + Object.assign(env, paperclipBridge.env); + const runtimeEnv = ensurePathInEnv({ ...process.env, ...env }); + loggedEnv = buildInvocationEnvForLogs(env, { + runtimeEnv, + includeRuntimeKeys: ["HOME", "CLAUDE_CONFIG_DIR"], + resolvedCommand, + }); + if (remoteClaudeConfigDir) { + loggedEnv.CLAUDE_CONFIG_DIR = remoteClaudeConfigDir; + } + } + } const runtimeSessionParams = parseObject(runtime.sessionParams); const runtimeSessionId = asString(runtimeSessionParams.sessionId, runtime.sessionId ?? ""); @@ -766,6 +846,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise preparedExecutionTargetRuntime.restoreWorkspace() : null; + let paperclipBridge: Awaited> = null; const remoteCodexHome = executionTargetIsRemote ? preparedExecutionTargetRuntime?.assetDirs.home ?? path.posix.join(effectiveExecutionCwd, ".paperclip-runtime", "codex", "home") @@ -456,6 +459,19 @@ export async function execute(ctx: AdapterExecutionContext): Promise typeof entry[1] === "string", @@ -780,6 +796,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise Promise) | null = null; let localSkillsDir: string | null = null; + let remoteRuntimeRootDir: string | null = null; + let paperclipBridge: Awaited> = null; if (executionTargetIsRemote) { try { @@ -359,6 +363,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise preparedExecutionTargetRuntime.restoreWorkspace(); + remoteRuntimeRootDir = preparedExecutionTargetRuntime.runtimeRootDir; const managedHome = adapterExecutionTargetUsesManagedHome(executionTarget); if (managedHome && preparedExecutionTargetRuntime.runtimeRootDir) { env.HOME = preparedExecutionTargetRuntime.runtimeRootDir; @@ -389,6 +394,24 @@ export async function execute(ctx: AdapterExecutionContext): Promise Promise) | null = null; let remoteSkillsDir: string | null = null; let localSkillsDir: string | null = null; + let remoteRuntimeRootDir: string | null = null; + let paperclipBridge: Awaited> = null; if (executionTargetIsRemote) { try { @@ -304,6 +308,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise preparedExecutionTargetRuntime.restoreWorkspace(); + remoteRuntimeRootDir = preparedExecutionTargetRuntime.runtimeRootDir; const managedHome = adapterExecutionTargetUsesManagedHome(executionTarget); if (managedHome && preparedExecutionTargetRuntime.runtimeRootDir) { env.HOME = preparedExecutionTargetRuntime.runtimeRootDir; @@ -334,6 +339,24 @@ export async function execute(ctx: AdapterExecutionContext): Promise undefined) : Promise.resolve(), ]); diff --git a/packages/adapters/opencode-local/src/server/execute.ts b/packages/adapters/opencode-local/src/server/execute.ts index fbb8d433..82f62f1d 100644 --- a/packages/adapters/opencode-local/src/server/execute.ts +++ b/packages/adapters/opencode-local/src/server/execute.ts @@ -10,6 +10,7 @@ import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetSessionMatches, adapterExecutionTargetUsesManagedHome, + adapterExecutionTargetUsesPaperclipBridge, describeAdapterExecutionTarget, ensureAdapterExecutionTargetCommandResolvable, prepareAdapterExecutionTargetRuntime, @@ -18,6 +19,7 @@ import { resolveAdapterExecutionTargetCommandForLogs, runAdapterExecutionTargetProcess, runAdapterExecutionTargetShellCommand, + startAdapterExecutionTargetPaperclipBridge, } from "@paperclipai/adapter-utils/execution-target"; import { asString, @@ -234,7 +236,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise Promise) | null = null; let localSkillsDir: string | null = null; + let remoteRuntimeRootDir: string | null = null; + let paperclipBridge: Awaited> = null; if (executionTargetIsRemote) { localSkillsDir = await buildOpenCodeSkillsDir(config); @@ -285,6 +289,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise preparedExecutionTargetRuntime.restoreWorkspace(); + remoteRuntimeRootDir = preparedExecutionTargetRuntime.runtimeRootDir; const managedHome = adapterExecutionTargetUsesManagedHome(executionTarget); if (managedHome && preparedExecutionTargetRuntime.runtimeRootDir) { preparedRuntimeConfig.env.HOME = preparedExecutionTargetRuntime.runtimeRootDir; @@ -311,6 +316,28 @@ export async function execute(ctx: AdapterExecutionContext): Promise typeof entry[1] === "string", + ), + ), + includeRuntimeKeys: ["HOME"], + resolvedCommand, + }); + } + } const runtimeSessionParams = parseObject(runtime.sessionParams); const runtimeSessionId = asString(runtimeSessionParams.sessionId, runtime.sessionId ?? ""); @@ -538,6 +565,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise undefined) : Promise.resolve(), ]); diff --git a/packages/adapters/pi-local/src/server/execute.ts b/packages/adapters/pi-local/src/server/execute.ts index 00b011a6..ef7f654b 100644 --- a/packages/adapters/pi-local/src/server/execute.ts +++ b/packages/adapters/pi-local/src/server/execute.ts @@ -10,6 +10,7 @@ import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetSessionMatches, adapterExecutionTargetUsesManagedHome, + adapterExecutionTargetUsesPaperclipBridge, describeAdapterExecutionTarget, ensureAdapterExecutionTargetCommandResolvable, ensureAdapterExecutionTargetFile, @@ -17,6 +18,7 @@ import { readAdapterExecutionTarget, resolveAdapterExecutionTargetCommandForLogs, runAdapterExecutionTargetProcess, + startAdapterExecutionTargetPaperclipBridge, } from "@paperclipai/adapter-utils/execution-target"; import { asString, @@ -278,7 +280,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise> = null; if (executionTargetIsRemote) { try { @@ -338,6 +341,28 @@ export async function execute(ctx: AdapterExecutionContext): Promise typeof entry[1] === "string", + ), + ), + includeRuntimeKeys: ["HOME"], + resolvedCommand, + }); + } + } const runtimeSessionParams = parseObject(runtime.sessionParams); const runtimeSessionId = asString(runtimeSessionParams.sessionId, runtime.sessionId ?? ""); @@ -654,6 +679,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise undefined) : Promise.resolve(), ]); diff --git a/packages/plugins/sandbox-providers/e2b/src/e2b.d.ts b/packages/plugins/sandbox-providers/e2b/src/e2b.d.ts index 2536d896..1161d6a6 100644 --- a/packages/plugins/sandbox-providers/e2b/src/e2b.d.ts +++ b/packages/plugins/sandbox-providers/e2b/src/e2b.d.ts @@ -35,6 +35,10 @@ declare module "e2b" { setTimeout(timeoutMs: number): Promise; kill(): Promise; pause(): Promise; + files: { + write(path: string, data: string | ArrayBuffer): Promise; + remove(path: string): Promise; + }; commands: { run( command: string, diff --git a/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts b/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts index b8766eaf..3cb1764b 100644 --- a/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/e2b/src/plugin.test.ts @@ -16,7 +16,18 @@ const { MockCommandExitError, MockSandboxNotFoundError, MockTimeoutError } = vi. } } class MockSandboxNotFoundError extends Error {} - class MockTimeoutError extends Error {} + class MockTimeoutError extends Error { + stdout: string; + stderr: string; + result?: { stdout?: string; stderr?: string }; + + constructor(message: string, streams: { stdout?: string; stderr?: string; nested?: boolean } = {}) { + super(message); + this.stdout = streams.nested ? "" : (streams.stdout ?? ""); + this.stderr = streams.nested ? "" : (streams.stderr ?? ""); + this.result = streams.nested ? { stdout: streams.stdout, stderr: streams.stderr } : undefined; + } + } return { MockCommandExitError, MockSandboxNotFoundError, MockTimeoutError }; }); @@ -54,6 +65,10 @@ function createMockSandbox(overrides: { setTimeout: vi.fn().mockResolvedValue(undefined), kill: vi.fn().mockResolvedValue(undefined), pause: vi.fn().mockResolvedValue(undefined), + files: { + write: vi.fn().mockResolvedValue(undefined), + remove: vi.fn().mockResolvedValue(undefined), + }, commands: { run: vi.fn(async (command: string, options?: { background?: boolean }) => { if (options?.background) return handle; @@ -228,8 +243,23 @@ describe("E2B sandbox provider plugin", () => { expect(sandbox.kill).toHaveBeenCalled(); }); - it("executes commands through a connected sandbox", async () => { + it("executes commands through a connected sandbox when stdin is provided", async () => { const sandbox = createMockSandbox(); + sandbox.commands.run.mockImplementation(async (command: string, options?: { background?: boolean }) => { + if (options?.background) return sandbox.handle; + if (command === "pwd") { + return { + exitCode: 0, + stdout: "/home/user\n", + stderr: "", + }; + } + return { + exitCode: 0, + stdout: "stdin\n", + stderr: "", + }; + }); mockConnect.mockResolvedValue(sandbox); const result = await plugin.definition.onEnvironmentExecute?.({ @@ -252,27 +282,91 @@ describe("E2B sandbox provider plugin", () => { }); expect(mockConnect).toHaveBeenCalledWith("sandbox-123", expect.objectContaining({ apiKey: "resolved-key" })); - expect(sandbox.commands.run).toHaveBeenCalledWith("exec 'printf' 'hello'", expect.objectContaining({ - background: true, + expect(sandbox.files.write).toHaveBeenCalledWith(expect.stringMatching(/^\/tmp\/paperclip-stdin-/), "input"); + expect(sandbox.commands.run).toHaveBeenCalledWith(expect.stringMatching( + /^exec 'printf' 'hello' < '\/tmp\/paperclip-stdin-/, + ), expect.objectContaining({ cwd: "/workspace", envs: { FOO: "bar" }, - stdin: true, timeoutMs: 1000, })); - expect(sandbox.commands.sendStdin).toHaveBeenCalledWith(42, "input"); - expect(sandbox.commands.closeStdin).toHaveBeenCalledWith(42); + expect(sandbox.commands.run).not.toHaveBeenCalledWith( + "exec 'printf' 'hello'", + expect.objectContaining({ background: true }), + ); + expect(sandbox.commands.sendStdin).not.toHaveBeenCalled(); + expect(sandbox.commands.closeStdin).not.toHaveBeenCalled(); + expect(sandbox.handle.wait).not.toHaveBeenCalled(); + expect(sandbox.files.remove).toHaveBeenCalledWith(expect.stringMatching(/^\/tmp\/paperclip-stdin-/)); expect(result).toEqual({ exitCode: 0, timedOut: false, - stdout: "ok\n", + stdout: "stdin\n", stderr: "", }); }); - it("closes stdin even when sendStdin throws unexpectedly", async () => { + it("executes non-stdin commands in foreground mode", async () => { const sandbox = createMockSandbox(); - const failure = new Error("send failed"); - sandbox.commands.sendStdin.mockRejectedValueOnce(failure); + sandbox.commands.run.mockImplementation(async (command: string, options?: { background?: boolean }) => { + if (options?.background) return sandbox.handle; + if (command === "pwd") { + return { + exitCode: 0, + stdout: "/home/user\n", + stderr: "", + }; + } + return { + exitCode: 0, + stdout: "foreground\n", + stderr: "", + }; + }); + mockConnect.mockResolvedValue(sandbox); + + const result = await plugin.definition.onEnvironmentExecute?.({ + driverKey: "e2b", + companyId: "company-1", + environmentId: "env-1", + config: { + template: "base", + apiKey: "resolved-key", + timeoutMs: 300000, + reuseLease: false, + }, + lease: { providerLeaseId: "sandbox-123", metadata: {} }, + command: "printf", + args: ["hello"], + cwd: "/workspace", + env: { FOO: "bar" }, + timeoutMs: 1000, + }); + + expect(sandbox.commands.run).toHaveBeenCalledWith("exec 'printf' 'hello'", expect.objectContaining({ + cwd: "/workspace", + envs: { FOO: "bar" }, + timeoutMs: 1000, + })); + expect(sandbox.commands.run).not.toHaveBeenCalledWith( + "exec 'printf' 'hello'", + expect.objectContaining({ background: true }), + ); + expect(sandbox.commands.sendStdin).not.toHaveBeenCalled(); + expect(sandbox.commands.closeStdin).not.toHaveBeenCalled(); + expect(sandbox.handle.wait).not.toHaveBeenCalled(); + expect(result).toEqual({ + exitCode: 0, + timedOut: false, + stdout: "foreground\n", + stderr: "", + }); + }); + + it("cleans up staged stdin even when writing it fails", async () => { + const sandbox = createMockSandbox(); + const failure = new Error("write failed"); + sandbox.files.write.mockRejectedValueOnce(failure); mockConnect.mockResolvedValue(sandbox); await expect(plugin.definition.onEnvironmentExecute?.({ @@ -292,12 +386,103 @@ describe("E2B sandbox provider plugin", () => { env: { FOO: "bar" }, stdin: "input", timeoutMs: 1000, - })).rejects.toThrow("send failed"); + })).rejects.toThrow("write failed"); - expect(sandbox.commands.closeStdin).toHaveBeenCalledWith(42); + expect(sandbox.files.remove).toHaveBeenCalledWith(expect.stringMatching(/^\/tmp\/paperclip-stdin-/)); + expect(sandbox.commands.sendStdin).not.toHaveBeenCalled(); expect(sandbox.handle.wait).not.toHaveBeenCalled(); }); + it("preserves partial foreground output when a non-stdin command times out", async () => { + const sandbox = createMockSandbox(); + sandbox.commands.run.mockImplementation(async (command: string, options?: { background?: boolean }) => { + if (options?.background) return sandbox.handle; + if (command === "pwd") { + return { + exitCode: 0, + stdout: "/home/user\n", + stderr: "", + }; + } + throw new MockTimeoutError("command timed out", { + stdout: "partial stdout\n", + stderr: "partial stderr\n", + }); + }); + mockConnect.mockResolvedValue(sandbox); + + const result = await plugin.definition.onEnvironmentExecute?.({ + driverKey: "e2b", + companyId: "company-1", + environmentId: "env-1", + config: { + template: "base", + apiKey: "resolved-key", + timeoutMs: 300000, + reuseLease: false, + }, + lease: { providerLeaseId: "sandbox-123", metadata: {} }, + command: "printf", + args: ["hello"], + cwd: "/workspace", + env: { FOO: "bar" }, + timeoutMs: 1000, + }); + + expect(result).toEqual({ + exitCode: null, + timedOut: true, + stdout: "partial stdout\n", + stderr: "partial stderr\ncommand timed out\n", + }); + }); + + it("preserves partial foreground output when a stdin command times out", async () => { + const sandbox = createMockSandbox(); + sandbox.commands.run.mockImplementation(async (command: string, options?: { background?: boolean }) => { + if (options?.background) return sandbox.handle; + if (command === "pwd") { + return { + exitCode: 0, + stdout: "/home/user\n", + stderr: "", + }; + } + throw new MockTimeoutError("command timed out", { + stdout: "stdin stdout\n", + stderr: "stdin stderr\n", + nested: true, + }); + }); + mockConnect.mockResolvedValue(sandbox); + + const result = await plugin.definition.onEnvironmentExecute?.({ + driverKey: "e2b", + companyId: "company-1", + environmentId: "env-1", + config: { + template: "base", + apiKey: "resolved-key", + timeoutMs: 300000, + reuseLease: false, + }, + lease: { providerLeaseId: "sandbox-123", metadata: {} }, + command: "printf", + args: ["hello"], + cwd: "/workspace", + env: { FOO: "bar" }, + stdin: "input", + timeoutMs: 1000, + }); + + expect(result).toEqual({ + exitCode: null, + timedOut: true, + stdout: "stdin stdout\n", + stderr: "stdin stderr\ncommand timed out\n", + }); + }); + it("pauses reusable leases and kills ephemeral leases on release", async () => { const reusable = createMockSandbox({ sandboxId: "sandbox-reusable" }); const ephemeral = createMockSandbox({ sandboxId: "sandbox-ephemeral" }); diff --git a/packages/plugins/sandbox-providers/e2b/src/plugin.ts b/packages/plugins/sandbox-providers/e2b/src/plugin.ts index 28e77f8c..3504620f 100644 --- a/packages/plugins/sandbox-providers/e2b/src/plugin.ts +++ b/packages/plugins/sandbox-providers/e2b/src/plugin.ts @@ -63,6 +63,34 @@ function formatErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function readTimeoutStream(error: TimeoutError, key: "stdout" | "stderr"): string { + const record = error as unknown as Record; + const direct = record[key]; + if (typeof direct === "string" && direct.length > 0) return direct; + const nested = (record as { result?: Record }).result?.[key]; + if (typeof nested === "string") return nested; + return typeof direct === "string" ? direct : ""; +} + +function buildTimeoutExecuteResult(error: TimeoutError): PluginEnvironmentExecuteResult { + const stdout = readTimeoutStream(error, "stdout"); + const stderrOutput = readTimeoutStream(error, "stderr"); + const message = error.message.trim(); + const stderr = stderrOutput.length > 0 + ? message.length > 0 && !stderrOutput.includes(message) + ? `${stderrOutput}${stderrOutput.endsWith("\n") ? "" : "\n"}${message}\n` + : stderrOutput + : message.length > 0 + ? `${message}\n` + : ""; + return { + exitCode: null, + timedOut: true, + stdout, + stderr, + }; +} + async function ensureSandboxWorkspace(sandbox: Sandbox, remoteCwd: string): Promise { await sandbox.commands.run(`mkdir -p ${shellQuote(remoteCwd)}`); } @@ -316,33 +344,64 @@ const plugin = definePlugin({ const config = parseDriverConfig(params.config); const sandbox = await connectSandbox(config, params.lease.providerLeaseId); - const started = await sandbox.commands.run(buildCommandLine(params.command, params.args), { - background: true, - stdin: params.stdin != null, + const command = buildCommandLine(params.command, params.args); + if (params.stdin == null) { + try { + const result = await sandbox.commands.run(command, { + cwd: params.cwd, + envs: params.env, + timeoutMs: params.timeoutMs ?? config.timeoutMs, + }) as Awaited> & { + exitCode: number; + stdout: string; + stderr: string; + }; + return { + exitCode: result.exitCode, + timedOut: false, + stdout: result.stdout, + stderr: result.stderr, + }; + } catch (error) { + if (error instanceof CommandExitError) { + const commandError = error as CommandExitError; + return { + exitCode: commandError.exitCode, + timedOut: false, + stdout: commandError.stdout, + stderr: commandError.stderr, + }; + } + if (error instanceof TimeoutError) { + return buildTimeoutExecuteResult(error); + } + throw error; + } + } + + const started = await sandbox.commands.run(command, { + stdin: true, cwd: params.cwd, envs: params.env, timeoutMs: params.timeoutMs ?? config.timeoutMs, }) as Awaited> & { pid: number; + exitCode: number; stdout: string; stderr: string; - wait(): Promise<{ exitCode: number; stdout: string; stderr: string }>; }; try { - if (params.stdin != null) { - try { - await sandbox.commands.sendStdin(started.pid, params.stdin); - } finally { - await sandbox.commands.closeStdin(started.pid); - } + try { + await sandbox.commands.sendStdin(started.pid, params.stdin); + } finally { + await sandbox.commands.closeStdin(started.pid); } - const result = await started.wait(); return { - exitCode: result.exitCode, + exitCode: started.exitCode, timedOut: false, - stdout: result.stdout, - stderr: result.stderr, + stdout: started.stdout, + stderr: started.stderr, }; } catch (error) { if (error instanceof CommandExitError) { @@ -355,13 +414,7 @@ const plugin = definePlugin({ }; } if (error instanceof TimeoutError) { - const timeoutError = error as TimeoutError; - return { - exitCode: null, - timedOut: true, - stdout: started.stdout, - stderr: started.stderr || `${timeoutError.message}\n`, - }; + return buildTimeoutExecuteResult(error); } throw error; } diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.ts index cd62d951..cad32753 100644 --- a/server/src/__tests__/claude-local-execute.test.ts +++ b/server/src/__tests__/claude-local-execute.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { runChildProcess } from "@paperclipai/adapter-utils/server-utils"; import { execute } from "@paperclipai/adapter-claude-local/server"; async function writeFailingClaudeCommand( @@ -37,6 +38,12 @@ const payload = { instructionsContents: instructionsFilePath ? fs.readFileSync(instructionsFilePath, "utf8") : null, skillEntries: addDir ? fs.readdirSync(path.join(addDir, ".claude", "skills")).sort() : [], claudeConfigDir: process.env.CLAUDE_CONFIG_DIR || null, + claudeConfigEntries: process.env.CLAUDE_CONFIG_DIR && fs.existsSync(process.env.CLAUDE_CONFIG_DIR) + ? fs.readdirSync(process.env.CLAUDE_CONFIG_DIR).sort() + : [], + paperclipApiUrl: process.env.PAPERCLIP_API_URL || null, + paperclipApiKey: process.env.PAPERCLIP_API_KEY || null, + paperclipApiBridgeMode: process.env.PAPERCLIP_API_BRIDGE_MODE || null, }; if (capturePath) { fs.writeFileSync(capturePath, JSON.stringify(payload), "utf8"); @@ -57,6 +64,10 @@ type CapturePayload = { instructionsContents: string | null; skillEntries: string[]; claudeConfigDir: string | null; + claudeConfigEntries?: string[]; + paperclipApiUrl?: string | null; + paperclipApiKey?: string | null; + paperclipApiBridgeMode?: string | null; appendedSystemPromptFilePath?: string | null; appendedSystemPromptFileContents?: string | null; }; @@ -129,6 +140,40 @@ async function setupExecuteEnv( }; } +function createLocalSandboxRunner() { + let counter = 0; + return { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + onSpawn?: (meta: { pid: number; startedAt: string }) => Promise; + }) => { + counter += 1; + return runChildProcess( + `sandbox-run-${counter}`, + input.command, + input.args ?? [], + { + cwd: input.cwd ?? process.cwd(), + env: input.env ?? {}, + stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), + graceSec: 5, + onLog: input.onLog ?? (async () => {}), + onSpawn: input.onSpawn + ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt }) + : undefined, + }, + ); + }, + }; +} + describe("claude execute", () => { /** * Regression tests for https://github.com/paperclipai/paperclip/issues/2848 @@ -398,6 +443,82 @@ describe("claude execute", () => { } }); + it("injects bridge env into sandbox-managed remote runs", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-sandbox-")); + const localWorkspace = path.join(root, "workspace"); + const remoteWorkspace = path.join(root, "sandbox-$HOME"); + const binDir = path.join(root, "bin"); + const commandPath = path.join(binDir, "claude"); + const capturePath = path.join(remoteWorkspace, "capture.json"); + const claudeRoot = path.join(root, ".claude"); + const previousHome = process.env.HOME; + const previousPath = process.env.PATH; + + await fs.mkdir(localWorkspace, { recursive: true }); + await fs.mkdir(remoteWorkspace, { recursive: true }); + await fs.mkdir(binDir, { recursive: true }); + await fs.mkdir(claudeRoot, { recursive: true }); + await fs.writeFile(path.join(claudeRoot, "settings.json"), JSON.stringify({ theme: "test" }), "utf8"); + await writeFakeClaudeCommand(commandPath); + + process.env.HOME = root; + process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`; + + try { + const result = await execute({ + runId: "run-sandbox-auth", + agent: { + id: "agent-1", + companyId: "company-1", + name: "Claude Coder", + adapterType: "claude_local", + adapterConfig: {}, + }, + runtime: { + sessionId: null, + sessionParams: null, + sessionDisplayId: null, + taskKey: null, + }, + config: { + command: commandPath, + cwd: localWorkspace, + env: { + PAPERCLIP_TEST_CAPTURE_PATH: capturePath, + }, + promptTemplate: "Follow the paperclip heartbeat.", + }, + context: {}, + executionTarget: { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd: remoteWorkspace, + timeoutMs: 30_000, + runner: createLocalSandboxRunner(), + }, + authToken: "run-jwt-token", + onLog: async () => {}, + }); + + expect(result.exitCode).toBe(0); + const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload; + expect(capture.claudeConfigDir).toBe(path.join(remoteWorkspace, ".paperclip-runtime", "claude", "config")); + expect(capture.claudeConfigEntries).toContain("settings.json"); + expect(capture.paperclipApiUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(capture.paperclipApiKey).not.toBe("run-jwt-token"); + expect(capture.paperclipApiBridgeMode).toBe("queue_v1"); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + await fs.rm(root, { recursive: true, force: true }); + } + }); + it("reuses a stable Paperclip-managed Claude prompt bundle across equivalent runs", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-bundle-")); const workspace = path.join(root, "workspace"); diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.ts index 5a5bc9f4..0f78f677 100644 --- a/server/src/__tests__/codex-local-execute.test.ts +++ b/server/src/__tests__/codex-local-execute.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { runChildProcess } from "@paperclipai/adapter-utils/server-utils"; import { execute } from "@paperclipai/adapter-codex-local/server"; async function writeFakeCodexCommand(commandPath: string): Promise { @@ -14,6 +15,9 @@ const payload = { prompt: fs.readFileSync(0, "utf8"), codexHome: process.env.CODEX_HOME || null, paperclipWakePayloadJson: process.env.PAPERCLIP_WAKE_PAYLOAD_JSON || null, + paperclipApiUrl: process.env.PAPERCLIP_API_URL || null, + paperclipApiKey: process.env.PAPERCLIP_API_KEY || null, + paperclipApiBridgeMode: process.env.PAPERCLIP_API_BRIDGE_MODE || null, paperclipEnvKeys: Object.keys(process.env) .filter((key) => key.startsWith("PAPERCLIP_")) .sort(), @@ -43,6 +47,9 @@ type CapturePayload = { prompt: string; codexHome: string | null; paperclipWakePayloadJson: string | null; + paperclipApiUrl?: string | null; + paperclipApiKey?: string | null; + paperclipApiBridgeMode?: string | null; paperclipEnvKeys: string[]; }; @@ -51,6 +58,40 @@ type LogEntry = { chunk: string; }; +function createLocalSandboxRunner() { + let counter = 0; + return { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + onSpawn?: (meta: { pid: number; startedAt: string }) => Promise; + }) => { + counter += 1; + return runChildProcess( + `sandbox-run-${counter}`, + input.command, + input.args ?? [], + { + cwd: input.cwd ?? process.cwd(), + env: input.env ?? {}, + stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), + graceSec: 5, + onLog: input.onLog ?? (async () => {}), + onSpawn: input.onSpawn + ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt }) + : undefined, + }, + ); + }, + }; +} + describe("codex execute", () => { it("uses a Paperclip-managed CODEX_HOME outside worktree mode while preserving shared auth and config", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-default-")); @@ -270,6 +311,80 @@ describe("codex execute", () => { } }); + it("injects bridge env into sandbox-managed remote runs", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-sandbox-")); + const localWorkspace = path.join(root, "workspace"); + const remoteWorkspace = path.join(root, "sandbox"); + const binDir = path.join(root, "bin"); + const commandPath = path.join(binDir, "codex"); + const capturePath = path.join(remoteWorkspace, "capture.json"); + const previousHome = process.env.HOME; + const previousPath = process.env.PATH; + + await fs.mkdir(localWorkspace, { recursive: true }); + await fs.mkdir(remoteWorkspace, { recursive: true }); + await fs.mkdir(binDir, { recursive: true }); + await writeFakeCodexCommand(commandPath); + + process.env.HOME = root; + process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`; + + try { + const result = await execute({ + runId: "run-sandbox-auth", + agent: { + id: "agent-1", + companyId: "company-1", + name: "Codex Coder", + adapterType: "codex_local", + adapterConfig: {}, + }, + runtime: { + sessionId: null, + sessionParams: null, + sessionDisplayId: null, + taskKey: null, + }, + config: { + command: commandPath, + cwd: localWorkspace, + env: { + PAPERCLIP_TEST_CAPTURE_PATH: capturePath, + }, + promptTemplate: "Follow the paperclip heartbeat.", + }, + context: {}, + executionTarget: { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd: remoteWorkspace, + timeoutMs: 30_000, + runner: createLocalSandboxRunner(), + }, + authToken: "run-jwt-token", + onLog: async () => {}, + }); + + expect(result.exitCode).toBe(0); + expect(result.errorMessage).toBeNull(); + + const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload; + expect(capture.codexHome).toBe(path.join(remoteWorkspace, ".paperclip-runtime", "codex", "home")); + expect(capture.paperclipApiUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(capture.paperclipApiKey).not.toBe("run-jwt-token"); + expect(capture.paperclipApiBridgeMode).toBe("queue_v1"); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + await fs.rm(root, { recursive: true, force: true }); + } + }); + it("injects structured Paperclip wake payloads into env and prompt", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-wake-")); const workspace = path.join(root, "workspace"); diff --git a/server/src/__tests__/environment-execution-target.test.ts b/server/src/__tests__/environment-execution-target.test.ts index b9e581d9..cd07ee9c 100644 --- a/server/src/__tests__/environment-execution-target.test.ts +++ b/server/src/__tests__/environment-execution-target.test.ts @@ -16,6 +16,8 @@ import { describe("resolveEnvironmentExecutionTarget", () => { beforeEach(() => { mockResolveEnvironmentDriverConfigForRuntime.mockReset(); + delete process.env.PAPERCLIP_API_URL; + delete process.env.PAPERCLIP_RUNTIME_API_URL; }); it("uses a bounded default cwd for sandbox targets when lease metadata omits remoteCwd", async () => { @@ -52,7 +54,47 @@ describe("resolveEnvironmentExecutionTarget", () => { remoteCwd: DEFAULT_SANDBOX_REMOTE_CWD, leaseId: "lease-1", environmentId: "env-1", + paperclipTransport: "bridge", timeoutMs: 30_000, }); }); + + it("prefers an explicit Paperclip API URL from lease metadata for sandbox targets", async () => { + process.env.PAPERCLIP_API_URL = "https://paperclip.example.test"; + process.env.PAPERCLIP_RUNTIME_API_URL = "http://paperclip.example.test:3200"; + mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({ + driver: "sandbox", + config: { + provider: "fake-plugin", + reuseLease: false, + timeoutMs: 30_000, + }, + }); + + const target = await resolveEnvironmentExecutionTarget({ + db: {} as never, + companyId: "company-1", + adapterType: "codex_local", + environment: { + id: "env-1", + driver: "sandbox", + config: { + provider: "fake-plugin", + }, + }, + leaseId: "lease-1", + leaseMetadata: { + paperclipApiUrl: "https://paperclip.example.test", + }, + lease: null, + environmentRuntime: null, + }); + + expect(target).toMatchObject({ + kind: "remote", + transport: "sandbox", + paperclipApiUrl: "https://paperclip.example.test", + paperclipTransport: "direct", + }); + }); }); diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 88213901..4817c448 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -174,6 +174,13 @@ function makeMockRuntime(overrides: Partial = {}): En return { acquireRunLease: vi.fn(), releaseRunLeases: vi.fn(), + execute: vi.fn().mockResolvedValue({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + }), realizeWorkspace: vi.fn().mockResolvedValue({ cwd: "/workspace/project", metadata: { @@ -347,4 +354,118 @@ describe("environmentRunOrchestrator — realizeForRun", () => { expect(result.lease).toEqual(updatedLease); expect(result.persistedExecutionWorkspace).toEqual(updatedEw); }); + + it("runs a remote provision command after workspace realization when configured", async () => { + mockBuildWorkspaceRealizationRequest.mockReturnValue({ + version: 1, + adapterType: "claude_local", + companyId: "company-1", + environmentId: "env-1", + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: "run-1", + requestedMode: null, + source: { + kind: "project_primary", + localPath: "/workspace/project", + projectId: null, + projectWorkspaceId: null, + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + runtimeOverlay: { + provisionCommand: "npm install -g @anthropic-ai/claude-code", + }, + }); + mockResolveEnvironmentExecutionTarget.mockResolvedValue({ + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + remoteCwd: "/remote/workspace", + environmentId: "env-1", + leaseId: "lease-1", + }); + + const runtime = makeMockRuntime({ + realizeWorkspace: vi.fn().mockResolvedValue({ + cwd: "/remote/workspace", + metadata: { + workspaceRealization: { + version: 1, + transport: "sandbox", + remote: { path: "/remote/workspace" }, + }, + }, + }), + }); + const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }); + + await orchestrator.realizeForRun(makeRealizeInput({ + environment: makeEnvironment("sandbox"), + })); + + expect(runtime.execute).toHaveBeenCalledOnce(); + expect(runtime.execute).toHaveBeenCalledWith(expect.objectContaining({ + environment: expect.objectContaining({ driver: "sandbox" }), + lease: expect.objectContaining({ id: "lease-1" }), + command: "bash", + args: ["-lc", "npm install -g @anthropic-ai/claude-code"], + cwd: "/remote/workspace", + env: { + SHELL: "/bin/bash", + }, + })); + }); + + it("surfaces remote provision command failures before resolving the adapter target", async () => { + mockBuildWorkspaceRealizationRequest.mockReturnValue({ + version: 1, + adapterType: "claude_local", + companyId: "company-1", + environmentId: "env-1", + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: "run-1", + requestedMode: null, + source: { + kind: "project_primary", + localPath: "/workspace/project", + projectId: null, + projectWorkspaceId: null, + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + runtimeOverlay: { + provisionCommand: "install-tool", + }, + }); + + const runtime = makeMockRuntime({ + execute: vi.fn().mockResolvedValue({ + exitCode: 127, + signal: null, + timedOut: false, + stdout: "", + stderr: "/bin/sh: install-tool: not found\n", + }), + }); + const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }); + + await expect(orchestrator.realizeForRun(makeRealizeInput({ + environment: makeEnvironment("sandbox"), + }))).rejects.toSatisfy( + (err: unknown) => + err instanceof EnvironmentRunError && + err.code === "workspace_realization_failed" && + String(err.message).includes("install-tool: not found"), + ); + + expect(mockResolveEnvironmentExecutionTarget).not.toHaveBeenCalled(); + }); }); diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 92ad85bd..49ceac60 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -60,9 +60,7 @@ export async function resolveEnvironmentExecutionTarget(input: { const paperclipApiUrl = typeof input.leaseMetadata?.paperclipApiUrl === "string" && input.leaseMetadata.paperclipApiUrl.trim().length > 0 ? input.leaseMetadata.paperclipApiUrl.trim() - : typeof process.env.PAPERCLIP_RUNTIME_API_URL === "string" && process.env.PAPERCLIP_RUNTIME_API_URL.trim().length > 0 - ? process.env.PAPERCLIP_RUNTIME_API_URL.trim() - : null; + : null; return { kind: "remote", @@ -72,6 +70,7 @@ export async function resolveEnvironmentExecutionTarget(input: { environmentId: input.environment.id ?? null, leaseId: input.leaseId ?? null, paperclipApiUrl, + paperclipTransport: paperclipApiUrl ? "direct" : "bridge", timeoutMs, runner: input.environmentRuntime && input.lease ? { diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 38f147d4..9b0a2066 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -114,6 +114,33 @@ export interface EnvironmentReleaseResult { errors: Array<{ leaseId: string; error: unknown }>; } +function firstNonEmptyLine(text: string | null | undefined): string | null { + if (!text) return null; + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (line) return line; + } + return null; +} + +function formatProvisionFailureDetail(result: { + exitCode: number | null; + signal?: string | null; + timedOut: boolean; + stdout: string; + stderr: string; +}): string { + if (result.timedOut) { + return "provision command timed out"; + } + const signal = typeof result.signal === "string" && result.signal.trim().length > 0 + ? ` (signal ${result.signal.trim()})` + : ""; + const detail = firstNonEmptyLine(result.stderr) ?? firstNonEmptyLine(result.stdout); + const status = `exit code ${result.exitCode ?? "null"}${signal}`; + return detail ? `${status}: ${detail}` : status; +} + // --------------------------------------------------------------------------- // Service factory // --------------------------------------------------------------------------- @@ -342,6 +369,7 @@ export function environmentRunOrchestrator( // Step 2: Realize workspace in the environment via the runtime driver let workspaceRealization: Record = {}; + let realizedWorkspaceCwd: string | null = null; if ( environment.driver === "local" || environment.driver === "ssh" || @@ -364,6 +392,10 @@ export function environmentRunOrchestrator( }, }, }); + realizedWorkspaceCwd = + typeof workspaceRealizationResult.cwd === "string" && workspaceRealizationResult.cwd.trim().length > 0 + ? workspaceRealizationResult.cwd.trim() + : null; workspaceRealization = parseObject(workspaceRealizationResult.metadata?.workspaceRealization); } catch (err) { throw new EnvironmentRunError( @@ -378,6 +410,41 @@ export function environmentRunOrchestrator( } } + const provisionCommand = workspaceRealizationRequest.runtimeOverlay.provisionCommand?.trim() ?? ""; + const realizedCwd = + realizedWorkspaceCwd ?? + (typeof lease.metadata?.remoteCwd === "string" && lease.metadata.remoteCwd.trim().length > 0 + ? lease.metadata.remoteCwd.trim() + : executionWorkspace.cwd); + if (provisionCommand && environment.driver !== "local") { + try { + const provisionResult = await environmentRuntime.execute({ + environment, + lease, + command: "bash", + args: ["-lc", provisionCommand], + cwd: realizedCwd, + env: { + SHELL: "/bin/bash", + }, + timeoutMs: 300_000, + }); + if (provisionResult.exitCode !== 0 || provisionResult.timedOut) { + throw new Error(formatProvisionFailureDetail(provisionResult)); + } + } catch (err) { + throw new EnvironmentRunError( + "workspace_realization_failed", + `Failed to provision workspace for environment "${environment.name}" (${environment.driver}): ${err instanceof Error ? err.message : String(err)}`, + { + environmentId: environment.id, + driver: environment.driver, + cause: err, + }, + ); + } + } + // Step 3: Persist realization metadata on lease and execution workspace if (Object.keys(workspaceRealization).length > 0) { const nextLeaseMetadata = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 92023f43..37cdb765 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -637,11 +637,12 @@ function createSandboxEnvironmentDriver( lease: input.lease, provider: providerKey, }); + const sanitizedConfig = stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig); return await pluginWorkerManager.call(pluginId, "environmentExecute", { driverKey: providerKey, companyId: input.lease.companyId, environmentId: input.environment.id, - config: stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig), + config: sanitizedConfig, lease: { providerLeaseId: input.lease.providerLeaseId, metadata: input.lease.metadata ?? undefined,