Files
paperclip/packages/db/src/migration-runtime.ts
T
Dotta ad6effa65c [codex] Improve runtime and import reliability (#6549)
## Thinking Path

> - Paperclip coordinates autonomous company work through local and
hosted runtime surfaces.
> - Local embedded Postgres and tenant import/export paths are
foundational reliability pieces.
> - A runtime failure in either path can stop agents or imports before
useful work begins.
> - The branch included remaining fixes for embedded native library
bootstrap and async tenant import handling.
> - This pull request groups those runtime/import reliability changes
into one standalone PR.
> - The benefit is a more robust local runtime and safer cloud tenant
import behavior.

## What Changed

- Prepared embedded Postgres native runtime before startup in
CLI/server/test entrypoints.
- Added embedded Postgres native bootstrap coverage.
- Added async tenant import job handling and deferred validation
coverage.
- Kept the runtime/import changes based directly on current
`origin/master` after related upstream PRs had already merged.

## Verification

- `pnpm --filter @paperclipai/plugin-sdk build`
- `NODE_ENV=test pnpm exec vitest run
packages/db/src/embedded-postgres-native.test.ts
server/src/__tests__/company-portability-routes.test.ts`

## Risks

- Medium-low: this touches startup/import paths, but the branch is small
and covered by targeted tests.
- The embedded Postgres change depends on platform-specific
native-library behavior, so CI and follow-up checks should still verify
supported runners.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI GPT-5 Codex via `codex_local`, tool-enabled coding session;
exact context window not exposed by this runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
2026-05-22 09:57:22 -05:00

197 lines
6.5 KiB
TypeScript

import { existsSync, readFileSync, rmSync } from "node:fs";
import { createServer } from "node:net";
import path from "node:path";
import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js";
import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
import { prepareEmbeddedPostgresNativeRuntime } from "./embedded-postgres-native.js";
import { resolveDatabaseTarget } from "./runtime-config.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
export type MigrationConnection = {
connectionString: string;
source: string;
stop: () => Promise<void>;
};
function readRunningPostmasterPid(postmasterPidFile: string): number | null {
if (!existsSync(postmasterPidFile)) return null;
try {
const pid = Number(readFileSync(postmasterPidFile, "utf8").split("\n")[0]?.trim());
if (!Number.isInteger(pid) || pid <= 0) return null;
process.kill(pid, 0);
return pid;
} catch {
return null;
}
}
function readPidFilePort(postmasterPidFile: string): number | null {
if (!existsSync(postmasterPidFile)) return null;
try {
const lines = readFileSync(postmasterPidFile, "utf8").split("\n");
const port = Number(lines[3]?.trim());
return Number.isInteger(port) && port > 0 ? port : null;
} catch {
return null;
}
}
async function isPortInUse(port: number): Promise<boolean> {
return await new Promise((resolve) => {
const server = createServer();
server.unref();
server.once("error", (error: NodeJS.ErrnoException) => {
resolve(error.code === "EADDRINUSE");
});
server.listen(port, "127.0.0.1", () => {
server.close();
resolve(false);
});
});
}
async function findAvailablePort(startPort: number): Promise<number> {
const maxLookahead = 20;
let port = startPort;
for (let i = 0; i < maxLookahead; i += 1, port += 1) {
if (!(await isPortInUse(port))) return port;
}
throw new Error(
`Embedded PostgreSQL could not find a free port from ${startPort} to ${startPort + maxLookahead - 1}`,
);
}
async function loadEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
try {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
} catch {
throw new Error(
"Embedded PostgreSQL support requires dependency `embedded-postgres`. Reinstall dependencies and try again.",
);
}
}
async function ensureEmbeddedPostgresConnection(
dataDir: string,
preferredPort: number,
): Promise<MigrationConnection> {
const EmbeddedPostgres = await loadEmbeddedPostgresCtor();
await prepareEmbeddedPostgresNativeRuntime();
const selectedPort = await findAvailablePort(preferredPort);
const postmasterPidFile = path.resolve(dataDir, "postmaster.pid");
const pgVersionFile = path.resolve(dataDir, "PG_VERSION");
const runningPid = readRunningPostmasterPid(postmasterPidFile);
const runningPort = readPidFilePort(postmasterPidFile);
const preferredAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`;
const logBuffer = createEmbeddedPostgresLogBuffer();
if (!runningPid && existsSync(pgVersionFile)) {
try {
const actualDataDir = await getPostgresDataDirectory(preferredAdminConnectionString);
const matchesDataDir =
typeof actualDataDir === "string" &&
path.resolve(actualDataDir) === path.resolve(dataDir);
if (!matchesDataDir) {
throw new Error("reachable postgres does not use the expected embedded data directory");
}
await ensurePostgresDatabase(preferredAdminConnectionString, "paperclip");
process.emitWarning(
`Adopting an existing PostgreSQL instance on port ${preferredPort} for embedded data dir ${dataDir} because postmaster.pid is missing.`,
);
return {
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/paperclip`,
source: `embedded-postgres@${preferredPort}`,
stop: async () => {},
};
} catch {
// Fall through and attempt to start the configured embedded cluster.
}
}
if (runningPid) {
const port = runningPort ?? preferredPort;
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
return {
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`,
source: `embedded-postgres@${port}`,
stop: async () => {},
};
}
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port: selectedPort,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: logBuffer.append,
onError: logBuffer.append,
});
if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
try {
await instance.initialise();
} catch (error) {
throw formatEmbeddedPostgresError(error, {
fallbackMessage:
`Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
recentLogs: logBuffer.getRecentLogs(),
});
}
}
if (existsSync(postmasterPidFile)) {
rmSync(postmasterPidFile, { force: true });
}
try {
await instance.start();
} catch (error) {
throw formatEmbeddedPostgresError(error, {
fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`,
recentLogs: logBuffer.getRecentLogs(),
});
}
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
return {
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/paperclip`,
source: `embedded-postgres@${selectedPort}`,
stop: async () => {
await instance.stop();
},
};
}
export async function resolveMigrationConnection(): Promise<MigrationConnection> {
const target = resolveDatabaseTarget();
if (target.mode === "postgres") {
return {
connectionString: target.connectionString,
source: target.source,
stop: async () => {},
};
}
return ensureEmbeddedPostgresConnection(target.dataDir, target.port);
}