forked from farhoodlabs/paperclip
[codex] Add local Cloud Upstream sync (#6548)
## Thinking Path > - Paperclip is the control plane for AI-agent companies. > - Operators need a path to move local company state toward Paperclip Cloud without losing local-first control. > - The Cloud Upstream flow needs API, persistence, CLI, and board UI surfaces that agree on the same manifest/run model. > - The existing branch had the feature work plus UX and error-handling follow-ups. > - This pull request packages the remaining Cloud Upstream sync work into one standalone branch. > - The benefit is an inspectable local-to-cloud sync workflow with preview, conflicts, activation, and captured UX review states. ## What Changed - Added Cloud Upstream shared types, server routes/services, and persisted run schema/migration. - Added Paperclip Cloud CLI sync helpers and local connection storage. - Added the Cloud Upstream board UI, settings entry points, query keys, and UX lab page. - Added preview/activation checklist behavior, redirect handling, manifest-only preview support, friendly errors, in-flight hints, and entity count summaries. ## Verification - `pnpm --filter @paperclipai/plugin-sdk build` - `NODE_ENV=test pnpm exec vitest run cli/src/__tests__/cloud.test.ts server/src/__tests__/instance-settings-routes.test.ts server/src/__tests__/instance-settings-service.test.ts ui/src/pages/CloudUpstream.test.tsx ui/src/components/CompanySettingsSidebar.test.tsx` - `NODE_ENV=test pnpm exec vitest run server/src/__tests__/cloud-upstreams.test.ts` Worktree setup note: the isolated worktree install skipped native sqlite build scripts, so I copied the already-built local sqlite binding from the main checkout before running `server/src/__tests__/cloud-upstreams.test.ts`. The test then passed. ## Risks - Medium: this adds a database migration and a broad feature path across CLI/server/UI. - Merge order: this is the only PR in this split with a DB migration; merge it before any future Cloud Upstream migration follow-up. - Mitigation: the PR is based directly on current `origin/master`, has targeted route/service/UI tests, and keeps the feature behind existing experimental Cloud Sync settings. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via `codex_local`, tool-enabled coding session; exact context window not exposed by this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, screenshot artifacts are intentionally omitted per reviewer request - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta
@@ -9,6 +9,7 @@ import {
|
||||
createDb,
|
||||
documents,
|
||||
documentRevisions,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
@@ -42,6 +43,7 @@ describeEmbeddedPostgres("cleanup removal services", () => {
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(issueReadStates);
|
||||
await db.delete(issueComments);
|
||||
@@ -228,4 +230,32 @@ describeEmbeddedPostgres("cleanup removal services", () => {
|
||||
await expect(db.select().from(issueReadStates).where(eq(issueReadStates.companyId, companyId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(activityLog).where(eq(activityLog.companyId, companyId))).resolves.toHaveLength(0);
|
||||
});
|
||||
|
||||
it("removes heartbeat events by run id before deleting company-owned runs", async () => {
|
||||
const { agentId, companyId, runId } = await seedFixture();
|
||||
const otherCompanyId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: otherCompanyId,
|
||||
name: "Other Company",
|
||||
issuePrefix: `O${otherCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(heartbeatRunEvents).values({
|
||||
companyId: otherCompanyId,
|
||||
runId,
|
||||
agentId,
|
||||
seq: 1,
|
||||
eventType: "output",
|
||||
message: "event with mismatched company scope",
|
||||
});
|
||||
|
||||
const removed = await companyService(db).remove(companyId);
|
||||
|
||||
expect(removed?.id).toBe(companyId);
|
||||
await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(companies).where(eq(companies.id, otherCompanyId))).resolves.toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { companies, cloudUpstreamConnections, cloudUpstreamRuns, companySkills, createDb } from "@paperclipai/db";
|
||||
|
||||
import { HttpError } from "../errors.js";
|
||||
import {
|
||||
cloudUpstreamRemoteFailureReport,
|
||||
cloudUpstreamService,
|
||||
reconcileCloudUpstreamRunsOnStartup,
|
||||
sealCloudUpstreamCredential,
|
||||
unsealCloudUpstreamCredential,
|
||||
} from "../services/cloud-upstreams.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres cloud upstream tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describe("cloud upstream remote failures", () => {
|
||||
it("preserves the cloud response body and message on run reports", () => {
|
||||
const body = {
|
||||
error: "bad_request",
|
||||
message: "entities[42].body must be an object",
|
||||
errors: [{ path: "entities[42].body" }],
|
||||
};
|
||||
|
||||
expect(cloudUpstreamRemoteFailureReport(new HttpError(400, "bad_request", body))).toEqual({
|
||||
error: "bad_request",
|
||||
errorMessage: "entities[42].body must be an object",
|
||||
details: body,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the thrown error message for non-remote failures", () => {
|
||||
expect(cloudUpstreamRemoteFailureReport(new Error("network failed"))).toEqual({
|
||||
error: "network failed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("cloud upstream credential storage", () => {
|
||||
const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
|
||||
|
||||
afterEach(() => {
|
||||
if (previousMasterKey === undefined) {
|
||||
delete process.env.PAPERCLIP_SECRETS_MASTER_KEY;
|
||||
} else {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey;
|
||||
}
|
||||
});
|
||||
|
||||
it("stores new credentials as encrypted envelopes and preserves legacy plaintext reads", async () => {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012";
|
||||
const sealed = await sealCloudUpstreamCredential("cloud-access-token");
|
||||
|
||||
expect(sealed).toMatch(/^paperclip-cloud-credential:/);
|
||||
expect(sealed).not.toContain("cloud-access-token");
|
||||
await expect(unsealCloudUpstreamCredential(sealed)).resolves.toBe("cloud-access-token");
|
||||
await expect(unsealCloudUpstreamCredential("legacy-plaintext-token")).resolves.toBe("legacy-plaintext-token");
|
||||
});
|
||||
});
|
||||
|
||||
describeEmbeddedPostgres("cloud upstream persistence", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012";
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cloud-upstreams-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await db.delete(cloudUpstreamRuns);
|
||||
await db.delete(cloudUpstreamConnections);
|
||||
await db.delete(companySkills);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (previousMasterKey === undefined) {
|
||||
delete process.env.PAPERCLIP_SECRETS_MASTER_KEY;
|
||||
} else {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey;
|
||||
}
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("encrypts stored upstream credentials while keeping connection flows usable", async () => {
|
||||
const companyId = randomUUID();
|
||||
await seedCompany(companyId);
|
||||
const tokenUrl = "https://cloud.example.test/oauth/token";
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
|
||||
const url = String(input);
|
||||
if (url.startsWith("https://cloud.example.test/.well-known/paperclip-upstream")) {
|
||||
return jsonResponse({
|
||||
product: "Paperclip Cloud",
|
||||
stack: {
|
||||
id: "stack-1",
|
||||
companyId: "cloud-company-1",
|
||||
origin: "https://cloud.example.test",
|
||||
primaryHost: "cloud.example.test",
|
||||
},
|
||||
transfer: {
|
||||
supportedSchemaMajor: 1,
|
||||
maxChunkBytes: 8192,
|
||||
},
|
||||
auth: {
|
||||
scopes: ["upstream_import:write"],
|
||||
pkce: {
|
||||
authorizeUrl: "https://cloud.example.test/oauth/authorize",
|
||||
tokenUrl,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
if (url === tokenUrl && init?.method === "POST") {
|
||||
const payload = JSON.parse(String(init.body));
|
||||
expect(payload.codeVerifier).toEqual(expect.any(String));
|
||||
expect(payload.codeVerifier).not.toContain("paperclip-cloud-credential:");
|
||||
return jsonResponse({
|
||||
accessToken: "cloud-access-token",
|
||||
token: {
|
||||
id: "token-1",
|
||||
expiresAt: "2026-05-22T13:00:00.000Z",
|
||||
globalUserId: "user-1",
|
||||
},
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
});
|
||||
|
||||
const service = cloudUpstreamService(db, { instanceId: "test" });
|
||||
const started = await service.startConnect({
|
||||
companyId,
|
||||
remoteUrl: "https://cloud.example.test",
|
||||
redirectUri: "http://localhost:3100/callback",
|
||||
});
|
||||
await service.finishConnect({
|
||||
pendingConnectionId: started.pendingConnectionId,
|
||||
code: "auth-code",
|
||||
state: new URL(started.authorizationUrl).searchParams.get("state") ?? "",
|
||||
});
|
||||
|
||||
const [row] = await db.select().from(cloudUpstreamConnections);
|
||||
expect(row.privateKeyPem).toMatch(/^paperclip-cloud-credential:/);
|
||||
expect(row.privateKeyPem).not.toContain("BEGIN PRIVATE KEY");
|
||||
expect(row.accessToken).toMatch(/^paperclip-cloud-credential:/);
|
||||
expect(row.accessToken).not.toContain("cloud-access-token");
|
||||
});
|
||||
|
||||
it("marks orphaned running runs failed during startup reconciliation", async () => {
|
||||
const companyId = randomUUID();
|
||||
const connectionId = randomUUID();
|
||||
const runningRunId = randomUUID();
|
||||
const succeededRunId = randomUUID();
|
||||
const reconciledAt = new Date("2026-05-22T13:00:00.000Z");
|
||||
await seedCompany(companyId);
|
||||
await db.insert(cloudUpstreamConnections).values({
|
||||
id: connectionId,
|
||||
companyId,
|
||||
remoteUrl: "https://cloud.example.test",
|
||||
sourceInstanceId: "source-1",
|
||||
sourceInstanceFingerprint: "sha256:test",
|
||||
sourcePublicKey: "public-key",
|
||||
privateKeyPem: "legacy-private-key",
|
||||
tokenStatus: "connected",
|
||||
scopes: ["upstream_import:write"],
|
||||
authorizedGlobalUserId: "user-1",
|
||||
accessToken: "legacy-token",
|
||||
tokenId: "token-1",
|
||||
targetStackId: "stack-1",
|
||||
targetCompanyId: "cloud-company-1",
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
targetPrimaryHost: "cloud.example.test",
|
||||
targetProduct: "Paperclip Cloud",
|
||||
targetSchemaMajor: 1,
|
||||
targetMaxChunkBytes: 8192,
|
||||
});
|
||||
await db.insert(cloudUpstreamRuns).values([
|
||||
cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }),
|
||||
cloudRunRow({ id: succeededRunId, connectionId, companyId, status: "succeeded", completedAt: reconciledAt }),
|
||||
]);
|
||||
|
||||
await expect(reconcileCloudUpstreamRunsOnStartup(db, reconciledAt)).resolves.toEqual({ reconciled: 1 });
|
||||
|
||||
const rows = await db.select().from(cloudUpstreamRuns);
|
||||
const running = rows.find((row) => row.id === runningRunId);
|
||||
const succeeded = rows.find((row) => row.id === succeededRunId);
|
||||
expect(running?.status).toBe("failed");
|
||||
expect(running?.completedAt?.toISOString()).toBe(reconciledAt.toISOString());
|
||||
expect(running?.events.at(-1)?.message).toContain("server startup");
|
||||
expect(running?.report).toMatchObject({
|
||||
error: "orphaned_running_run",
|
||||
reconciledAt: reconciledAt.toISOString(),
|
||||
});
|
||||
expect(succeeded?.status).toBe("succeeded");
|
||||
});
|
||||
|
||||
it("rejects a new run when the connection already has a running run", async () => {
|
||||
const companyId = randomUUID();
|
||||
const connectionId = randomUUID();
|
||||
const runningRunId = randomUUID();
|
||||
await seedCompany(companyId);
|
||||
await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId }));
|
||||
await db.insert(cloudUpstreamRuns).values(
|
||||
cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }),
|
||||
);
|
||||
|
||||
await expect(cloudUpstreamService(db).createRun({ connectionId, companyId })).rejects.toMatchObject({
|
||||
status: 409,
|
||||
details: { runId: runningRunId },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a cancelled run when an in-flight createRun tries to finish", async () => {
|
||||
const companyId = randomUUID();
|
||||
const connectionId = randomUUID();
|
||||
await seedCompany(companyId);
|
||||
await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId }));
|
||||
|
||||
const service = cloudUpstreamService(db);
|
||||
const remoteCalls: string[] = [];
|
||||
globalThis.fetch = vi.fn(async (input) => {
|
||||
const path = new URL(String(input)).pathname;
|
||||
remoteCalls.push(path);
|
||||
if (path.endsWith("/upstream-imports/runs")) {
|
||||
return jsonResponse({ run: { id: "remote-run-1" } });
|
||||
}
|
||||
if (path.endsWith("/chunks")) {
|
||||
const run = await db.select().from(cloudUpstreamRuns).then((rows) => rows[0]);
|
||||
expect(run?.status).toBe("running");
|
||||
await service.cancelRun(connectionId, run.id, companyId);
|
||||
return jsonResponse({ ok: true });
|
||||
}
|
||||
if (path.endsWith("/cancel")) {
|
||||
return jsonResponse({ ok: true });
|
||||
}
|
||||
if (path.endsWith("/apply")) {
|
||||
return jsonResponse({ ok: true });
|
||||
}
|
||||
if (path.endsWith("/events")) {
|
||||
return jsonResponse({ events: [] });
|
||||
}
|
||||
return jsonResponse({ error: "not_found" }, 404);
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await service.createRun({ connectionId, companyId });
|
||||
|
||||
expect(result.status).toBe("cancelled");
|
||||
expect(remoteCalls.some((path) => path.endsWith("/apply"))).toBe(false);
|
||||
const rows = await db.select().from(cloudUpstreamRuns);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]?.status).toBe("cancelled");
|
||||
});
|
||||
|
||||
async function seedCompany(companyId: string) {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
function jsonResponse(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function cloudConnectionRow(input: { id: string; companyId: string }) {
|
||||
const { privateKey } = generateKeyPairSync("ed25519");
|
||||
return {
|
||||
id: input.id,
|
||||
companyId: input.companyId,
|
||||
remoteUrl: "https://cloud.example.test",
|
||||
sourceInstanceId: "source-1",
|
||||
sourceInstanceFingerprint: "sha256:test",
|
||||
sourcePublicKey: "public-key",
|
||||
privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
|
||||
tokenStatus: "connected",
|
||||
scopes: ["upstream_import:write"],
|
||||
authorizedGlobalUserId: "user-1",
|
||||
accessToken: "legacy-token",
|
||||
tokenId: "token-1",
|
||||
targetStackId: "stack-1",
|
||||
targetCompanyId: "cloud-company-1",
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
targetPrimaryHost: "cloud.example.test",
|
||||
targetProduct: "Paperclip Cloud",
|
||||
targetSchemaMajor: 1,
|
||||
targetMaxChunkBytes: 8192,
|
||||
};
|
||||
}
|
||||
|
||||
function cloudRunRow(input: {
|
||||
id: string;
|
||||
connectionId: string;
|
||||
companyId: string;
|
||||
status: string;
|
||||
completedAt?: Date;
|
||||
}) {
|
||||
return {
|
||||
id: input.id,
|
||||
connectionId: input.connectionId,
|
||||
companyId: input.companyId,
|
||||
status: input.status,
|
||||
activeStep: "push",
|
||||
progressPercent: input.status === "running" ? 45 : 100,
|
||||
dryRun: false,
|
||||
summary: [],
|
||||
warnings: [],
|
||||
conflicts: [],
|
||||
events: [],
|
||||
report: {},
|
||||
idempotencyKey: `key-${input.id}`,
|
||||
manifestHash: `sha256:${input.id.replace(/-/g, "")}`,
|
||||
targetUrl: "https://cloud.example.test",
|
||||
completedAt: input.completedAt,
|
||||
};
|
||||
}
|
||||
@@ -64,6 +64,7 @@ describe("instance settings routes", () => {
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
enableCloudSync: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
@@ -81,6 +82,7 @@ describe("instance settings routes", () => {
|
||||
experimental: {
|
||||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
enableCloudSync: true,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
@@ -123,6 +125,7 @@ describe("instance settings routes", () => {
|
||||
expect(getRes.body).toEqual({
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
enableCloudSync: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeExperimentalSettings } from "../services/instance-settings.js";
|
||||
|
||||
describe("instance settings service", () => {
|
||||
it("ignores retired experimental flags without resetting current settings", () => {
|
||||
expect(normalizeExperimentalSettings({
|
||||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
enableCloudSync: true,
|
||||
autoRestartDevServerWhenIdle: true,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 48,
|
||||
enableNewestFirstIssueThread: true,
|
||||
})).toEqual({
|
||||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
enableCloudSync: true,
|
||||
autoRestartDevServerWhenIdle: true,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 48,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -166,6 +166,7 @@ vi.mock("../services/index.js", () => ({
|
||||
},
|
||||
})),
|
||||
})),
|
||||
reconcileCloudUpstreamRunsOnStartup: vi.fn(async () => ({ reconciled: 0 })),
|
||||
reconcilePersistedRuntimeServicesOnStartup: vi.fn(async () => ({ reconciled: 0 })),
|
||||
routineService: vi.fn(() => ({
|
||||
tickScheduledTriggers: vi.fn(async () => ({ triggered: 0 })),
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
backfillPrincipalAccessCompatibility,
|
||||
heartbeatService,
|
||||
instanceSettingsService,
|
||||
reconcileCloudUpstreamRunsOnStartup,
|
||||
reconcilePersistedRuntimeServicesOnStartup,
|
||||
routineService,
|
||||
} from "./services/index.js";
|
||||
@@ -699,6 +700,19 @@ export async function startServer(): Promise<StartedServer> {
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "startup reconciliation of persisted runtime services failed");
|
||||
});
|
||||
|
||||
void reconcileCloudUpstreamRunsOnStartup(db as any)
|
||||
.then((result) => {
|
||||
if (result.reconciled > 0) {
|
||||
logger.warn(
|
||||
{ reconciled: result.reconciled },
|
||||
"reconciled cloud upstream runs from a previous server process",
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "startup reconciliation of cloud upstream runs failed");
|
||||
});
|
||||
|
||||
if (config.heartbeatSchedulerEnabled) {
|
||||
const heartbeat = heartbeatService(db as any, { pluginWorkerManager });
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { Router } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { badRequest, notFound } from "../errors.js";
|
||||
import { assertBoardOrgAccess } from "./authz.js";
|
||||
import { cloudUpstreamService, instanceSettingsService } from "../services/index.js";
|
||||
|
||||
export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {}) {
|
||||
const router = Router();
|
||||
const service = cloudUpstreamService(db, options);
|
||||
const settings = instanceSettingsService(db);
|
||||
|
||||
async function assertEnabled() {
|
||||
const experimental = await settings.getExperimental();
|
||||
if (experimental.enableCloudSync !== true) {
|
||||
throw notFound("Cloud sync is not enabled");
|
||||
}
|
||||
}
|
||||
|
||||
router.get("/cloud-upstreams", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
const companyId = stringQuery(req.query.companyId, "companyId");
|
||||
res.json(await service.list(companyId));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/connect/start", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
const companyId = stringBody(req.body, "companyId");
|
||||
const remoteUrl = stringBody(req.body, "remoteUrl");
|
||||
const redirectUri = stringBody(req.body, "redirectUri");
|
||||
res.json(await service.startConnect({ companyId, remoteUrl, redirectUri }));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/connect/finish", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.finishConnect({
|
||||
pendingConnectionId: stringBody(req.body, "pendingConnectionId"),
|
||||
code: stringBody(req.body, "code"),
|
||||
state: stringBody(req.body, "state"),
|
||||
}));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/:connectionId/push-runs/preview", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.preview(req.params.connectionId, stringBody(req.body, "companyId")));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/:connectionId/push-runs", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.createRun({
|
||||
connectionId: req.params.connectionId,
|
||||
companyId: stringBody(req.body, "companyId"),
|
||||
retryOfRunId: optionalString(req.body?.retryOfRunId),
|
||||
}));
|
||||
});
|
||||
|
||||
router.get("/cloud-upstreams/:connectionId/push-runs/:runId", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.readRun(req.params.connectionId, req.params.runId, stringQuery(req.query.companyId, "companyId")));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/:connectionId/push-runs/:runId/cancel", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.cancelRun(req.params.connectionId, req.params.runId, stringBody(req.body, "companyId")));
|
||||
});
|
||||
|
||||
router.post("/cloud-upstreams/:connectionId/push-runs/:runId/activation", async (req, res) => {
|
||||
assertBoardOrgAccess(req);
|
||||
await assertEnabled();
|
||||
res.json(await service.activateRunEntities({
|
||||
connectionId: req.params.connectionId,
|
||||
runId: req.params.runId,
|
||||
companyId: stringBody(req.body, "companyId"),
|
||||
entityType: activationEntityTypeBody(req.body),
|
||||
}));
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
function stringQuery(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || value.trim().length === 0) {
|
||||
throw badRequest(`${label} is required`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function stringBody(body: unknown, key: string): string {
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) {
|
||||
throw badRequest(`${key} is required`);
|
||||
}
|
||||
const value = (body as Record<string, unknown>)[key];
|
||||
if (typeof value !== "string" || value.trim().length === 0) {
|
||||
throw badRequest(`${key} is required`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function activationEntityTypeBody(body: unknown): "agents" | "routines" | "monitors" {
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) {
|
||||
throw badRequest("entityType is required");
|
||||
}
|
||||
const value = (body as Record<string, unknown>).entityType;
|
||||
if (value !== "agents" && value !== "routines" && value !== "monitors") {
|
||||
throw badRequest("entityType must be agents, routines, or monitors");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -19,3 +19,4 @@ export { llmRoutes } from "./llms.js";
|
||||
export { accessRoutes } from "./access.js";
|
||||
export { instanceSettingsRoutes } from "./instance-settings.js";
|
||||
export { instanceDatabaseBackupRoutes } from "./instance-database-backups.js";
|
||||
export { cloudUpstreamRoutes } from "./cloud-upstreams.js";
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -265,7 +265,17 @@ export function companyService(db: Db) {
|
||||
remove: (id: string) =>
|
||||
db.transaction(async (tx) => {
|
||||
// Delete from child tables in dependency order
|
||||
const companyRunIds = await tx
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, id));
|
||||
|
||||
await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, id));
|
||||
if (companyRunIds.length > 0) {
|
||||
await tx
|
||||
.delete(heartbeatRunEvents)
|
||||
.where(inArray(heartbeatRunEvents.runId, companyRunIds.map((run) => run.id)));
|
||||
}
|
||||
await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, id));
|
||||
await tx.delete(activityLog).where(eq(activityLog.companyId, id));
|
||||
await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id));
|
||||
|
||||
@@ -59,6 +59,7 @@ export type {
|
||||
} from "./authorization.js";
|
||||
export { boardAuthService } from "./board-auth.js";
|
||||
export { instanceSettingsService } from "./instance-settings.js";
|
||||
export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js";
|
||||
export { companyPortabilityService } from "./company-portability.js";
|
||||
export { environmentService } from "./environments.js";
|
||||
export { executionWorkspaceService } from "./execution-workspaces.js";
|
||||
|
||||
@@ -15,9 +15,11 @@ import {
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
const DEFAULT_SINGLETON_KEY = "default";
|
||||
const instanceGeneralSettingsStorageSchema = instanceGeneralSettingsSchema.strip();
|
||||
const instanceExperimentalSettingsStorageSchema = instanceExperimentalSettingsSchema.strip();
|
||||
|
||||
function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {
|
||||
const parsed = instanceGeneralSettingsSchema.safeParse(raw ?? {});
|
||||
const parsed = instanceGeneralSettingsStorageSchema.safeParse(raw ?? {});
|
||||
if (parsed.success) {
|
||||
return {
|
||||
censorUsernameInLogs: parsed.data.censorUsernameInLogs ?? false,
|
||||
@@ -35,12 +37,13 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettings {
|
||||
const parsed = instanceExperimentalSettingsSchema.safeParse(raw ?? {});
|
||||
export function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettings {
|
||||
const parsed = instanceExperimentalSettingsStorageSchema.safeParse(raw ?? {});
|
||||
if (parsed.success) {
|
||||
return {
|
||||
enableEnvironments: parsed.data.enableEnvironments ?? false,
|
||||
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
|
||||
enableCloudSync: parsed.data.enableCloudSync ?? false,
|
||||
autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false,
|
||||
enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? false,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
@@ -51,6 +54,7 @@ function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettin
|
||||
return {
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
enableCloudSync: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
|
||||
Reference in New Issue
Block a user