forked from farhoodlabs/paperclip
Expand plugin host surface (#5205)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - The plugin system is the extension boundary for optional product capabilities > - Rich plugins need more than a worker entrypoint: they need scoped database storage, local project folders, managed agents/routines, host navigation, and reusable UI components > - The LLM Wiki work exposed those missing host surfaces while keeping plugin code outside the core control plane > - This pull request expands the core plugin host, SDK, server APIs, and UI bridge so plugins can declare and use those surfaces > - The benefit is that future plugins can integrate with Paperclip through documented, validated contracts instead of bespoke server or UI imports ## What Changed - Added plugin-managed database namespaces and migration tracking, including Drizzle schema/migration files and SQL validation for namespace isolation. - Added server support for plugin local folders, managed agents, managed routines, scoped plugin APIs, and plugin operation visibility. - Expanded shared plugin manifest/types/validators and SDK host/testing/UI exports for richer plugin surfaces. - Added reusable UI pieces for file trees, managed routines, resizable sidebars, route sidebars, and plugin bridge initialization. - Updated plugin docs and example plugins to use the expanded host and SDK surface. ## Verification - `pnpm install --frozen-lockfile` - `pnpm run preflight:workspace-links && pnpm exec vitest run packages/shared/src/validators/plugin.test.ts server/src/__tests__/plugin-database.test.ts server/src/__tests__/plugin-local-folders.test.ts server/src/__tests__/plugin-managed-agents.test.ts server/src/__tests__/plugin-managed-routines.test.ts server/src/__tests__/plugin-orchestration-apis.test.ts ui/src/api/plugins.test.ts ui/src/components/FileTree.test.tsx ui/src/components/ResizableSidebarPane.test.tsx ui/src/pages/PluginPage.test.tsx ui/src/plugins/bridge.test.ts` passed: 11 files, 67 tests. - Confirmed this PR changes 89 files and does not include `pnpm-lock.yaml` or `.github/workflows/*`. ## Risks - Medium: this expands plugin host contracts across db/shared/server/ui and includes a new core migration (`0076_useful_elektra.sql`). - The plugin database namespace validator is intentionally restrictive; plugin authors may need follow-up affordances for SQL patterns that remain blocked. - Merge this before the LLM Wiki plugin PR so the plugin can resolve the new SDK and host APIs. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 coding agent, tool-enabled shell/git/GitHub workflow. Context window size was not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
@@ -303,7 +303,19 @@ function resolveMigrationsDir(packageRoot: string, migrationsDir: string): strin
|
||||
return resolvedDir;
|
||||
}
|
||||
|
||||
export function pluginDatabaseService(db: Db) {
|
||||
type PluginDatabaseClient = Pick<Db, "select" | "insert" | "update" | "execute">;
|
||||
type PluginDatabaseRootClient = PluginDatabaseClient & Partial<Pick<Db, "transaction">>;
|
||||
|
||||
export interface ApplyPluginMigrationsOptions {
|
||||
/**
|
||||
* Persist failed migration ledger rows. Fresh install uses false because the
|
||||
* caller owns a larger transaction and must roll back the plugin row and
|
||||
* namespace together.
|
||||
*/
|
||||
persistFailure?: boolean;
|
||||
}
|
||||
|
||||
export function pluginDatabaseService(db: PluginDatabaseRootClient) {
|
||||
async function getPluginRecord(pluginId: string) {
|
||||
const rows = await db.select().from(plugins).where(eq(plugins.id, pluginId)).limit(1);
|
||||
const plugin = rows[0];
|
||||
@@ -311,14 +323,18 @@ export function pluginDatabaseService(db: Db) {
|
||||
return plugin;
|
||||
}
|
||||
|
||||
async function ensureNamespace(pluginId: string, manifest: PaperclipPluginManifestV1) {
|
||||
async function ensureNamespaceWithClient(
|
||||
client: PluginDatabaseClient,
|
||||
pluginId: string,
|
||||
manifest: PaperclipPluginManifestV1,
|
||||
) {
|
||||
if (!manifest.database) return null;
|
||||
const namespaceName = derivePluginDatabaseNamespace(
|
||||
manifest.id,
|
||||
manifest.database.namespaceSlug,
|
||||
);
|
||||
await db.execute(sql.raw(`CREATE SCHEMA IF NOT EXISTS ${quoteIdentifier(namespaceName)}`));
|
||||
const rows = await db
|
||||
await client.execute(sql.raw(`CREATE SCHEMA IF NOT EXISTS ${quoteIdentifier(namespaceName)}`));
|
||||
const rows = await client
|
||||
.insert(pluginDatabaseNamespaces)
|
||||
.values({
|
||||
pluginId,
|
||||
@@ -341,6 +357,10 @@ export function pluginDatabaseService(db: Db) {
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function ensureNamespace(pluginId: string, manifest: PaperclipPluginManifestV1) {
|
||||
return ensureNamespaceWithClient(db, pluginId, manifest);
|
||||
}
|
||||
|
||||
async function getNamespace(pluginId: string) {
|
||||
const rows = await db
|
||||
.select()
|
||||
@@ -358,7 +378,7 @@ export function pluginDatabaseService(db: Db) {
|
||||
return namespace.namespaceName;
|
||||
}
|
||||
|
||||
async function recordMigrationFailure(input: {
|
||||
async function recordMigrationFailure(client: PluginDatabaseClient, input: {
|
||||
pluginId: string;
|
||||
pluginKey: string;
|
||||
namespaceName: string;
|
||||
@@ -368,7 +388,7 @@ export function pluginDatabaseService(db: Db) {
|
||||
error: unknown;
|
||||
}): Promise<void> {
|
||||
const message = input.error instanceof Error ? input.error.message : String(input.error);
|
||||
await db
|
||||
await client
|
||||
.insert(pluginMigrations)
|
||||
.values({
|
||||
pluginId: input.pluginId,
|
||||
@@ -391,7 +411,7 @@ export function pluginDatabaseService(db: Db) {
|
||||
appliedAt: null,
|
||||
},
|
||||
});
|
||||
await db
|
||||
await client
|
||||
.update(pluginDatabaseNamespaces)
|
||||
.set({ status: "migration_failed", updatedAt: new Date() })
|
||||
.where(eq(pluginDatabaseNamespaces.pluginId, input.pluginId));
|
||||
@@ -400,7 +420,12 @@ export function pluginDatabaseService(db: Db) {
|
||||
return {
|
||||
ensureNamespace,
|
||||
|
||||
async applyMigrations(pluginId: string, manifest: PaperclipPluginManifestV1, packageRoot: string) {
|
||||
async applyMigrations(
|
||||
pluginId: string,
|
||||
manifest: PaperclipPluginManifestV1,
|
||||
packageRoot: string,
|
||||
options: ApplyPluginMigrationsOptions = {},
|
||||
) {
|
||||
if (!manifest.database) return null;
|
||||
const namespace = await ensureNamespace(pluginId, manifest);
|
||||
if (!namespace) return null;
|
||||
@@ -409,13 +434,14 @@ export function pluginDatabaseService(db: Db) {
|
||||
const migrationFiles = await listSqlMigrationFiles(migrationDir);
|
||||
const coreReadTables = manifest.database.coreReadTables ?? [];
|
||||
const lockKey = Number.parseInt(createHash("sha256").update(pluginId).digest("hex").slice(0, 12), 16);
|
||||
const persistFailure = options.persistFailure ?? true;
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`SELECT pg_advisory_xact_lock(${lockKey})`);
|
||||
const applyWithClient = async (client: PluginDatabaseClient) => {
|
||||
await client.execute(sql`SELECT pg_advisory_xact_lock(${lockKey})`);
|
||||
for (const migrationKey of migrationFiles) {
|
||||
const content = await readFile(path.join(migrationDir, migrationKey), "utf8");
|
||||
const checksum = createHash("sha256").update(content).digest("hex");
|
||||
const existingRows = await tx
|
||||
const existingRows = await client
|
||||
.select()
|
||||
.from(pluginMigrations)
|
||||
.where(and(eq(pluginMigrations.pluginId, pluginId), eq(pluginMigrations.migrationKey, migrationKey)))
|
||||
@@ -435,9 +461,9 @@ export function pluginDatabaseService(db: Db) {
|
||||
}
|
||||
for (const statement of statements) {
|
||||
validatePluginMigrationStatement(statement, namespace.namespaceName, coreReadTables);
|
||||
await tx.execute(sql.raw(statement));
|
||||
await client.execute(sql.raw(statement));
|
||||
}
|
||||
await tx
|
||||
await client
|
||||
.insert(pluginMigrations)
|
||||
.values({
|
||||
pluginId,
|
||||
@@ -461,19 +487,27 @@ export function pluginDatabaseService(db: Db) {
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
await recordMigrationFailure({
|
||||
pluginId,
|
||||
pluginKey: manifest.id,
|
||||
namespaceName: namespace.namespaceName,
|
||||
migrationKey,
|
||||
checksum,
|
||||
pluginVersion: manifest.version,
|
||||
error,
|
||||
});
|
||||
if (persistFailure) {
|
||||
await recordMigrationFailure(db, {
|
||||
pluginId,
|
||||
pluginKey: manifest.id,
|
||||
namespaceName: namespace.namespaceName,
|
||||
migrationKey,
|
||||
checksum,
|
||||
pluginVersion: manifest.version,
|
||||
error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
if (typeof db.transaction === "function") {
|
||||
await db.transaction(async (tx) => applyWithClient(tx as PluginDatabaseClient));
|
||||
} else {
|
||||
await applyWithClient(db);
|
||||
}
|
||||
|
||||
return namespace;
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user