diff --git a/packages/plugins/plugin-llm-wiki/migrations/003_spaces.sql b/packages/plugins/plugin-llm-wiki/migrations/003_spaces.sql index e9c4a0fd..17d900a8 100644 --- a/packages/plugins/plugin-llm-wiki/migrations/003_spaces.sql +++ b/packages/plugins/plugin-llm-wiki/migrations/003_spaces.sql @@ -140,37 +140,14 @@ ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_runs ALTER COLUMN ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_source_snapshots ALTER COLUMN space_id SET NOT NULL; ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_page_bindings ALTER COLUMN space_id SET NOT NULL; -DO $$ -DECLARE - target record; - constraint_name text; -BEGIN - FOR target IN - SELECT * FROM (VALUES - ('wiki_pages', ARRAY['company_id', 'wiki_id', 'path']::text[]), - ('paperclip_distillation_cursors', ARRAY['company_id', 'wiki_id', 'source_scope', 'scope_key', 'source_kind']::text[]), - ('paperclip_distillation_work_items', ARRAY['company_id', 'wiki_id', 'idempotency_key']::text[]), - ('paperclip_page_bindings', ARRAY['company_id', 'wiki_id', 'page_path']::text[]) - ) AS targets(table_name, column_names) - LOOP - FOR constraint_name IN - SELECT c.conname - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE n.nspname = 'plugin_llm_wiki_8f50da974f' - AND t.relname = target.table_name - AND c.contype = 'u' - AND ( - SELECT array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[] - FROM unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality) - JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum - ) = target.column_names - LOOP - EXECUTE format('ALTER TABLE %I.%I DROP CONSTRAINT %I', 'plugin_llm_wiki_8f50da974f', target.table_name, constraint_name); - END LOOP; - END LOOP; -END $$; +ALTER TABLE plugin_llm_wiki_8f50da974f.wiki_pages + DROP CONSTRAINT IF EXISTS wiki_pages_company_id_wiki_id_path_key; +ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_cursors + DROP CONSTRAINT IF EXISTS paperclip_distillation_cursor_company_id_wiki_id_source_sco_key; +ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_work_items + DROP CONSTRAINT IF EXISTS paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key; +ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_page_bindings + DROP CONSTRAINT IF EXISTS paperclip_page_bindings_company_id_wiki_id_page_path_key; ALTER TABLE plugin_llm_wiki_8f50da974f.wiki_pages DROP CONSTRAINT IF EXISTS wiki_pages_company_wiki_space_path_key; diff --git a/packages/plugins/plugin-llm-wiki/package.json b/packages/plugins/plugin-llm-wiki/package.json index 4461c845..6f175b4a 100644 --- a/packages/plugins/plugin-llm-wiki/package.json +++ b/packages/plugins/plugin-llm-wiki/package.json @@ -4,6 +4,14 @@ "type": "module", "private": true, "description": "Local-file LLM Wiki plugin for source ingestion, wiki browsing, query, lint, and maintenance workflows.", + "files": [ + "agents", + "dist", + "migrations", + "skills", + "templates", + "README.md" + ], "scripts": { "prebuild": "pnpm --filter @paperclipai/plugin-sdk ensure-build-deps", "build": "node ./esbuild.config.mjs", diff --git a/packages/plugins/plugin-llm-wiki/src/templates.ts b/packages/plugins/plugin-llm-wiki/src/templates.ts index c07bf6f6..3eb5284a 100644 --- a/packages/plugins/plugin-llm-wiki/src/templates.ts +++ b/packages/plugins/plugin-llm-wiki/src/templates.ts @@ -46,7 +46,7 @@ export const DEFAULT_AGENT_INSTRUCTIONS = DEFAULT_AGENT_INSTRUCTION_FILES["AGENT export const DEFAULT_IDEA = templateFile("IDEA.md"); export const DEFAULT_INDEX = templateFile("wiki/index.md"); export const DEFAULT_LOG = templateFile("wiki/log.md"); -export const DEFAULT_GITIGNORE = templateFile(".gitignore"); +export const DEFAULT_GITIGNORE = templateFile("gitignore.template"); export const QUERY_PROMPT = `Answer from the LLM Wiki using the installed wiki-query skill. diff --git a/packages/plugins/plugin-llm-wiki/templates/.gitignore b/packages/plugins/plugin-llm-wiki/templates/gitignore.template similarity index 100% rename from packages/plugins/plugin-llm-wiki/templates/.gitignore rename to packages/plugins/plugin-llm-wiki/templates/gitignore.template diff --git a/server/src/__tests__/plugin-database.test.ts b/server/src/__tests__/plugin-database.test.ts index 5ae677e5..d69fd34a 100644 --- a/server/src/__tests__/plugin-database.test.ts +++ b/server/src/__tests__/plugin-database.test.ts @@ -30,6 +30,7 @@ import { buildPluginWorkerEnv, pluginLoader } from "../services/plugin-loader.js const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; const multiMigrationPluginKey = "paperclip.dbfixture"; +const llmWikiPluginKey = "paperclipai.plugin-llm-wiki"; if (!embeddedPostgresSupport.supported) { console.warn( @@ -48,6 +49,63 @@ describe("plugin database SQL validation", () => { ).not.toThrow(); }); + it("allows qualified index creation and namespace-scoped migration backfills", () => { + expect(() => + validatePluginMigrationStatement( + "CREATE INDEX IF NOT EXISTS rows_issue_idx ON plugin_test.rows (issue_id)", + "plugin_test", + ) + ).not.toThrow(); + expect(() => + validatePluginMigrationStatement( + ` + WITH source_rows AS ( + SELECT id FROM plugin_test.rows + ) + INSERT INTO plugin_test.row_copies (id) + SELECT id FROM source_rows + ON CONFLICT (id) DO NOTHING + `, + "plugin_test", + ) + ).not.toThrow(); + expect(() => + validatePluginMigrationStatement( + ` + UPDATE plugin_test.rows r + SET copied_from_id = s.id + FROM plugin_test.source_rows s + WHERE s.id = r.id + `, + "plugin_test", + ) + ).not.toThrow(); + }); + + it("keeps migration backfill writes scoped to the plugin namespace", () => { + expect(() => + validatePluginMigrationStatement( + "CREATE TABLE rows (id uuid PRIMARY KEY, issue_id uuid REFERENCES public.issues(id))", + "plugin_test", + ["issues"], + ) + ).toThrow(/fully qualified/i); + expect(() => + validatePluginMigrationStatement( + "WITH source_rows AS (SELECT id FROM plugin_test.rows) INSERT INTO public.issues (id) SELECT id FROM source_rows", + "plugin_test", + ["issues"], + ) + ).toThrow(/public/i); + expect(() => + validatePluginMigrationStatement( + "UPDATE public.issues SET title = 'bad'", + "plugin_test", + ["issues"], + ) + ).toThrow(/public/i); + }); + it("rejects migrations that create public objects", () => { expect(() => validatePluginMigrationStatement( @@ -137,10 +195,11 @@ describeEmbeddedPostgres("plugin database namespaces", () => { }, 20_000); afterEach(async () => { - for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey]) { + for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey, llmWikiPluginKey]) { const namespace = derivePluginDatabaseNamespace(pluginKey); await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${namespace}" CASCADE`)); } + await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${derivePluginDatabaseNamespace(llmWikiPluginKey, "llm_wiki")}" CASCADE`)); await db.delete(pluginMigrations); await db.delete(pluginDatabaseNamespaces); await db.delete(plugins); @@ -164,6 +223,29 @@ describeEmbeddedPostgres("plugin database namespaces", () => { return packageRoot; } + function llmWikiManifest(): PaperclipPluginManifestV1 { + return { + id: llmWikiPluginKey, + apiVersion: 1, + version: "0.1.0", + displayName: "LLM Wiki", + description: "Local-file LLM Wiki plugin.", + author: "Paperclip", + categories: ["automation", "ui"], + capabilities: [ + "database.namespace.migrate", + "database.namespace.read", + "database.namespace.write", + ], + entrypoints: { worker: "./dist/worker.js" }, + database: { + namespaceSlug: "llm_wiki", + migrationsDir: "migrations", + coreReadTables: ["companies", "issues", "projects", "agents"], + }, + }; + } + async function createInstallablePluginPackage( pluginManifest: PaperclipPluginManifestV1, migrationSql: string, @@ -252,6 +334,61 @@ describeEmbeddedPostgres("plugin database namespaces", () => { expect(migrations).toHaveLength(2); }); + it("applies the bundled LLM Wiki migrations through the production validator", async () => { + const pluginManifest = llmWikiManifest(); + const repoRoot = path.basename(process.cwd()) === "server" ? path.resolve(process.cwd(), "..") : process.cwd(); + const packageRoot = path.join(repoRoot, "packages", "plugins", "plugin-llm-wiki"); + const namespace = derivePluginDatabaseNamespace(pluginManifest.id, pluginManifest.database?.namespaceSlug); + const pluginId = await installPluginRecord(pluginManifest); + + await pluginDatabaseService(db).applyMigrations(pluginId, pluginManifest, packageRoot); + + const migrations = await db + .select() + .from(pluginMigrations) + .where(and(eq(pluginMigrations.pluginId, pluginId), eq(pluginMigrations.status, "applied"))); + expect(migrations.map((migration) => migration.migrationKey)).toEqual([ + "001_llm_wiki.sql", + "002_paperclip_distillation.sql", + "003_spaces.sql", + ]); + + const constraintRows = Array.from( + await db.execute( + sql<{ table_name: string; conname: string; columns: string[] }>` + SELECT t.relname AS table_name, c.conname, array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[] AS columns + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality) ON true + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum + WHERE c.connamespace = ${namespace}::regnamespace AND c.contype = 'u' + GROUP BY t.relname, c.conname + ORDER BY t.relname, c.conname + `, + ) as Iterable<{ table_name: string; conname: string; columns: string[] }>, + ); + const constraints = constraintRows.map((row) => row.conname); + const uniqueColumnSets = new Set( + constraintRows.map((row) => `${row.table_name}:${row.columns.join(",")}`), + ); + expect(constraints).toEqual( + expect.arrayContaining([ + "wiki_pages_company_wiki_space_path_key", + "distillation_cursors_company_wiki_space_scope_key", + "distillation_work_items_company_wiki_space_idempotency_key", + "page_bindings_company_wiki_space_page_path_key", + ]), + ); + expect(constraints).not.toContain("wiki_pages_company_id_wiki_id_path_key"); + expect(constraints).not.toContain("paperclip_distillation_cursor_company_id_wiki_id_source_sco_key"); + expect(constraints).not.toContain("paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key"); + expect(constraints).not.toContain("paperclip_page_bindings_company_id_wiki_id_page_path_key"); + expect(uniqueColumnSets).not.toContain("wiki_pages:company_id,wiki_id,path"); + expect(uniqueColumnSets).not.toContain("paperclip_distillation_cursors:company_id,wiki_id,source_scope,scope_key,source_kind"); + expect(uniqueColumnSets).not.toContain("paperclip_distillation_work_items:company_id,wiki_id,idempotency_key"); + expect(uniqueColumnSets).not.toContain("paperclip_page_bindings:company_id,wiki_id,page_path"); + }); + it("applies migrations once and allows whitelisted core joins at runtime", async () => { const pluginManifest = manifest(); const namespace = derivePluginDatabaseNamespace(pluginManifest.id); diff --git a/server/src/services/plugin-database.ts b/server/src/services/plugin-database.ts index e6811702..d6b4cdda 100644 --- a/server/src/services/plugin-database.ts +++ b/server/src/services/plugin-database.ts @@ -19,6 +19,9 @@ const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const MAX_POSTGRES_IDENTIFIER_LENGTH = 63; type SqlRef = { schema: string; table: string; keyword: string }; +type QualifiedRefPattern = + | { pattern: RegExp; groups: "keyword-schema-table" } + | { pattern: RegExp; groups: "schema-table"; keyword: string }; export type PluginDatabaseRuntimeResult> = { rows?: T[]; @@ -123,14 +126,29 @@ function normaliseSql(input: string): string { function extractQualifiedRefs(statement: string): SqlRef[] { const refs: SqlRef[] = []; - const patterns = [ - /\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi, - /\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi, + const patterns: QualifiedRefPattern[] = [ + { + pattern: /\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi, + groups: "keyword-schema-table", + }, + { + pattern: /\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi, + groups: "keyword-schema-table", + }, + { + pattern: /\bcreate\s+(?:unique\s+)?index(?:\s+concurrently)?\s+(?:if\s+not\s+exists\s+)?"?[A-Za-z_][A-Za-z0-9_]*"?\s+on\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi, + groups: "schema-table", + keyword: "create index", + }, ]; - for (const pattern of patterns) { + for (const { pattern, ...mapping } of patterns) { for (const match of statement.matchAll(pattern)) { - refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! }); + if (mapping.groups === "keyword-schema-table") { + refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! }); + } else { + refs.push({ keyword: mapping.keyword, schema: match[1]!, table: match[2]! }); + } } } return refs; @@ -182,9 +200,16 @@ export function validatePluginMigrationStatement( throw new Error("Destructive plugin migrations are not allowed in Phase 1"); } - const ddlAllowed = /^(create|alter|comment)\b/.test(normalized); - if (!ddlAllowed) { - throw new Error("Plugin migrations may contain DDL statements only"); + if (/\bdelete\s+from\b/.test(normalized)) { + throw new Error("Plugin migrations cannot delete data"); + } + + const ddlOrBackfillAllowed = + /^(create|alter|comment)\b/.test(normalized) || + /^(insert\s+into|update)\b/.test(normalized) || + (normalized.startsWith("with ") && /\b(insert\s+into|update)\b/.test(normalized)); + if (!ddlOrBackfillAllowed) { + throw new Error("Plugin migrations may contain DDL or namespace-scoped backfill statements only"); } const refs = extractQualifiedRefs(statement); @@ -192,6 +217,21 @@ export function validatePluginMigrationStatement( throw new Error("Plugin migration objects must use fully qualified schema names"); } + const objectRefKeywords = new Set([ + "alter table", + "create index", + "create table", + "create view", + "drop table", + "into", + "truncate table", + "update", + ]); + const hasQualifiedObjectRef = refs.some((ref) => objectRefKeywords.has(ref.keyword)); + if (!hasQualifiedObjectRef && !normalized.startsWith("comment ")) { + throw new Error("Plugin migration objects must use fully qualified schema names"); + } + const allowedCoreReadTables = new Set(coreReadTables); for (const ref of refs) { if (ref.schema === namespace) continue;