Fix LLM Wiki package and migration validation (#6010)

## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - Plugins extend the control plane with optional capabilities such as LLM Wiki. > - LLM Wiki needs its package assets and plugin-owned database migrations to work when installed from the packaged plugin. > - The bundled spaces migration used validation-hostile dynamic SQL, and the packaged plugin could omit non-dist runtime assets. > - This pull request makes the LLM Wiki package include its required assets and cuts the spaces migration over to explicit, idempotent SQL that passes the production plugin database validator. > - The benefit is a simpler plugin install path that validates and applies the bundled LLM Wiki migrations without adding plugin-specific legacy handling to Paperclip core. ## What Changed - Added the LLM Wiki package asset allowlist so agents, migrations, skills, templates, dist output, and README are included when packaged. - Renamed the bootstrap `.gitignore` template to `gitignore.template` and updated the runtime lookup so package tooling does not drop the hidden template file. - Relaxed plugin migration validation to allow namespace-scoped `INSERT`/`UPDATE` backfills and `CREATE INDEX` statements while continuing to reject destructive or cross-namespace SQL. - Replaced the LLM Wiki spaces migration's dynamic constraint-drop DO block with explicit `DROP CONSTRAINT IF EXISTS` statements. - Replaced fragile regex-source dispatch in SQL reference extraction with explicit capture-group descriptors. - Added regression coverage that applies the bundled LLM Wiki migrations through the production validator and checks the expected constraints. ## Verification - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/plugin-database.test.ts --pool=forks --poolOptions.forks.isolate=true` - `pnpm --filter @paperclipai/plugin-llm-wiki build` - `git diff --check` - Confirmed `pnpm-lock.yaml` is not included in the branch diff. ## Risks - Low migration risk for current users: LLM Wiki spaces are new, so this intentionally cuts over the plugin migration instead of adding legacy handling in core. - Validator behavior is broader than before, but still requires fully qualified plugin namespace targets, blocks deletes/destructive DDL, and keeps public table access read-only and allowlisted. > Checked [`ROADMAP.md`](ROADMAP.md); this is a targeted plugin packaging/migration fix and does not duplicate planned core feature work. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 based coding agent, tool-enabled local repo access, reasoning mode managed by the Paperclip/Codex runtime. Exact context window was not surfaced in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-15 10:20:02 -05:00
parent dfcebf082b
commit eb38b226c2
6 changed files with 203 additions and 41 deletions
@@ -140,37 +140,14 @@ ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_runs ALTER COLUMN
 ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_source_snapshots ALTER COLUMN space_id SET NOT NULL;
 ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_page_bindings ALTER COLUMN space_id SET NOT NULL;

-DO $$
-DECLARE
-  target record;
-  constraint_name text;
-BEGIN
-  FOR target IN
-    SELECT * FROM (VALUES
-      ('wiki_pages', ARRAY['company_id', 'wiki_id', 'path']::text[]),
-      ('paperclip_distillation_cursors', ARRAY['company_id', 'wiki_id', 'source_scope', 'scope_key', 'source_kind']::text[]),
-      ('paperclip_distillation_work_items', ARRAY['company_id', 'wiki_id', 'idempotency_key']::text[]),
-      ('paperclip_page_bindings', ARRAY['company_id', 'wiki_id', 'page_path']::text[])
-    ) AS targets(table_name, column_names)
-  LOOP
-    FOR constraint_name IN
-      SELECT c.conname
-      FROM pg_constraint c
-      JOIN pg_class t ON t.oid = c.conrelid
-      JOIN pg_namespace n ON n.oid = t.relnamespace
-      WHERE n.nspname = 'plugin_llm_wiki_8f50da974f'
-        AND t.relname = target.table_name
-        AND c.contype = 'u'
-        AND (
-          SELECT array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[]
-          FROM unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality)
-          JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum
-        ) = target.column_names
-    LOOP
-      EXECUTE format('ALTER TABLE %I.%I DROP CONSTRAINT %I', 'plugin_llm_wiki_8f50da974f', target.table_name, constraint_name);
-    END LOOP;
-  END LOOP;
-END $$;
+ALTER TABLE plugin_llm_wiki_8f50da974f.wiki_pages
+  DROP CONSTRAINT IF EXISTS wiki_pages_company_id_wiki_id_path_key;
+ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_cursors
+  DROP CONSTRAINT IF EXISTS paperclip_distillation_cursor_company_id_wiki_id_source_sco_key;
+ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_distillation_work_items
+  DROP CONSTRAINT IF EXISTS paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key;
+ALTER TABLE plugin_llm_wiki_8f50da974f.paperclip_page_bindings
+  DROP CONSTRAINT IF EXISTS paperclip_page_bindings_company_id_wiki_id_page_path_key;

 ALTER TABLE plugin_llm_wiki_8f50da974f.wiki_pages
  DROP CONSTRAINT IF EXISTS wiki_pages_company_wiki_space_path_key;
@@ -4,6 +4,14 @@
  "type": "module",
  "private": true,
  "description": "Local-file LLM Wiki plugin for source ingestion, wiki browsing, query, lint, and maintenance workflows.",
+  "files": [
+    "agents",
+    "dist",
+    "migrations",
+    "skills",
+    "templates",
+    "README.md"
+  ],
  "scripts": {
    "prebuild": "pnpm --filter @paperclipai/plugin-sdk ensure-build-deps",
    "build": "node ./esbuild.config.mjs",
@@ -46,7 +46,7 @@ export const DEFAULT_AGENT_INSTRUCTIONS = DEFAULT_AGENT_INSTRUCTION_FILES["AGENT
 export const DEFAULT_IDEA = templateFile("IDEA.md");
 export const DEFAULT_INDEX = templateFile("wiki/index.md");
 export const DEFAULT_LOG = templateFile("wiki/log.md");
-export const DEFAULT_GITIGNORE = templateFile(".gitignore");
+export const DEFAULT_GITIGNORE = templateFile("gitignore.template");

 export const QUERY_PROMPT = `Answer from the LLM Wiki using the installed wiki-query skill.

@@ -30,6 +30,7 @@ import { buildPluginWorkerEnv, pluginLoader } from "../services/plugin-loader.js
 const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
 const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
 const multiMigrationPluginKey = "paperclip.dbfixture";
+const llmWikiPluginKey = "paperclipai.plugin-llm-wiki";

 if (!embeddedPostgresSupport.supported) {
  console.warn(
@@ -48,6 +49,63 @@ describe("plugin database SQL validation", () => {
    ).not.toThrow();
  });

+  it("allows qualified index creation and namespace-scoped migration backfills", () => {
+    expect(() =>
+      validatePluginMigrationStatement(
+        "CREATE INDEX IF NOT EXISTS rows_issue_idx ON plugin_test.rows (issue_id)",
+        "plugin_test",
+      )
+    ).not.toThrow();
+    expect(() =>
+      validatePluginMigrationStatement(
+        `
+        WITH source_rows AS (
+          SELECT id FROM plugin_test.rows
+        )
+        INSERT INTO plugin_test.row_copies (id)
+        SELECT id FROM source_rows
+        ON CONFLICT (id) DO NOTHING
+        `,
+        "plugin_test",
+      )
+    ).not.toThrow();
+    expect(() =>
+      validatePluginMigrationStatement(
+        `
+        UPDATE plugin_test.rows r
+        SET copied_from_id = s.id
+        FROM plugin_test.source_rows s
+        WHERE s.id = r.id
+        `,
+        "plugin_test",
+      )
+    ).not.toThrow();
+  });
+
+  it("keeps migration backfill writes scoped to the plugin namespace", () => {
+    expect(() =>
+      validatePluginMigrationStatement(
+        "CREATE TABLE rows (id uuid PRIMARY KEY, issue_id uuid REFERENCES public.issues(id))",
+        "plugin_test",
+        ["issues"],
+      )
+    ).toThrow(/fully qualified/i);
+    expect(() =>
+      validatePluginMigrationStatement(
+        "WITH source_rows AS (SELECT id FROM plugin_test.rows) INSERT INTO public.issues (id) SELECT id FROM source_rows",
+        "plugin_test",
+        ["issues"],
+      )
+    ).toThrow(/public/i);
+    expect(() =>
+      validatePluginMigrationStatement(
+        "UPDATE public.issues SET title = 'bad'",
+        "plugin_test",
+        ["issues"],
+      )
+    ).toThrow(/public/i);
+  });
+
  it("rejects migrations that create public objects", () => {
    expect(() =>
      validatePluginMigrationStatement(
@@ -137,10 +195,11 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
  }, 20_000);

  afterEach(async () => {
-    for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey]) {
+    for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey, llmWikiPluginKey]) {
      const namespace = derivePluginDatabaseNamespace(pluginKey);
      await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${namespace}" CASCADE`));
    }
+    await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${derivePluginDatabaseNamespace(llmWikiPluginKey, "llm_wiki")}" CASCADE`));
    await db.delete(pluginMigrations);
    await db.delete(pluginDatabaseNamespaces);
    await db.delete(plugins);
@@ -164,6 +223,29 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
    return packageRoot;
  }

+  function llmWikiManifest(): PaperclipPluginManifestV1 {
+    return {
+      id: llmWikiPluginKey,
+      apiVersion: 1,
+      version: "0.1.0",
+      displayName: "LLM Wiki",
+      description: "Local-file LLM Wiki plugin.",
+      author: "Paperclip",
+      categories: ["automation", "ui"],
+      capabilities: [
+        "database.namespace.migrate",
+        "database.namespace.read",
+        "database.namespace.write",
+      ],
+      entrypoints: { worker: "./dist/worker.js" },
+      database: {
+        namespaceSlug: "llm_wiki",
+        migrationsDir: "migrations",
+        coreReadTables: ["companies", "issues", "projects", "agents"],
+      },
+    };
+  }
+
  async function createInstallablePluginPackage(
    pluginManifest: PaperclipPluginManifestV1,
    migrationSql: string,
@@ -252,6 +334,61 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
    expect(migrations).toHaveLength(2);
  });

+  it("applies the bundled LLM Wiki migrations through the production validator", async () => {
+    const pluginManifest = llmWikiManifest();
+    const repoRoot = path.basename(process.cwd()) === "server" ? path.resolve(process.cwd(), "..") : process.cwd();
+    const packageRoot = path.join(repoRoot, "packages", "plugins", "plugin-llm-wiki");
+    const namespace = derivePluginDatabaseNamespace(pluginManifest.id, pluginManifest.database?.namespaceSlug);
+    const pluginId = await installPluginRecord(pluginManifest);
+
+    await pluginDatabaseService(db).applyMigrations(pluginId, pluginManifest, packageRoot);
+
+    const migrations = await db
+      .select()
+      .from(pluginMigrations)
+      .where(and(eq(pluginMigrations.pluginId, pluginId), eq(pluginMigrations.status, "applied")));
+    expect(migrations.map((migration) => migration.migrationKey)).toEqual([
+      "001_llm_wiki.sql",
+      "002_paperclip_distillation.sql",
+      "003_spaces.sql",
+    ]);
+
+    const constraintRows = Array.from(
+      await db.execute(
+        sql<{ table_name: string; conname: string; columns: string[] }>`
+          SELECT t.relname AS table_name, c.conname, array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[] AS columns
+          FROM pg_constraint c
+          JOIN pg_class t ON t.oid = c.conrelid
+          JOIN unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality) ON true
+          JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum
+          WHERE c.connamespace = ${namespace}::regnamespace AND c.contype = 'u'
+          GROUP BY t.relname, c.conname
+          ORDER BY t.relname, c.conname
+        `,
+      ) as Iterable<{ table_name: string; conname: string; columns: string[] }>,
+    );
+    const constraints = constraintRows.map((row) => row.conname);
+    const uniqueColumnSets = new Set(
+      constraintRows.map((row) => `${row.table_name}:${row.columns.join(",")}`),
+    );
+    expect(constraints).toEqual(
+      expect.arrayContaining([
+        "wiki_pages_company_wiki_space_path_key",
+        "distillation_cursors_company_wiki_space_scope_key",
+        "distillation_work_items_company_wiki_space_idempotency_key",
+        "page_bindings_company_wiki_space_page_path_key",
+      ]),
+    );
+    expect(constraints).not.toContain("wiki_pages_company_id_wiki_id_path_key");
+    expect(constraints).not.toContain("paperclip_distillation_cursor_company_id_wiki_id_source_sco_key");
+    expect(constraints).not.toContain("paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key");
+    expect(constraints).not.toContain("paperclip_page_bindings_company_id_wiki_id_page_path_key");
+    expect(uniqueColumnSets).not.toContain("wiki_pages:company_id,wiki_id,path");
+    expect(uniqueColumnSets).not.toContain("paperclip_distillation_cursors:company_id,wiki_id,source_scope,scope_key,source_kind");
+    expect(uniqueColumnSets).not.toContain("paperclip_distillation_work_items:company_id,wiki_id,idempotency_key");
+    expect(uniqueColumnSets).not.toContain("paperclip_page_bindings:company_id,wiki_id,page_path");
+  });
+
  it("applies migrations once and allows whitelisted core joins at runtime", async () => {
    const pluginManifest = manifest();
    const namespace = derivePluginDatabaseNamespace(pluginManifest.id);
@@ -19,6 +19,9 @@ const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
 const MAX_POSTGRES_IDENTIFIER_LENGTH = 63;

 type SqlRef = { schema: string; table: string; keyword: string };
+type QualifiedRefPattern =
+  | { pattern: RegExp; groups: "keyword-schema-table" }
+  | { pattern: RegExp; groups: "schema-table"; keyword: string };

 export type PluginDatabaseRuntimeResult<T = Record<string, unknown>> = {
  rows?: T[];
@@ -123,14 +126,29 @@ function normaliseSql(input: string): string {

 function extractQualifiedRefs(statement: string): SqlRef[] {
  const refs: SqlRef[] = [];
-  const patterns = [
-    /\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
-    /\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
+  const patterns: QualifiedRefPattern[] = [
+    {
+      pattern: /\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
+      groups: "keyword-schema-table",
+    },
+    {
+      pattern: /\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
+      groups: "keyword-schema-table",
+    },
+    {
+      pattern: /\bcreate\s+(?:unique\s+)?index(?:\s+concurrently)?\s+(?:if\s+not\s+exists\s+)?"?[A-Za-z_][A-Za-z0-9_]*"?\s+on\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
+      groups: "schema-table",
+      keyword: "create index",
+    },
  ];

-  for (const pattern of patterns) {
+  for (const { pattern, ...mapping } of patterns) {
    for (const match of statement.matchAll(pattern)) {
-      refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! });
+      if (mapping.groups === "keyword-schema-table") {
+        refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! });
+      } else {
+        refs.push({ keyword: mapping.keyword, schema: match[1]!, table: match[2]! });
+      }
    }
  }
  return refs;
@@ -182,9 +200,16 @@ export function validatePluginMigrationStatement(
    throw new Error("Destructive plugin migrations are not allowed in Phase 1");
  }

-  const ddlAllowed = /^(create|alter|comment)\b/.test(normalized);
-  if (!ddlAllowed) {
-    throw new Error("Plugin migrations may contain DDL statements only");
+  if (/\bdelete\s+from\b/.test(normalized)) {
+    throw new Error("Plugin migrations cannot delete data");
+  }
+
+  const ddlOrBackfillAllowed =
+    /^(create|alter|comment)\b/.test(normalized) ||
+    /^(insert\s+into|update)\b/.test(normalized) ||
+    (normalized.startsWith("with ") && /\b(insert\s+into|update)\b/.test(normalized));
+  if (!ddlOrBackfillAllowed) {
+    throw new Error("Plugin migrations may contain DDL or namespace-scoped backfill statements only");
  }

  const refs = extractQualifiedRefs(statement);
@@ -192,6 +217,21 @@ export function validatePluginMigrationStatement(
    throw new Error("Plugin migration objects must use fully qualified schema names");
  }

+  const objectRefKeywords = new Set([
+    "alter table",
+    "create index",
+    "create table",
+    "create view",
+    "drop table",
+    "into",
+    "truncate table",
+    "update",
+  ]);
+  const hasQualifiedObjectRef = refs.some((ref) => objectRefKeywords.has(ref.keyword));
+  if (!hasQualifiedObjectRef && !normalized.startsWith("comment ")) {
+    throw new Error("Plugin migration objects must use fully qualified schema names");
+  }
+
  const allowedCoreReadTables = new Set(coreReadTables);
  for (const ref of refs) {
    if (ref.schema === namespace) continue;