Fix LLM Wiki package and migration validation (#6010)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - Plugins extend the control plane with optional capabilities such as LLM Wiki. > - LLM Wiki needs its package assets and plugin-owned database migrations to work when installed from the packaged plugin. > - The bundled spaces migration used validation-hostile dynamic SQL, and the packaged plugin could omit non-dist runtime assets. > - This pull request makes the LLM Wiki package include its required assets and cuts the spaces migration over to explicit, idempotent SQL that passes the production plugin database validator. > - The benefit is a simpler plugin install path that validates and applies the bundled LLM Wiki migrations without adding plugin-specific legacy handling to Paperclip core. ## What Changed - Added the LLM Wiki package asset allowlist so agents, migrations, skills, templates, dist output, and README are included when packaged. - Renamed the bootstrap `.gitignore` template to `gitignore.template` and updated the runtime lookup so package tooling does not drop the hidden template file. - Relaxed plugin migration validation to allow namespace-scoped `INSERT`/`UPDATE` backfills and `CREATE INDEX` statements while continuing to reject destructive or cross-namespace SQL. - Replaced the LLM Wiki spaces migration's dynamic constraint-drop DO block with explicit `DROP CONSTRAINT IF EXISTS` statements. - Replaced fragile regex-source dispatch in SQL reference extraction with explicit capture-group descriptors. - Added regression coverage that applies the bundled LLM Wiki migrations through the production validator and checks the expected constraints. ## Verification - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/plugin-database.test.ts --pool=forks --poolOptions.forks.isolate=true` - `pnpm --filter @paperclipai/plugin-llm-wiki build` - `git diff --check` - Confirmed `pnpm-lock.yaml` is not included in the branch diff. ## Risks - Low migration risk for current users: LLM Wiki spaces are new, so this intentionally cuts over the plugin migration instead of adding legacy handling in core. - Validator behavior is broader than before, but still requires fully qualified plugin namespace targets, blocks deletes/destructive DDL, and keeps public table access read-only and allowlisted. > Checked [`ROADMAP.md`](ROADMAP.md); this is a targeted plugin packaging/migration fix and does not duplicate planned core feature work. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 based coding agent, tool-enabled local repo access, reasoning mode managed by the Paperclip/Codex runtime. Exact context window was not surfaced in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
@@ -30,6 +30,7 @@ import { buildPluginWorkerEnv, pluginLoader } from "../services/plugin-loader.js
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
const multiMigrationPluginKey = "paperclip.dbfixture";
|
||||
const llmWikiPluginKey = "paperclipai.plugin-llm-wiki";
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
@@ -48,6 +49,63 @@ describe("plugin database SQL validation", () => {
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("allows qualified index creation and namespace-scoped migration backfills", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"CREATE INDEX IF NOT EXISTS rows_issue_idx ON plugin_test.rows (issue_id)",
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
`
|
||||
WITH source_rows AS (
|
||||
SELECT id FROM plugin_test.rows
|
||||
)
|
||||
INSERT INTO plugin_test.row_copies (id)
|
||||
SELECT id FROM source_rows
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
`,
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
`
|
||||
UPDATE plugin_test.rows r
|
||||
SET copied_from_id = s.id
|
||||
FROM plugin_test.source_rows s
|
||||
WHERE s.id = r.id
|
||||
`,
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("keeps migration backfill writes scoped to the plugin namespace", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"CREATE TABLE rows (id uuid PRIMARY KEY, issue_id uuid REFERENCES public.issues(id))",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/fully qualified/i);
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"WITH source_rows AS (SELECT id FROM plugin_test.rows) INSERT INTO public.issues (id) SELECT id FROM source_rows",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/public/i);
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"UPDATE public.issues SET title = 'bad'",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/public/i);
|
||||
});
|
||||
|
||||
it("rejects migrations that create public objects", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
@@ -137,10 +195,11 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey]) {
|
||||
for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey, llmWikiPluginKey]) {
|
||||
const namespace = derivePluginDatabaseNamespace(pluginKey);
|
||||
await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${namespace}" CASCADE`));
|
||||
}
|
||||
await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${derivePluginDatabaseNamespace(llmWikiPluginKey, "llm_wiki")}" CASCADE`));
|
||||
await db.delete(pluginMigrations);
|
||||
await db.delete(pluginDatabaseNamespaces);
|
||||
await db.delete(plugins);
|
||||
@@ -164,6 +223,29 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
||||
return packageRoot;
|
||||
}
|
||||
|
||||
function llmWikiManifest(): PaperclipPluginManifestV1 {
|
||||
return {
|
||||
id: llmWikiPluginKey,
|
||||
apiVersion: 1,
|
||||
version: "0.1.0",
|
||||
displayName: "LLM Wiki",
|
||||
description: "Local-file LLM Wiki plugin.",
|
||||
author: "Paperclip",
|
||||
categories: ["automation", "ui"],
|
||||
capabilities: [
|
||||
"database.namespace.migrate",
|
||||
"database.namespace.read",
|
||||
"database.namespace.write",
|
||||
],
|
||||
entrypoints: { worker: "./dist/worker.js" },
|
||||
database: {
|
||||
namespaceSlug: "llm_wiki",
|
||||
migrationsDir: "migrations",
|
||||
coreReadTables: ["companies", "issues", "projects", "agents"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createInstallablePluginPackage(
|
||||
pluginManifest: PaperclipPluginManifestV1,
|
||||
migrationSql: string,
|
||||
@@ -252,6 +334,61 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
||||
expect(migrations).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("applies the bundled LLM Wiki migrations through the production validator", async () => {
|
||||
const pluginManifest = llmWikiManifest();
|
||||
const repoRoot = path.basename(process.cwd()) === "server" ? path.resolve(process.cwd(), "..") : process.cwd();
|
||||
const packageRoot = path.join(repoRoot, "packages", "plugins", "plugin-llm-wiki");
|
||||
const namespace = derivePluginDatabaseNamespace(pluginManifest.id, pluginManifest.database?.namespaceSlug);
|
||||
const pluginId = await installPluginRecord(pluginManifest);
|
||||
|
||||
await pluginDatabaseService(db).applyMigrations(pluginId, pluginManifest, packageRoot);
|
||||
|
||||
const migrations = await db
|
||||
.select()
|
||||
.from(pluginMigrations)
|
||||
.where(and(eq(pluginMigrations.pluginId, pluginId), eq(pluginMigrations.status, "applied")));
|
||||
expect(migrations.map((migration) => migration.migrationKey)).toEqual([
|
||||
"001_llm_wiki.sql",
|
||||
"002_paperclip_distillation.sql",
|
||||
"003_spaces.sql",
|
||||
]);
|
||||
|
||||
const constraintRows = Array.from(
|
||||
await db.execute(
|
||||
sql<{ table_name: string; conname: string; columns: string[] }>`
|
||||
SELECT t.relname AS table_name, c.conname, array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[] AS columns
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class t ON t.oid = c.conrelid
|
||||
JOIN unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality) ON true
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum
|
||||
WHERE c.connamespace = ${namespace}::regnamespace AND c.contype = 'u'
|
||||
GROUP BY t.relname, c.conname
|
||||
ORDER BY t.relname, c.conname
|
||||
`,
|
||||
) as Iterable<{ table_name: string; conname: string; columns: string[] }>,
|
||||
);
|
||||
const constraints = constraintRows.map((row) => row.conname);
|
||||
const uniqueColumnSets = new Set(
|
||||
constraintRows.map((row) => `${row.table_name}:${row.columns.join(",")}`),
|
||||
);
|
||||
expect(constraints).toEqual(
|
||||
expect.arrayContaining([
|
||||
"wiki_pages_company_wiki_space_path_key",
|
||||
"distillation_cursors_company_wiki_space_scope_key",
|
||||
"distillation_work_items_company_wiki_space_idempotency_key",
|
||||
"page_bindings_company_wiki_space_page_path_key",
|
||||
]),
|
||||
);
|
||||
expect(constraints).not.toContain("wiki_pages_company_id_wiki_id_path_key");
|
||||
expect(constraints).not.toContain("paperclip_distillation_cursor_company_id_wiki_id_source_sco_key");
|
||||
expect(constraints).not.toContain("paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key");
|
||||
expect(constraints).not.toContain("paperclip_page_bindings_company_id_wiki_id_page_path_key");
|
||||
expect(uniqueColumnSets).not.toContain("wiki_pages:company_id,wiki_id,path");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_distillation_cursors:company_id,wiki_id,source_scope,scope_key,source_kind");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_distillation_work_items:company_id,wiki_id,idempotency_key");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_page_bindings:company_id,wiki_id,page_path");
|
||||
});
|
||||
|
||||
it("applies migrations once and allows whitelisted core joins at runtime", async () => {
|
||||
const pluginManifest = manifest();
|
||||
const namespace = derivePluginDatabaseNamespace(pluginManifest.id);
|
||||
|
||||
@@ -19,6 +19,9 @@ const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
const MAX_POSTGRES_IDENTIFIER_LENGTH = 63;
|
||||
|
||||
type SqlRef = { schema: string; table: string; keyword: string };
|
||||
type QualifiedRefPattern =
|
||||
| { pattern: RegExp; groups: "keyword-schema-table" }
|
||||
| { pattern: RegExp; groups: "schema-table"; keyword: string };
|
||||
|
||||
export type PluginDatabaseRuntimeResult<T = Record<string, unknown>> = {
|
||||
rows?: T[];
|
||||
@@ -123,14 +126,29 @@ function normaliseSql(input: string): string {
|
||||
|
||||
function extractQualifiedRefs(statement: string): SqlRef[] {
|
||||
const refs: SqlRef[] = [];
|
||||
const patterns = [
|
||||
/\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
|
||||
/\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
|
||||
const patterns: QualifiedRefPattern[] = [
|
||||
{
|
||||
pattern: /\b(from|join|references|into|update)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
|
||||
groups: "keyword-schema-table",
|
||||
},
|
||||
{
|
||||
pattern: /\b(alter\s+table|create\s+table|create\s+view|drop\s+table|truncate\s+table)\s+(?:if\s+(?:not\s+)?exists\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
|
||||
groups: "keyword-schema-table",
|
||||
},
|
||||
{
|
||||
pattern: /\bcreate\s+(?:unique\s+)?index(?:\s+concurrently)?\s+(?:if\s+not\s+exists\s+)?"?[A-Za-z_][A-Za-z0-9_]*"?\s+on\s+"?([A-Za-z_][A-Za-z0-9_]*)"?\."?([A-Za-z_][A-Za-z0-9_]*)"?/gi,
|
||||
groups: "schema-table",
|
||||
keyword: "create index",
|
||||
},
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
for (const { pattern, ...mapping } of patterns) {
|
||||
for (const match of statement.matchAll(pattern)) {
|
||||
refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! });
|
||||
if (mapping.groups === "keyword-schema-table") {
|
||||
refs.push({ keyword: match[1]!.toLowerCase(), schema: match[2]!, table: match[3]! });
|
||||
} else {
|
||||
refs.push({ keyword: mapping.keyword, schema: match[1]!, table: match[2]! });
|
||||
}
|
||||
}
|
||||
}
|
||||
return refs;
|
||||
@@ -182,9 +200,16 @@ export function validatePluginMigrationStatement(
|
||||
throw new Error("Destructive plugin migrations are not allowed in Phase 1");
|
||||
}
|
||||
|
||||
const ddlAllowed = /^(create|alter|comment)\b/.test(normalized);
|
||||
if (!ddlAllowed) {
|
||||
throw new Error("Plugin migrations may contain DDL statements only");
|
||||
if (/\bdelete\s+from\b/.test(normalized)) {
|
||||
throw new Error("Plugin migrations cannot delete data");
|
||||
}
|
||||
|
||||
const ddlOrBackfillAllowed =
|
||||
/^(create|alter|comment)\b/.test(normalized) ||
|
||||
/^(insert\s+into|update)\b/.test(normalized) ||
|
||||
(normalized.startsWith("with ") && /\b(insert\s+into|update)\b/.test(normalized));
|
||||
if (!ddlOrBackfillAllowed) {
|
||||
throw new Error("Plugin migrations may contain DDL or namespace-scoped backfill statements only");
|
||||
}
|
||||
|
||||
const refs = extractQualifiedRefs(statement);
|
||||
@@ -192,6 +217,21 @@ export function validatePluginMigrationStatement(
|
||||
throw new Error("Plugin migration objects must use fully qualified schema names");
|
||||
}
|
||||
|
||||
const objectRefKeywords = new Set([
|
||||
"alter table",
|
||||
"create index",
|
||||
"create table",
|
||||
"create view",
|
||||
"drop table",
|
||||
"into",
|
||||
"truncate table",
|
||||
"update",
|
||||
]);
|
||||
const hasQualifiedObjectRef = refs.some((ref) => objectRefKeywords.has(ref.keyword));
|
||||
if (!hasQualifiedObjectRef && !normalized.startsWith("comment ")) {
|
||||
throw new Error("Plugin migration objects must use fully qualified schema names");
|
||||
}
|
||||
|
||||
const allowedCoreReadTables = new Set(coreReadTables);
|
||||
for (const ref of refs) {
|
||||
if (ref.schema === namespace) continue;
|
||||
|
||||
Reference in New Issue
Block a user