forked from farhoodlabs/paperclip
0096b56a1c
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - The plugin system needs host contracts and runtime support before large plugins can integrate cleanly. > - The source branch mixed the LLM Wiki package with supporting host/runtime work, managed plugin skills, root-level storage spaces, and a bookmarks reference plugin. > - [PAP-9173](/PAP/issues/PAP-9173) asked for the current branch to be split by file boundary: plugin package separately from everything else. > - [PAP-9188](/PAP/issues/PAP-9188) clarified that LLM Wiki may have plugin-local spaces, but Paperclip core should not reorganize top-level local storage into spaces. > - Follow-up review clarified that the bookmarks example should not ship in this PR either. > - This pull request contains the non-`packages/plugins/plugin-llm-wiki/` host/runtime work, keeps runtime state under the selected Paperclip instance root, and no longer includes the bookmarks example. ## What Changed - Added/updated plugin host contracts, SDK types, worker RPC plumbing, managed plugin skill support, and related server tests. - Removed the bookmarks example plugin package and its bundled-example/workspace references. - Removed the root-level local spaces CLI/migration surface and restored instance-root runtime defaults for config, db, logs, storage, secrets, workspaces, projects, and adapter homes. - Replaced shared root `space-paths` helpers with `home-paths` helpers for core runtime storage. - Tightened stranded recovery unique-conflict detection so concurrent recovery scans reuse the raced recovery issue when Postgres errors are wrapped. - Kept `packages/plugins/plugin-llm-wiki/` out of this PR diff; plugin-local spaces remain in the stacked plugin-only PR. ## Verification - `pnpm exec vitest run cli/src/__tests__/data-dir.test.ts cli/src/__tests__/home-paths.test.ts cli/src/__tests__/onboard.test.ts packages/shared/src/home-paths.test.ts packages/db/src/runtime-config.test.ts server/src/__tests__/agent-instructions-service.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts` - `pnpm exec vitest run packages/db/src/runtime-config.test.ts` - `pnpm exec vitest run server/src/__tests__/plugin-routes-authz.test.ts` - `pnpm --filter @paperclipai/server typecheck` - `pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "reuses the raced stranded recovery issue"` skipped locally because embedded Postgres did not initialize on this macOS temp host; the code path was typechecked and is covered by Linux CI. - Boundary check: no core references remain for `PAPERCLIP_SPACE_ID`, `spaces migrate-default`, `@paperclipai/shared/space-paths`, `registerSpacesCommands`, or the removed bookmarks example. - Previous PR head `4f23e034` had green GitHub checks: `verify`, all four serialized server shards, `e2e`, `Canary Dry Run`, `policy`, Snyk, and `Greptile Review`. Current head `582f466d` is re-running checks after the bookmarks deletion. ## Risks - Plugin host changes touch shared runtime paths, so regressions would most likely appear in adapter startup, plugin loading, or local dev path defaults. - Removing the bookmarks example also removes one demonstration of plugin database namespaces plus local-folder persistence; remaining plugin examples still cover bundled example discovery and plugin host flows. - The plugin package itself is intentionally deferred to the stacked plugin-only PR, where LLM Wiki plugin-local spaces live. - Existing installs that tested the transient root-level spaces CLI should stop using it; this PR intentionally removes that unsupported migration surface before merge. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via Codex CLI, tool use and local code execution enabled; context window not exposed. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass, except where noted above for host-specific embedded Postgres initialization - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Stacked follow-up: PR #5592 contains only `packages/plugins/plugin-llm-wiki/` and targets this branch. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
360 lines
11 KiB
TypeScript
360 lines
11 KiB
TypeScript
import { and, eq } from "drizzle-orm";
|
|
import type { Db } from "@paperclipai/db";
|
|
import {
|
|
pluginManagedResources,
|
|
} from "@paperclipai/db";
|
|
import { normalizeAgentUrlKey } from "@paperclipai/shared";
|
|
import type {
|
|
CompanySkill,
|
|
PaperclipPluginManifestV1,
|
|
PluginManagedSkillDeclaration,
|
|
PluginManagedSkillResolution,
|
|
} from "@paperclipai/shared";
|
|
import { notFound } from "../errors.js";
|
|
import { logActivity } from "./activity-log.js";
|
|
import { companySkillService } from "./company-skills.js";
|
|
|
|
const MANAGED_SKILL_RESOURCE_KIND = "skill";
|
|
|
|
interface PluginManagedSkillServiceOptions {
|
|
pluginId: string;
|
|
pluginKey: string;
|
|
manifest?: PaperclipPluginManifestV1 | null;
|
|
}
|
|
|
|
function pluginKeySlug(pluginKey: string) {
|
|
return normalizeAgentUrlKey(pluginKey) ?? "plugin";
|
|
}
|
|
|
|
function canonicalSkillKey(pluginKey: string, skillKey: string) {
|
|
return `plugin/${pluginKeySlug(pluginKey)}/${skillKey}`;
|
|
}
|
|
|
|
function yamlString(value: string) {
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function buildDefaultMarkdown(
|
|
pluginKey: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
) {
|
|
const description = declaration.description?.trim() || `${declaration.displayName} plugin skill.`;
|
|
return [
|
|
"---",
|
|
`name: ${yamlString(declaration.displayName)}`,
|
|
`description: ${yamlString(description)}`,
|
|
`key: ${yamlString(canonicalSkillKey(pluginKey, declaration.skillKey))}`,
|
|
"---",
|
|
"",
|
|
`# ${declaration.displayName}`,
|
|
"",
|
|
description,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function withManagedSkillKey(markdown: string, canonicalKey: string) {
|
|
const keyLine = `key: ${yamlString(canonicalKey)}`;
|
|
const normalized = markdown.replace(/\r\n/g, "\n");
|
|
const frontmatter = /^---\n([\s\S]*?)\n---(\n?)/.exec(normalized);
|
|
if (!frontmatter) {
|
|
return [
|
|
"---",
|
|
keyLine,
|
|
"---",
|
|
"",
|
|
normalized,
|
|
].join("\n");
|
|
}
|
|
|
|
const currentBody = frontmatter[1] ?? "";
|
|
const nextBody = /^key\s*:/m.test(currentBody)
|
|
? currentBody.replace(/^key\s*:.*$/m, keyLine)
|
|
: [currentBody, keyLine].filter(Boolean).join("\n");
|
|
return `---\n${nextBody}\n---${frontmatter[2] ?? ""}${normalized.slice(frontmatter[0].length)}`;
|
|
}
|
|
|
|
function buildPackageFiles(
|
|
pluginKey: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
) {
|
|
const root = declaration.skillKey;
|
|
const canonicalKey = canonicalSkillKey(pluginKey, declaration.skillKey);
|
|
const files: Record<string, string> = {
|
|
[`${root}/SKILL.md`]: declaration.markdown?.trim()
|
|
? withManagedSkillKey(declaration.markdown, canonicalKey)
|
|
: buildDefaultMarkdown(pluginKey, declaration),
|
|
};
|
|
for (const file of declaration.files ?? []) {
|
|
files[`${root}/${file.path}`] = file.content;
|
|
}
|
|
return files;
|
|
}
|
|
|
|
function buildDeclaredSkillFiles(
|
|
pluginKey: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
) {
|
|
const packageFiles = buildPackageFiles(pluginKey, declaration);
|
|
const root = declaration.skillKey;
|
|
const prefix = `${root}/`;
|
|
const files: Record<string, string> = {};
|
|
for (const [filePath, content] of Object.entries(packageFiles)) {
|
|
files[filePath.startsWith(prefix) ? filePath.slice(prefix.length) : filePath] = content;
|
|
}
|
|
return files;
|
|
}
|
|
|
|
function buildSkillDefaults(
|
|
pluginKey: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
) {
|
|
return {
|
|
skillKey: declaration.skillKey,
|
|
displayName: declaration.displayName,
|
|
slug: declaration.slug ?? declaration.skillKey,
|
|
description: declaration.description ?? null,
|
|
canonicalKey: canonicalSkillKey(pluginKey, declaration.skillKey),
|
|
files: [
|
|
"SKILL.md",
|
|
...(declaration.files ?? []).map((file) => file.path),
|
|
],
|
|
};
|
|
}
|
|
|
|
function stableJson(value: unknown): string {
|
|
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
|
if (value && typeof value === "object") {
|
|
return `{${Object.entries(value as Record<string, unknown>)
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`)
|
|
.join(",")}}`;
|
|
}
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function resolution(
|
|
pluginKey: string,
|
|
companyId: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
skill: CompanySkill | null,
|
|
status: PluginManagedSkillResolution["status"],
|
|
defaultDrift: PluginManagedSkillResolution["defaultDrift"] = null,
|
|
): PluginManagedSkillResolution {
|
|
return {
|
|
pluginKey,
|
|
resourceKind: "skill",
|
|
resourceKey: declaration.skillKey,
|
|
companyId,
|
|
skillId: skill?.id ?? null,
|
|
skill,
|
|
status,
|
|
defaultDrift,
|
|
};
|
|
}
|
|
|
|
export function pluginManagedSkillService(
|
|
db: Db,
|
|
options: PluginManagedSkillServiceOptions,
|
|
) {
|
|
const skills = companySkillService(db);
|
|
|
|
function declarationFor(skillKey: string) {
|
|
const declaration = options.manifest?.skills?.find((skill) => skill.skillKey === skillKey);
|
|
if (!declaration) {
|
|
throw notFound(`Managed skill declaration not found: ${skillKey}`);
|
|
}
|
|
return declaration;
|
|
}
|
|
|
|
async function getBinding(companyId: string, skillKey: string) {
|
|
return db
|
|
.select()
|
|
.from(pluginManagedResources)
|
|
.where(and(
|
|
eq(pluginManagedResources.companyId, companyId),
|
|
eq(pluginManagedResources.pluginId, options.pluginId),
|
|
eq(pluginManagedResources.resourceKind, MANAGED_SKILL_RESOURCE_KIND),
|
|
eq(pluginManagedResources.resourceKey, skillKey),
|
|
))
|
|
.then((rows) => rows[0] ?? null);
|
|
}
|
|
|
|
async function upsertBinding(
|
|
companyId: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
skillId: string,
|
|
) {
|
|
const defaultsJson = buildSkillDefaults(options.pluginKey, declaration);
|
|
const existing = await getBinding(companyId, declaration.skillKey);
|
|
if (existing) {
|
|
if (
|
|
existing.resourceId === skillId &&
|
|
stableJson(existing.defaultsJson) === stableJson(defaultsJson)
|
|
) {
|
|
return existing;
|
|
}
|
|
return db
|
|
.update(pluginManagedResources)
|
|
.set({
|
|
resourceId: skillId,
|
|
defaultsJson,
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(pluginManagedResources.id, existing.id))
|
|
.returning()
|
|
.then((rows) => rows[0]);
|
|
}
|
|
return db
|
|
.insert(pluginManagedResources)
|
|
.values({
|
|
companyId,
|
|
pluginId: options.pluginId,
|
|
pluginKey: options.pluginKey,
|
|
resourceKind: MANAGED_SKILL_RESOURCE_KIND,
|
|
resourceKey: declaration.skillKey,
|
|
resourceId: skillId,
|
|
defaultsJson,
|
|
})
|
|
.returning()
|
|
.then((rows) => rows[0]);
|
|
}
|
|
|
|
async function getSkill(companyId: string, skillId: string) {
|
|
return skills.getById(companyId, skillId);
|
|
}
|
|
|
|
async function managedSkillDefaultDrift(
|
|
companyId: string,
|
|
skill: CompanySkill | null,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
): Promise<PluginManagedSkillResolution["defaultDrift"]> {
|
|
if (!skill) return null;
|
|
const declaredFiles = buildDeclaredSkillFiles(options.pluginKey, declaration);
|
|
const currentFiles: Record<string, string | null> = {};
|
|
const paths = new Set([
|
|
...Object.keys(declaredFiles),
|
|
...skill.fileInventory.map((entry) => entry.path),
|
|
]);
|
|
|
|
for (const filePath of paths) {
|
|
if (filePath === "SKILL.md") {
|
|
currentFiles[filePath] = skill.markdown;
|
|
continue;
|
|
}
|
|
try {
|
|
currentFiles[filePath] = (await skills.readFile(companyId, skill.id, filePath))?.content ?? null;
|
|
} catch {
|
|
currentFiles[filePath] = null;
|
|
}
|
|
}
|
|
|
|
const changedFiles = [...paths]
|
|
.filter((filePath) => (currentFiles[filePath] ?? null) !== (declaredFiles[filePath] ?? null))
|
|
.sort((left, right) => left.localeCompare(right));
|
|
return changedFiles.length > 0 ? { changedFiles } : null;
|
|
}
|
|
|
|
async function resolvedSkill(
|
|
companyId: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
skill: CompanySkill | null,
|
|
status: PluginManagedSkillResolution["status"],
|
|
) {
|
|
return resolution(
|
|
options.pluginKey,
|
|
companyId,
|
|
declaration,
|
|
skill,
|
|
status,
|
|
await managedSkillDefaultDrift(companyId, skill, declaration),
|
|
);
|
|
}
|
|
|
|
async function importDeclaredSkill(
|
|
companyId: string,
|
|
declaration: PluginManagedSkillDeclaration,
|
|
mode: "reconcile" | "reset",
|
|
) {
|
|
const beforeByKey = mode === "reconcile"
|
|
? await skills.getByKey(companyId, canonicalSkillKey(options.pluginKey, declaration.skillKey))
|
|
: null;
|
|
if (beforeByKey) {
|
|
await upsertBinding(companyId, declaration, beforeByKey.id);
|
|
return { skill: beforeByKey, status: "relinked" as const };
|
|
}
|
|
const results = await skills.importPackageFiles(
|
|
companyId,
|
|
buildPackageFiles(options.pluginKey, declaration),
|
|
{ onConflict: "replace" },
|
|
);
|
|
const imported = results.find((result) =>
|
|
result.skill.key === canonicalSkillKey(options.pluginKey, declaration.skillKey)
|
|
|| result.originalSlug === (declaration.slug ?? declaration.skillKey)
|
|
|| result.originalSlug === declaration.skillKey
|
|
)?.skill ?? results[0]?.skill ?? null;
|
|
if (!imported) {
|
|
throw notFound(`Managed skill was not imported: ${declaration.skillKey}`);
|
|
}
|
|
await upsertBinding(companyId, declaration, imported.id);
|
|
const status: PluginManagedSkillResolution["status"] = mode === "reset" ? "reset" : "created";
|
|
return { skill: imported, status };
|
|
}
|
|
|
|
async function get(skillKey: string, companyId: string) {
|
|
const declaration = declarationFor(skillKey);
|
|
const binding = await getBinding(companyId, skillKey);
|
|
if (!binding) return resolvedSkill(companyId, declaration, null, "missing");
|
|
const skill = await getSkill(companyId, binding.resourceId);
|
|
return resolvedSkill(companyId, declaration, skill, skill ? "resolved" : "missing");
|
|
}
|
|
|
|
async function reconcile(skillKey: string, companyId: string) {
|
|
const declaration = declarationFor(skillKey);
|
|
const current = await get(skillKey, companyId);
|
|
if (current.skill) {
|
|
await upsertBinding(companyId, declaration, current.skill.id);
|
|
return current;
|
|
}
|
|
const imported = await importDeclaredSkill(companyId, declaration, "reconcile");
|
|
await logActivity(db, {
|
|
companyId,
|
|
actorType: "plugin",
|
|
actorId: options.pluginId,
|
|
action: "plugin.managed_skill.reconciled",
|
|
entityType: "company_skill",
|
|
entityId: imported.skill.id,
|
|
details: {
|
|
sourcePluginKey: options.pluginKey,
|
|
managedResourceKey: declaration.skillKey,
|
|
status: imported.status,
|
|
},
|
|
});
|
|
return resolvedSkill(companyId, declaration, imported.skill, imported.status);
|
|
}
|
|
|
|
async function reset(skillKey: string, companyId: string) {
|
|
const declaration = declarationFor(skillKey);
|
|
const imported = await importDeclaredSkill(companyId, declaration, "reset");
|
|
await logActivity(db, {
|
|
companyId,
|
|
actorType: "plugin",
|
|
actorId: options.pluginId,
|
|
action: "plugin.managed_skill.reset",
|
|
entityType: "company_skill",
|
|
entityId: imported.skill.id,
|
|
details: {
|
|
sourcePluginKey: options.pluginKey,
|
|
managedResourceKey: declaration.skillKey,
|
|
},
|
|
});
|
|
return resolvedSkill(companyId, declaration, imported.skill, "reset");
|
|
}
|
|
|
|
return {
|
|
get,
|
|
reconcile,
|
|
reset,
|
|
};
|
|
}
|