fix(skills): pull upstream skill runtime resolution to stop event-loop starvation
Build: Production / build (push) Failing after 12m39s
Build: Production / build (push) Failing after 12m39s
The fork's listRuntimeSkillEntries rematerialized every skill's files from the DB on every heartbeat run dispatch — fs.rm + fs.mkdir + per-file readFile/writeFile, sequentially per skill. With 24 configured skills and 5 concurrent agents, this saturated the Node event loop badly enough that executeRun continuations couldn't reach activeRunExecutions.add() within the orphan-reaper's 5-min threshold, causing reaper to false-positive runs as "process_lost". Upstream's listRuntimeSkillEntries calls resolveRuntimeSkillSource, which checks if the materialized directory already exists on disk and short- circuits when it does. Fixes the symptom at the root. Replaces these files with upstream/master content: - server/src/services/company-skills.ts - server/src/services/heartbeat.ts - server/src/services/workspace-runtime.ts - server/src/services/company-portability.ts - server/src/routes/company-skills.ts - server/src/routes/agents.ts - packages/adapter-utils/src/server-utils.ts Pulls in supporting upstream files: - server/src/services/catalog-provenance.ts - server/src/services/skills-catalog.ts - server/src/services/github-fetch.ts - server/src/services/portable-path.ts - packages/skills-catalog/ (new package) - packages/db document_annotation_* schema + migration 0091 - packages/shared document-annotation types/validators Drops fork features (to be re-evaluated later): - Gitea/Forgejo git skill sources (server/src/services/git-source.ts deleted) - PAT support for private skill repos - Fork-specific secret-export portability extensions Adds agentId: null to acquireRunLease test-probe call in routes/agents.ts to satisfy the fork's environment-runtime agentId requirement (kept). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -27,11 +27,9 @@ import type {
|
||||
CompanyPortabilityIssueManifestEntry,
|
||||
CompanyPortabilitySidebarOrder,
|
||||
CompanyPortabilitySkillManifestEntry,
|
||||
CompanyPortabilitySecretEntry,
|
||||
CompanySkill,
|
||||
AgentEnvConfig,
|
||||
RoutineVariable,
|
||||
SecretProvider,
|
||||
} from "@paperclipai/shared";
|
||||
import {
|
||||
AGENT_DEFAULT_MAX_CONCURRENT_RUNS,
|
||||
@@ -56,8 +54,8 @@ import {
|
||||
} from "@paperclipai/adapter-utils/server-utils";
|
||||
import { requireOpenCodeModelId } from "@paperclipai/adapter-opencode-local/server";
|
||||
import { findServerAdapter } from "../adapters/index.js";
|
||||
import { forbidden, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { openRepoSnapshot, parseGitSourceUrl, resolveGitRef, type RepoSnapshot } from "./git-source.js";
|
||||
import { forbidden, notFound, unprocessable } from "../errors.js";
|
||||
import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { accessService } from "./access.js";
|
||||
import { agentService } from "./agents.js";
|
||||
@@ -72,6 +70,12 @@ import { issueService } from "./issues.js";
|
||||
import { projectService } from "./projects.js";
|
||||
import { routineService } from "./routines.js";
|
||||
import { secretService } from "./secrets.js";
|
||||
import {
|
||||
PORTABLE_CATALOG_PROVENANCE_STRING_KEYS,
|
||||
readCatalogStringList,
|
||||
readPortableCatalogProvenance,
|
||||
} from "./catalog-provenance.js";
|
||||
import { normalizePortablePath } from "./portable-path.js";
|
||||
|
||||
/** Build OrgNode tree from manifest agent list (slug + reportsToSlug). */
|
||||
function buildOrgTreeFromManifest(agents: CompanyPortabilityManifest["agents"]): OrgNode[] {
|
||||
@@ -230,6 +234,28 @@ function readSkillSourceKind(skill: CompanySkill) {
|
||||
return asString(metadata?.sourceKind);
|
||||
}
|
||||
|
||||
function buildPortableCatalogProvenance(skill: CompanySkill) {
|
||||
if (skill.sourceType !== "catalog") return null;
|
||||
const metadata = isPlainRecord(skill.metadata) ? skill.metadata : null;
|
||||
const provenance: Record<string, unknown> = {
|
||||
skillKey: skill.key,
|
||||
};
|
||||
|
||||
const sourceRef = asString(skill.sourceRef) ?? asString(metadata?.originHash);
|
||||
if (sourceRef) provenance.sourceRef = sourceRef;
|
||||
|
||||
for (const key of PORTABLE_CATALOG_PROVENANCE_STRING_KEYS) {
|
||||
if (key === "sourceRef") continue;
|
||||
const value = asString(metadata?.[key]);
|
||||
if (value) provenance[key] = value;
|
||||
}
|
||||
|
||||
const auditCodes = readCatalogStringList(metadata?.auditCodes);
|
||||
if (auditCodes) provenance.auditCodes = auditCodes;
|
||||
|
||||
return Object.keys(provenance).length > 1 ? provenance : null;
|
||||
}
|
||||
|
||||
function deriveLocalExportNamespace(skill: CompanySkill, slug: string) {
|
||||
const metadata = isPlainRecord(skill.metadata) ? skill.metadata : null;
|
||||
const candidates = [
|
||||
@@ -405,7 +431,7 @@ function normalizePortableProjectEnv(value: unknown): AgentEnvConfig | null {
|
||||
return parsed.success ? parsed.data : null;
|
||||
}
|
||||
|
||||
async function extractPortableScopedEnvInputs(
|
||||
function extractPortableScopedEnvInputs(
|
||||
scope: {
|
||||
label: string;
|
||||
warningPrefix: string;
|
||||
@@ -414,11 +440,7 @@ async function extractPortableScopedEnvInputs(
|
||||
},
|
||||
envValue: unknown,
|
||||
warnings: string[],
|
||||
secrets: { getById: (id: string) => Promise<{ name: string; provider: string; description: string | null; latestVersion: number } | null>; resolveSecretValue: (companyId: string, secretId: string, version: "latest") => Promise<string> },
|
||||
secretEntries: CompanyPortabilitySecretEntry[],
|
||||
includeSecrets: boolean,
|
||||
companyId: string,
|
||||
): Promise<CompanyPortabilityEnvInput[]> {
|
||||
): CompanyPortabilityEnvInput[] {
|
||||
if (!isPlainRecord(envValue)) return [];
|
||||
const env = envValue as Record<string, unknown>;
|
||||
const inputs: CompanyPortabilityEnvInput[] = [];
|
||||
@@ -430,7 +452,6 @@ async function extractPortableScopedEnvInputs(
|
||||
}
|
||||
|
||||
if (isPlainRecord(binding) && binding.type === "secret_ref") {
|
||||
const secret = await secrets.getById(String(binding.secretId));
|
||||
inputs.push({
|
||||
key,
|
||||
description: `Provide ${key} for ${scope.label}`,
|
||||
@@ -440,33 +461,7 @@ async function extractPortableScopedEnvInputs(
|
||||
requirement: "optional",
|
||||
defaultValue: "",
|
||||
portability: "portable",
|
||||
secretName: secret?.name ?? null,
|
||||
secretProvider: secret?.provider ?? null,
|
||||
});
|
||||
if (includeSecrets && secret && binding.secretId) {
|
||||
const alreadyExported = secretEntries.some((e) => e.name === secret.name);
|
||||
if (!alreadyExported) {
|
||||
try {
|
||||
const resolvedValue = await secrets.resolveSecretValue(companyId, String(binding.secretId), "latest");
|
||||
secretEntries.push({
|
||||
name: secret.name,
|
||||
provider: secret.provider as SecretProvider,
|
||||
description: secret.description,
|
||||
latestVersion: secret.latestVersion,
|
||||
currentValue: resolvedValue,
|
||||
});
|
||||
} catch {
|
||||
secretEntries.push({
|
||||
name: secret.name,
|
||||
provider: secret.provider as SecretProvider,
|
||||
description: secret.description,
|
||||
latestVersion: secret.latestVersion,
|
||||
currentValue: `<decryption-key-missing:${secret.name}>`,
|
||||
});
|
||||
warnings.push(`Secret "${secret.name}" could not be decrypted during export. Placeholder written.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -476,6 +471,9 @@ async function extractPortableScopedEnvInputs(
|
||||
const portability = defaultValue && isAbsoluteCommand(defaultValue)
|
||||
? "system_dependent"
|
||||
: "portable";
|
||||
if (portability === "system_dependent") {
|
||||
warnings.push(`${scope.warningPrefix} env ${key} default was exported as system-dependent.`);
|
||||
}
|
||||
inputs.push({
|
||||
key,
|
||||
description: `Optional default for ${key} on ${scope.label}`,
|
||||
@@ -491,6 +489,9 @@ async function extractPortableScopedEnvInputs(
|
||||
|
||||
if (typeof binding === "string") {
|
||||
const portability = isAbsoluteCommand(binding) ? "system_dependent" : "portable";
|
||||
if (portability === "system_dependent") {
|
||||
warnings.push(`${scope.warningPrefix} env ${key} default was exported as system-dependent.`);
|
||||
}
|
||||
inputs.push({
|
||||
key,
|
||||
description: `Optional default for ${key} on ${scope.label}`,
|
||||
@@ -598,14 +599,11 @@ type AgentLike = {
|
||||
};
|
||||
|
||||
type EnvInputRecord = {
|
||||
type?: "secret_ref" | "plain";
|
||||
kind: "secret" | "plain";
|
||||
requirement: "required" | "optional";
|
||||
default?: string | null;
|
||||
description?: string | null;
|
||||
portability?: "portable" | "system_dependent";
|
||||
secretName?: string | null;
|
||||
secretProvider?: string | null;
|
||||
};
|
||||
|
||||
const COMPANY_LOGO_CONTENT_TYPE_EXTENSIONS: Record<string, string> = {
|
||||
@@ -1445,20 +1443,6 @@ function normalizeInclude(input?: Partial<CompanyPortabilityInclude>): CompanyPo
|
||||
};
|
||||
}
|
||||
|
||||
function normalizePortablePath(input: string) {
|
||||
const normalized = input.replace(/\\/g, "/").replace(/^\.\/+/, "");
|
||||
const parts: string[] = [];
|
||||
for (const segment of normalized.split("/")) {
|
||||
if (!segment || segment === ".") continue;
|
||||
if (segment === "..") {
|
||||
if (parts.length > 0) parts.pop();
|
||||
continue;
|
||||
}
|
||||
parts.push(segment);
|
||||
}
|
||||
return parts.join("/");
|
||||
}
|
||||
|
||||
function resolvePortablePath(fromPath: string, targetPath: string) {
|
||||
const baseDir = path.posix.dirname(fromPath.replace(/\\/g, "/"));
|
||||
return normalizePortablePath(path.posix.join(baseDir, targetPath.replace(/\\/g, "/")));
|
||||
@@ -1747,15 +1731,11 @@ function isAbsoluteCommand(value: string) {
|
||||
return path.isAbsolute(value) || /^[A-Za-z]:[\\/]/.test(value);
|
||||
}
|
||||
|
||||
async function extractPortableEnvInputs(
|
||||
function extractPortableEnvInputs(
|
||||
agentSlug: string,
|
||||
envValue: unknown,
|
||||
warnings: string[],
|
||||
secrets: { getById: (id: string) => Promise<{ name: string; provider: string; description: string | null; latestVersion: number } | null>; resolveSecretValue: (companyId: string, secretId: string, version: "latest") => Promise<string> },
|
||||
secretEntries: CompanyPortabilitySecretEntry[],
|
||||
includeSecrets: boolean,
|
||||
companyId: string,
|
||||
): Promise<CompanyPortabilityEnvInput[]> {
|
||||
): CompanyPortabilityEnvInput[] {
|
||||
return extractPortableScopedEnvInputs(
|
||||
{
|
||||
label: `agent ${agentSlug}`,
|
||||
@@ -1765,22 +1745,14 @@ async function extractPortableEnvInputs(
|
||||
},
|
||||
envValue,
|
||||
warnings,
|
||||
secrets,
|
||||
secretEntries,
|
||||
includeSecrets,
|
||||
companyId,
|
||||
);
|
||||
}
|
||||
|
||||
async function extractPortableProjectEnvInputs(
|
||||
function extractPortableProjectEnvInputs(
|
||||
projectSlug: string,
|
||||
envValue: unknown,
|
||||
warnings: string[],
|
||||
secrets: { getById: (id: string) => Promise<{ name: string; provider: string; description: string | null; latestVersion: number } | null>; resolveSecretValue: (companyId: string, secretId: string, version: "latest") => Promise<string> },
|
||||
secretEntries: CompanyPortabilitySecretEntry[],
|
||||
includeSecrets: boolean,
|
||||
companyId: string,
|
||||
): Promise<CompanyPortabilityEnvInput[]> {
|
||||
): CompanyPortabilityEnvInput[] {
|
||||
return extractPortableScopedEnvInputs(
|
||||
{
|
||||
label: `project ${projectSlug}`,
|
||||
@@ -1790,10 +1762,6 @@ async function extractPortableProjectEnvInputs(
|
||||
},
|
||||
envValue,
|
||||
warnings,
|
||||
secrets,
|
||||
secretEntries,
|
||||
includeSecrets,
|
||||
companyId,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2172,12 +2140,14 @@ async function withSkillSourceMetadata(skill: CompanySkill, markdown: string) {
|
||||
if (sourceEntry) {
|
||||
metadata.sources = [...existingSources, sourceEntry];
|
||||
}
|
||||
const catalogProvenance = buildPortableCatalogProvenance(skill);
|
||||
metadata.skillKey = skill.key;
|
||||
metadata.paperclipSkillKey = skill.key;
|
||||
metadata.paperclip = {
|
||||
...(isPlainRecord(metadata.paperclip) ? metadata.paperclip : {}),
|
||||
skillKey: skill.key,
|
||||
slug: skill.slug,
|
||||
...(catalogProvenance ? { catalog: catalogProvenance } : {}),
|
||||
};
|
||||
const frontmatter = {
|
||||
...parsed.frontmatter,
|
||||
@@ -2339,6 +2309,42 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchText(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function fetchOptionalText(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function fetchBinary(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
const response = await ghFetch(url, {
|
||||
headers: {
|
||||
accept: "application/vnd.github+json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.json() as Promise<T>;
|
||||
}
|
||||
|
||||
function dedupeEnvInputs(values: CompanyPortabilityManifest["envInputs"]) {
|
||||
const seen = new Set<string>();
|
||||
@@ -2362,13 +2368,6 @@ function buildEnvInputMap(inputs: CompanyPortabilityEnvInput[]) {
|
||||
if (input.defaultValue !== null) entry.default = input.defaultValue;
|
||||
if (input.description) entry.description = input.description;
|
||||
if (input.portability === "system_dependent") entry.portability = "system_dependent";
|
||||
if (input.secretName) {
|
||||
entry.secretName = input.secretName;
|
||||
entry.type = "secret_ref";
|
||||
} else {
|
||||
entry.type = "plain";
|
||||
}
|
||||
if (input.secretProvider) entry.secretProvider = input.secretProvider;
|
||||
env[input.key] = entry;
|
||||
}
|
||||
return env;
|
||||
@@ -2413,9 +2412,6 @@ function readAgentEnvInputs(
|
||||
requirement: record.requirement === "required" ? "required" : "optional",
|
||||
defaultValue: typeof record.default === "string" ? record.default : null,
|
||||
portability: record.portability === "system_dependent" ? "system_dependent" : "portable",
|
||||
secretName: record.secretName ?? null,
|
||||
secretProvider: record.secretProvider ?? null,
|
||||
type: record.type,
|
||||
}];
|
||||
});
|
||||
}
|
||||
@@ -2440,9 +2436,6 @@ function readProjectEnvInputs(
|
||||
requirement: record.requirement === "required" ? "required" : "optional",
|
||||
defaultValue: typeof record.default === "string" ? record.default : null,
|
||||
portability: record.portability === "system_dependent" ? "system_dependent" : "portable",
|
||||
secretName: record.secretName ?? null,
|
||||
secretProvider: record.secretProvider ?? null,
|
||||
type: record.type,
|
||||
}];
|
||||
});
|
||||
}
|
||||
@@ -2489,7 +2482,6 @@ function buildManifestFromPackageFiles(
|
||||
const paperclipProjects = isPlainRecord(paperclipExtension.projects) ? paperclipExtension.projects : {};
|
||||
const paperclipTasks = isPlainRecord(paperclipExtension.tasks) ? paperclipExtension.tasks : {};
|
||||
const paperclipRoutines = isPlainRecord(paperclipExtension.routines) ? paperclipExtension.routines : {};
|
||||
const paperclipSecrets = Array.isArray(paperclipExtension.secrets) ? paperclipExtension.secrets : [];
|
||||
const companyName =
|
||||
asString(companyFrontmatter.name)
|
||||
?? opts?.sourceLabel?.companyName
|
||||
@@ -2573,7 +2565,6 @@ function buildManifestFromPackageFiles(
|
||||
projects: [],
|
||||
issues: [],
|
||||
envInputs: [],
|
||||
secrets: paperclipSecrets.length > 0 ? paperclipSecrets : undefined,
|
||||
};
|
||||
|
||||
const warnings: string[] = [];
|
||||
@@ -2693,10 +2684,17 @@ function buildManifestFromPackageFiles(
|
||||
normalizedMetadata = {
|
||||
sourceKind: "url",
|
||||
};
|
||||
} else if (metadata) {
|
||||
normalizedMetadata = {
|
||||
sourceKind: "catalog",
|
||||
};
|
||||
} else {
|
||||
const catalogProvenance = readPortableCatalogProvenance(metadata);
|
||||
if (catalogProvenance) {
|
||||
sourceType = "catalog";
|
||||
sourceRef = catalogProvenance.sourceRef;
|
||||
normalizedMetadata = catalogProvenance.metadata;
|
||||
} else if (metadata) {
|
||||
normalizedMetadata = {
|
||||
sourceKind: "catalog",
|
||||
};
|
||||
}
|
||||
}
|
||||
const key = deriveManifestSkillKey(frontmatter, slug, normalizedMetadata, sourceType, sourceLocator);
|
||||
|
||||
@@ -2828,37 +2826,52 @@ function normalizeGitHubSourcePath(value: string | null | undefined) {
|
||||
|
||||
export function parseGitHubSourceUrl(rawUrl: string) {
|
||||
const url = new URL(rawUrl);
|
||||
// Handle the portability-specific companyPath query param before delegating,
|
||||
// since git-source has no notion of it.
|
||||
const queryCompanyPath = normalizeGitHubSourcePath(url.searchParams.get("companyPath"));
|
||||
|
||||
const parsed = parseGitSourceUrl(rawUrl);
|
||||
|
||||
let companyPath: string;
|
||||
let basePath = parsed.basePath;
|
||||
if (queryCompanyPath) {
|
||||
companyPath = queryCompanyPath;
|
||||
if (!basePath) {
|
||||
const derived = path.posix.dirname(companyPath);
|
||||
basePath = derived === "." ? "" : derived;
|
||||
}
|
||||
} else if (parsed.filePath) {
|
||||
// blob-style URL pointed directly at a file
|
||||
companyPath = parsed.filePath;
|
||||
} else if (basePath) {
|
||||
companyPath = `${basePath}/COMPANY.md`;
|
||||
} else {
|
||||
companyPath = "COMPANY.md";
|
||||
if (url.protocol !== "https:") {
|
||||
throw unprocessable("GitHub source URL must use HTTPS");
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: parsed.hostname,
|
||||
owner: parsed.owner,
|
||||
repo: parsed.repo,
|
||||
ref: parsed.ref ?? "main",
|
||||
basePath,
|
||||
companyPath,
|
||||
};
|
||||
const hostname = url.hostname;
|
||||
const parts = url.pathname.split("/").filter(Boolean);
|
||||
if (parts.length < 2) {
|
||||
throw unprocessable("Invalid GitHub URL");
|
||||
}
|
||||
const owner = parts[0]!;
|
||||
const repo = parts[1]!.replace(/\.git$/i, "");
|
||||
const queryRef = url.searchParams.get("ref")?.trim();
|
||||
const queryPath = normalizeGitHubSourcePath(url.searchParams.get("path"));
|
||||
const queryCompanyPath = normalizeGitHubSourcePath(url.searchParams.get("companyPath"));
|
||||
if (queryRef || queryPath || queryCompanyPath) {
|
||||
const companyPath = queryCompanyPath || [queryPath, "COMPANY.md"].filter(Boolean).join("/") || "COMPANY.md";
|
||||
let basePath = queryPath;
|
||||
if (!basePath && companyPath !== "COMPANY.md") {
|
||||
basePath = path.posix.dirname(companyPath);
|
||||
if (basePath === ".") basePath = "";
|
||||
}
|
||||
return {
|
||||
hostname,
|
||||
owner,
|
||||
repo,
|
||||
ref: queryRef || "main",
|
||||
basePath,
|
||||
companyPath,
|
||||
};
|
||||
}
|
||||
let ref = "main";
|
||||
let basePath = "";
|
||||
let companyPath = "COMPANY.md";
|
||||
if (parts[2] === "tree") {
|
||||
ref = parts[3] ?? "main";
|
||||
basePath = parts.slice(4).join("/");
|
||||
} else if (parts[2] === "blob") {
|
||||
ref = parts[3] ?? "main";
|
||||
const blobPath = parts.slice(4).join("/");
|
||||
if (!blobPath) {
|
||||
throw unprocessable("Invalid GitHub blob URL");
|
||||
}
|
||||
companyPath = blobPath;
|
||||
basePath = path.posix.dirname(blobPath);
|
||||
if (basePath === ".") basePath = "";
|
||||
}
|
||||
return { hostname, owner, repo, ref, basePath, companyPath };
|
||||
}
|
||||
|
||||
|
||||
@@ -2962,38 +2975,30 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
);
|
||||
}
|
||||
|
||||
const sourceUrl = source.url;
|
||||
const parsed = parseGitHubSourceUrl(sourceUrl);
|
||||
const parsed = parseGitHubSourceUrl(source.url);
|
||||
let ref = parsed.ref;
|
||||
const warnings: string[] = [];
|
||||
const companyRelativePath = parsed.companyPath === "COMPANY.md"
|
||||
? [parsed.basePath, "COMPANY.md"].filter(Boolean).join("/")
|
||||
: parsed.companyPath;
|
||||
|
||||
async function openSnapshot(refName: string): Promise<RepoSnapshot> {
|
||||
const ps = parseGitSourceUrl(sourceUrl);
|
||||
const wanted = { ...ps, ref: refName, explicitRef: true };
|
||||
const resolved = await resolveGitRef(wanted);
|
||||
return openRepoSnapshot(wanted, resolved.trackingRef, resolved.pinnedSha);
|
||||
}
|
||||
|
||||
let ref = parsed.ref;
|
||||
let snapshot: RepoSnapshot;
|
||||
let companyMarkdown: string | null = null;
|
||||
try {
|
||||
snapshot = await openSnapshot(ref);
|
||||
companyMarkdown = await snapshot.readFileOptional(companyRelativePath);
|
||||
companyMarkdown = await fetchOptionalText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
|
||||
);
|
||||
} catch (err) {
|
||||
if (ref === "main") {
|
||||
ref = "master";
|
||||
warnings.push("Git ref main not found; falling back to master.");
|
||||
snapshot = await openSnapshot(ref);
|
||||
companyMarkdown = await snapshot.readFileOptional(companyRelativePath);
|
||||
warnings.push("GitHub ref main not found; falling back to master.");
|
||||
companyMarkdown = await fetchOptionalText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
|
||||
);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (!companyMarkdown) {
|
||||
throw unprocessable("Git company package is missing COMPANY.md");
|
||||
throw unprocessable("GitHub company package is missing COMPANY.md");
|
||||
}
|
||||
|
||||
const companyPath = parsed.companyPath === "COMPANY.md"
|
||||
@@ -3002,22 +3007,31 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const files: Record<string, CompanyPortabilityFileEntry> = {
|
||||
[companyPath]: companyMarkdown,
|
||||
};
|
||||
const apiBase = gitHubApiBase(parsed.hostname);
|
||||
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
|
||||
`${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
|
||||
).catch(() => ({ tree: [] }));
|
||||
const basePrefix = parsed.basePath ? `${parsed.basePath.replace(/^\/+|\/+$/g, "")}/` : "";
|
||||
const allPaths = await snapshot.listFiles();
|
||||
const candidatePaths = allPaths.filter((entry) => {
|
||||
if (basePrefix && !entry.startsWith(basePrefix)) return false;
|
||||
const relative = basePrefix ? entry.slice(basePrefix.length) : entry;
|
||||
return (
|
||||
relative.endsWith(".md") ||
|
||||
relative.startsWith("skills/") ||
|
||||
relative === ".paperclip.yaml" ||
|
||||
relative === ".paperclip.yml"
|
||||
);
|
||||
});
|
||||
const candidatePaths = (tree.tree ?? [])
|
||||
.filter((entry) => entry.type === "blob")
|
||||
.map((entry) => entry.path)
|
||||
.filter((entry): entry is string => typeof entry === "string")
|
||||
.filter((entry) => {
|
||||
if (basePrefix && !entry.startsWith(basePrefix)) return false;
|
||||
const relative = basePrefix ? entry.slice(basePrefix.length) : entry;
|
||||
return (
|
||||
relative.endsWith(".md") ||
|
||||
relative.startsWith("skills/") ||
|
||||
relative === ".paperclip.yaml" ||
|
||||
relative === ".paperclip.yml"
|
||||
);
|
||||
});
|
||||
for (const repoPath of candidatePaths) {
|
||||
const relativePath = basePrefix ? repoPath.slice(basePrefix.length) : repoPath;
|
||||
if (files[relativePath] !== undefined) continue;
|
||||
files[normalizePortablePath(relativePath)] = await snapshot.readFile(repoPath);
|
||||
files[normalizePortablePath(relativePath)] = await fetchText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
);
|
||||
}
|
||||
const companyDoc = parseFrontmatterMarkdown(companyMarkdown);
|
||||
const includeEntries = readIncludeEntries(companyDoc.frontmatter);
|
||||
@@ -3026,7 +3040,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const relativePath = normalizePortablePath(includeEntry.path);
|
||||
if (files[relativePath] !== undefined) continue;
|
||||
if (!(repoPath.endsWith(".md") || repoPath.endsWith(".yaml") || repoPath.endsWith(".yml"))) continue;
|
||||
files[relativePath] = await snapshot.readFile(repoPath);
|
||||
files[relativePath] = await fetchText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
);
|
||||
}
|
||||
|
||||
const resolved = buildManifestFromPackageFiles(files);
|
||||
@@ -3034,13 +3050,12 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
if (companyLogoPath && !resolved.files[companyLogoPath]) {
|
||||
const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/");
|
||||
try {
|
||||
const binary = await snapshot.readBinary(repoPath);
|
||||
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(
|
||||
Buffer.from(binary),
|
||||
inferContentTypeFromPath(companyLogoPath),
|
||||
const binary = await fetchBinary(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
);
|
||||
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath));
|
||||
} catch (err) {
|
||||
warnings.push(`Failed to fetch company logo ${companyLogoPath} from git: ${err instanceof Error ? err.message : String(err)}`);
|
||||
warnings.push(`Failed to fetch company logo ${companyLogoPath} from GitHub: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
resolved.warnings.unshift(...warnings);
|
||||
@@ -3067,9 +3082,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const files: Record<string, CompanyPortabilityFileEntry> = {};
|
||||
const warnings: string[] = [];
|
||||
const envInputs: CompanyPortabilityManifest["envInputs"] = [];
|
||||
const secretEntries: CompanyPortabilitySecretEntry[] = [];
|
||||
const requestedSidebarOrder = normalizePortableSidebarOrder(input.sidebarOrder);
|
||||
const includeSecrets = input.includeSecrets === true;
|
||||
const rootPath = normalizeAgentUrlKey(company.name) ?? "company-package";
|
||||
let companyLogoPath: string | null = null;
|
||||
|
||||
@@ -3349,14 +3362,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
warnings.push(...exportedInstructions.warnings);
|
||||
|
||||
const envInputsStart = envInputs.length;
|
||||
const exportedEnvInputs = await extractPortableEnvInputs(
|
||||
const exportedEnvInputs = extractPortableEnvInputs(
|
||||
slug,
|
||||
(agent.adapterConfig as Record<string, unknown>).env,
|
||||
warnings,
|
||||
secrets,
|
||||
secretEntries,
|
||||
includeSecrets,
|
||||
companyId,
|
||||
);
|
||||
envInputs.push(...exportedEnvInputs);
|
||||
const adapterDefaultRules = ADAPTER_DEFAULT_RULES_BY_TYPE[agent.adapterType] ?? [];
|
||||
@@ -3433,7 +3442,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const slug = projectSlugById.get(project.id)!;
|
||||
const projectPath = `projects/${slug}/PROJECT.md`;
|
||||
const envInputsStart = envInputs.length;
|
||||
const exportedEnvInputs = await extractPortableProjectEnvInputs(slug, project.env, warnings, secrets, secretEntries, includeSecrets, companyId);
|
||||
const exportedEnvInputs = extractPortableProjectEnvInputs(slug, project.env, warnings);
|
||||
envInputs.push(...exportedEnvInputs);
|
||||
const projectEnvInputs = dedupeEnvInputs(
|
||||
envInputs
|
||||
@@ -3653,20 +3662,8 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
skills: resolved.manifest.skills.length > 0,
|
||||
};
|
||||
resolved.manifest.envInputs = dedupeEnvInputs(envInputs);
|
||||
if (includeSecrets) {
|
||||
resolved.manifest.secrets = secretEntries.length > 0 ? secretEntries : undefined;
|
||||
}
|
||||
resolved.warnings.unshift(...warnings);
|
||||
|
||||
// Rebuild the YAML file to include secrets so files stay in sync with manifest
|
||||
// Only include secrets - other fields should come from the original YAML structure
|
||||
if (includeSecrets && resolved.manifest.secrets) {
|
||||
// Parse existing YAML and add secrets to it
|
||||
const existingYaml = parseYamlFile(readPortableTextFile(finalFiles, paperclipExtensionPath) ?? "") ?? {};
|
||||
existingYaml.secrets = resolved.manifest.secrets;
|
||||
finalFiles[paperclipExtensionPath] = buildYamlFile(existingYaml, { preserveEmptyStrings: true });
|
||||
}
|
||||
|
||||
return {
|
||||
rootPath,
|
||||
manifest: resolved.manifest,
|
||||
@@ -4231,7 +4228,6 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const resultAgents: CompanyPortabilityImportResult["agents"] = [];
|
||||
const resultProjects: CompanyPortabilityImportResult["projects"] = [];
|
||||
const importedSlugToAgentId = new Map<string, string>();
|
||||
const secretNameToId = new Map<string, string>();
|
||||
const existingSlugToAgentId = new Map<string, string>();
|
||||
const agentStatusById = new Map<string, string | null | undefined>();
|
||||
const existingAgents = await agents.list(targetCompany.id);
|
||||
@@ -4263,35 +4259,6 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
}
|
||||
}
|
||||
|
||||
// Create secrets in target company and build name->id map
|
||||
for (const secretEntry of sourceManifest.secrets ?? []) {
|
||||
if (secretEntry.currentValue.startsWith("<decryption-key-missing:")) {
|
||||
warnings.push(`Secret "${secretEntry.name}" could not be decrypted in source instance. ` +
|
||||
`Placeholder written for key. Create a secret with this name and update manually.`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const created = await secrets.create(targetCompany.id, {
|
||||
name: secretEntry.name,
|
||||
provider: secretEntry.provider,
|
||||
value: secretEntry.currentValue,
|
||||
description: secretEntry.description,
|
||||
});
|
||||
secretNameToId.set(secretEntry.name, created.id);
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError && err.status === 409) {
|
||||
const existing = await secrets.getByName(targetCompany.id, secretEntry.name);
|
||||
if (existing) {
|
||||
secretNameToId.set(secretEntry.name, existing.id);
|
||||
} else {
|
||||
warnings.push(`Secret "${secretEntry.name}" already exists but could not be resolved by name. Re-add env bindings for this secret manually.`);
|
||||
}
|
||||
} else {
|
||||
warnings.push(`Failed to create secret "${secretEntry.name}": ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (include.agents) {
|
||||
for (const planAgent of plan.preview.plan.agentPlans) {
|
||||
const manifestAgent = plan.selectedAgents.find((agent) => agent.slug === planAgent.slug);
|
||||
@@ -4348,30 +4315,6 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
desiredSkills,
|
||||
mode,
|
||||
);
|
||||
|
||||
// Reconstruct adapterConfig.env from manifest.envInputs for this agent
|
||||
const agentEnvInputs = (sourceManifest.envInputs ?? []).filter((e) => e.agentSlug === manifestAgent.slug);
|
||||
if (agentEnvInputs.length > 0) {
|
||||
const env: Record<string, unknown> = {};
|
||||
for (const ei of agentEnvInputs) {
|
||||
if (ei.kind === "secret" && ei.secretName) {
|
||||
const newSecretId = secretNameToId.get(ei.secretName);
|
||||
if (newSecretId) {
|
||||
env[ei.key] = { type: "secret_ref", secretId: newSecretId };
|
||||
} else {
|
||||
warnings.push(`Env key "${ei.key}" for agent ${manifestAgent.slug} references secret "${ei.secretName}" which was not included in this package. Re-add manually.`);
|
||||
}
|
||||
} else if (ei.kind === "secret" && !ei.secretName) {
|
||||
warnings.push(`Env key "${ei.key}" for agent ${manifestAgent.slug} could not be reconstructed (sensitive binding without secret reference). Re-add manually.`);
|
||||
} else if (ei.kind === "plain" && ei.defaultValue !== null) {
|
||||
env[ei.key] = { type: "plain", value: ei.defaultValue };
|
||||
}
|
||||
}
|
||||
if (Object.keys(env).length > 0) {
|
||||
normalizedAdapter.adapterConfig.env = await secrets.normalizeEnvBindingsForPersistence(targetCompany.id, env as any, { strictMode: strictSecretsMode });
|
||||
}
|
||||
}
|
||||
|
||||
const patch = {
|
||||
name: planAgent.plannedName,
|
||||
role: manifestAgent.role,
|
||||
@@ -4422,9 +4365,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const createdStatus = "idle";
|
||||
let created = await agents.create(targetCompany.id, {
|
||||
...patch,
|
||||
status: "idle",
|
||||
status: createdStatus,
|
||||
});
|
||||
await access.ensureMembership(targetCompany.id, "agent", created.id, "member", "active");
|
||||
await access.setPrincipalPermission(
|
||||
@@ -4444,7 +4388,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
} catch (err) {
|
||||
warnings.push(`Failed to materialize instructions bundle for ${manifestAgent.slug}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
agentStatusById.set(created.id, created.status ?? "idle");
|
||||
agentStatusById.set(created.id, created.status ?? createdStatus);
|
||||
importedSlugToAgentId.set(planAgent.slug, created.id);
|
||||
existingSlugToAgentId.set(normalizeAgentUrlKey(created.name) ?? created.id, created.id);
|
||||
resultAgents.push({
|
||||
@@ -4493,26 +4437,6 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
?? null
|
||||
: null;
|
||||
const projectWorkspaceIdByKey = new Map<string, string>();
|
||||
// Build project env from manifest.envInputs filtered by this project
|
||||
const projectEnvInputs = (sourceManifest.envInputs ?? []).filter((e) => e.projectSlug === planProject.slug);
|
||||
const reconstructedProjectEnv: Record<string, unknown> = {};
|
||||
for (const ei of projectEnvInputs) {
|
||||
if (ei.kind === "secret" && ei.secretName) {
|
||||
const newSecretId = secretNameToId.get(ei.secretName);
|
||||
if (newSecretId) {
|
||||
reconstructedProjectEnv[ei.key] = { type: "secret_ref", secretId: newSecretId };
|
||||
} else {
|
||||
warnings.push(`Env key "${ei.key}" for project ${planProject.slug} references secret "${ei.secretName}" which was not included in this package. Re-add manually.`);
|
||||
}
|
||||
} else if (ei.kind === "secret" && !ei.secretName) {
|
||||
warnings.push(`Env key "${ei.key}" for project ${planProject.slug} could not be reconstructed (sensitive binding without secret reference). Re-add manually.`);
|
||||
} else if (ei.kind === "plain" && ei.defaultValue !== null) {
|
||||
reconstructedProjectEnv[ei.key] = { type: "plain", value: ei.defaultValue };
|
||||
}
|
||||
}
|
||||
const projectEnvConfig = Object.keys(reconstructedProjectEnv).length > 0
|
||||
? await secrets.normalizeEnvBindingsForPersistence(targetCompany.id, reconstructedProjectEnv as any, { strictMode: strictSecretsMode })
|
||||
: null;
|
||||
const projectPatch = {
|
||||
name: planProject.plannedName,
|
||||
description: manifestProject.description,
|
||||
@@ -4522,7 +4446,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
status: manifestProject.status && PROJECT_STATUSES.includes(manifestProject.status as any)
|
||||
? manifestProject.status as typeof PROJECT_STATUSES[number]
|
||||
: "backlog",
|
||||
env: projectEnvConfig ?? undefined,
|
||||
env: manifestProject.env,
|
||||
executionWorkspacePolicy: stripPortableProjectExecutionWorkspaceRefs(manifestProject.executionWorkspacePolicy),
|
||||
};
|
||||
|
||||
@@ -4601,91 +4525,6 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
}
|
||||
}
|
||||
|
||||
// Remap secret_ref bindings in imported agent/project records to target company secret IDs
|
||||
for (const envInput of sourceManifest.envInputs ?? []) {
|
||||
if (envInput.kind !== "secret" || !envInput.secretName) continue;
|
||||
const newSecretId = secretNameToId.get(envInput.secretName);
|
||||
if (!newSecretId) {
|
||||
// secret wasn't created (decryption failure or error) — it's already a placeholder in the env
|
||||
continue;
|
||||
}
|
||||
if (envInput.agentSlug) {
|
||||
const agentId = importedSlugToAgentId.get(envInput.agentSlug);
|
||||
if (agentId) {
|
||||
const agent = await agents.getById(agentId);
|
||||
if (agent) {
|
||||
const adapterConfig = agent.adapterConfig as Record<string, unknown>;
|
||||
const env = adapterConfig.env as Record<string, unknown> | undefined;
|
||||
let mutated = false;
|
||||
if (env && typeof env[envInput.key] === "object" && env[envInput.key] !== null) {
|
||||
const binding = env[envInput.key] as Record<string, unknown>;
|
||||
if (binding.type === "secret_ref" && binding.secretId !== newSecretId) {
|
||||
binding.secretId = newSecretId;
|
||||
mutated = true;
|
||||
}
|
||||
}
|
||||
if (mutated) await agents.update(agentId, { adapterConfig });
|
||||
}
|
||||
}
|
||||
} else if (envInput.projectSlug) {
|
||||
const projectId = importedSlugToProjectId.get(envInput.projectSlug);
|
||||
if (projectId) {
|
||||
const project = await projects.getById(projectId);
|
||||
if (project && project.env && typeof project.env === "object") {
|
||||
const env = project.env as Record<string, unknown>;
|
||||
let mutated = false;
|
||||
if (typeof env[envInput.key] === "object" && env[envInput.key] !== null) {
|
||||
const binding = env[envInput.key] as Record<string, unknown>;
|
||||
if (binding.type === "secret_ref" && binding.secretId !== newSecretId) {
|
||||
binding.secretId = newSecretId;
|
||||
mutated = true;
|
||||
}
|
||||
}
|
||||
if (mutated) await projects.update(projectId, { env: env as import("@paperclipai/shared").AgentEnvConfig });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Note: the legacy secret remapping below is kept as a safety net for
|
||||
// agents/projects that were created/updated before this code existed.
|
||||
// It can be removed once the inline reconstruction above is stable.
|
||||
// Reconstruct plain env bindings and fill in missing env keys on imported agents/projects
|
||||
for (const envInput of sourceManifest.envInputs ?? []) {
|
||||
if (envInput.kind !== "plain" && !(envInput.kind === "secret" && !envInput.secretName)) continue;
|
||||
if (!envInput.defaultValue && envInput.kind === "plain") continue;
|
||||
|
||||
if (envInput.agentSlug) {
|
||||
const agentId = importedSlugToAgentId.get(envInput.agentSlug);
|
||||
if (!agentId) continue;
|
||||
const agent = await agents.getById(agentId);
|
||||
if (!agent) continue;
|
||||
const adapterConfig = agent.adapterConfig as Record<string, unknown>;
|
||||
const env = (adapterConfig.env as Record<string, unknown>) ?? {};
|
||||
let mutated = false;
|
||||
if (!env[envInput.key] && envInput.kind === "plain") {
|
||||
env[envInput.key] = { type: "plain", value: envInput.defaultValue ?? "" };
|
||||
mutated = true;
|
||||
}
|
||||
if (mutated) {
|
||||
adapterConfig.env = env;
|
||||
await agents.update(agentId, { adapterConfig });
|
||||
}
|
||||
} else if (envInput.projectSlug) {
|
||||
const projectId = importedSlugToProjectId.get(envInput.projectSlug);
|
||||
if (!projectId) continue;
|
||||
const project = await projects.getById(projectId);
|
||||
if (!project) continue;
|
||||
const env = (project.env as Record<string, unknown>) ?? {};
|
||||
let mutated = false;
|
||||
if (!env[envInput.key] && envInput.kind === "plain") {
|
||||
env[envInput.key] = { type: "plain", value: envInput.defaultValue ?? "" };
|
||||
mutated = true;
|
||||
}
|
||||
if (mutated) await projects.update(projectId, { env: env as import("@paperclipai/shared").AgentEnvConfig });
|
||||
}
|
||||
}
|
||||
|
||||
if (include.issues) {
|
||||
const routines = routineService(db);
|
||||
for (const manifestIssue of sourceManifest.issues) {
|
||||
|
||||
Reference in New Issue
Block a user