From e37180d3e3b5fdb7ded00efcb9321152d9999e9f Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Fri, 1 May 2026 21:00:08 -0400 Subject: [PATCH 1/8] chore(plugin-rpc): raise MAX_RPC_TIMEOUT_MS cap to 60 minutes --- server/src/services/plugin-worker-manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 8413af10..993a64e5 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -57,8 +57,8 @@ import { logger } from "../middleware/logger.js"; /** Default timeout for RPC calls in milliseconds. */ const DEFAULT_RPC_TIMEOUT_MS = 30_000; -/** Hard upper bound for any RPC timeout (5 minutes). Prevents unbounded waits. */ -const MAX_RPC_TIMEOUT_MS = 5 * 60 * 1_000; +/** Hard upper bound for any RPC timeout (60 minutes). Prevents unbounded waits. */ +const MAX_RPC_TIMEOUT_MS = 60 * 60 * 1_000; /** Timeout for the initialize RPC call. */ const INITIALIZE_TIMEOUT_MS = 15_000; -- 2.52.0 From 85cbbc9263e18777cffc748a29e61f8f3822c604 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Sun, 3 May 2026 10:23:54 -0400 Subject: [PATCH 2/8] revert: restore paperclip-dev skill (validation requires it for now) The earlier fix/remove-paperclip-dev-skill removed the bundled skill, but companies have stale company_skills rows that reference it as required, breaking 'Invalid company skill selection' validation. Put the file back to unblock; the underlying force-required-on-bundled bug remains and should be fixed in code rather than by deleting the skill. --- skills/paperclip-dev/SKILL.md | 267 ++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 skills/paperclip-dev/SKILL.md diff --git a/skills/paperclip-dev/SKILL.md b/skills/paperclip-dev/SKILL.md new file mode 100644 index 00000000..d392d327 --- /dev/null +++ b/skills/paperclip-dev/SKILL.md @@ -0,0 +1,267 @@ +--- +name: paperclip-dev +required: false +description: > + Develop and operate a local Paperclip instance — start and stop servers, + pull updates from master, run builds and tests, manage worktrees, back up + databases, and diagnose problems. Use whenever you need to work on the + Paperclip codebase itself or keep a running instance healthy. +--- + +# Paperclip Dev + +This skill covers the day-to-day workflows for developing and operating a local Paperclip instance. It assumes you are working inside the Paperclip repo checkout with `origin` pointing to `git@github.com:paperclipai/paperclip.git`. + +> **OPEN SOURCE HYGIENE:** This repository is public-facing. Treat anything you push to `origin` as publishable. Never commit or push secrets, API keys, tokens, private logs, PII, customer data, or machine-local configuration that should stay private. Keep git history tidy as well: avoid pushing throwaway branches, noisy checkpoint commits, or speculative work that does not need to be shared upstream. + +> **MANDATORY:** Before running any CLI command, building, testing, or managing worktrees, you MUST read `doc/DEVELOPING.md` in the Paperclip repo. It is the canonical reference for all `paperclipai` CLI commands, their options, build/test workflows, database operations, worktree management, and diagnostics. Do NOT guess at flags or options — read the doc first. + +## Quick Command Reference + +These are the most common commands. For full option tables and details, see `doc/DEVELOPING.md`. + +| Task | Command | +|------|---------| +| Start server (first time or normal) | `npx paperclipai run` | +| Dev mode with hot reload | `pnpm dev` | +| Stop dev server | `pnpm dev:stop` | +| Build | `pnpm build` | +| Type-check | `pnpm typecheck` | +| Run tests | `pnpm test` | +| Run migrations | `pnpm db:migrate` | +| Regenerate Drizzle client | `pnpm db:generate` | +| Back up database | `npx paperclipai db:backup` | +| Health check | `npx paperclipai doctor --repair` | +| Print env vars | `npx paperclipai env` | +| Trigger agent heartbeat | `npx paperclipai heartbeat run --agent-id ` | +| Install agent skills locally | `npx paperclipai agent local-cli --company-id ` | + +## Pulling from Master + +```bash +git fetch origin && git pull origin master +pnpm install && pnpm build +``` + +If schema changes landed, also run `pnpm db:generate && pnpm db:migrate`. + +## Worktrees + +Paperclip worktrees combine git worktrees with isolated Paperclip instances — each gets its own database, server port, and environment seeded from the primary instance. + +> **MANDATORY:** Before creating or managing worktrees, you MUST read the "Worktree-local Instances" and "Worktree CLI Reference" sections in `doc/DEVELOPING.md`. That is the canonical reference for all worktree commands, their options, seed modes, and environment variables. + +### When to Use Worktrees + +- Starting a feature branch that needs its own Paperclip environment +- Running parallel agent work without cross-contaminating the primary instance +- Testing Paperclip changes in isolation before merging + +### Command Overview + +The CLI has two tiers (see `doc/DEVELOPING.md` for full option tables): + +| Command | Purpose | +|---------|---------| +| `worktree:make ` | Create worktree + isolated instance in one step | +| `worktree:list` | List worktrees and their Paperclip status | +| `worktree:merge-history` | Preview/import issue history between worktrees | +| `worktree:cleanup ` | Remove worktree, branch, and instance data | +| `worktree init` | Bootstrap instance inside existing worktree | +| `worktree env` | Print shell exports for worktree instance | +| `worktree reseed` | Refresh worktree DB from another instance | +| `worktree repair` | Fix broken/missing worktree instance metadata | + +### Typical Workflow + +```bash +# 1. Create a worktree for a feature +npx paperclipai worktree:make my-feature --start-point origin/main + +# 2. Move into the worktree (path printed by worktree:make) and source the environment +cd +eval "$(npx paperclipai worktree env)" + +# 3. Start the isolated Paperclip server +npx paperclipai run + +# 4. Do your work + +# 5. When done, merge history back if needed +npx paperclipai worktree:merge-history --from paperclip-my-feature --to current --apply + +# 6. Clean up +npx paperclipai worktree:cleanup my-feature +``` + +## Forks — Prefer Pushing to a User Fork + +If the user has a personal fork of `paperclipai/paperclip` configured as a git remote, push your feature branches to **that fork** instead of creating branches on the main repo. This keeps the upstream branch list clean and matches the standard open-source contribution flow. + +### Detect a fork remote + +Before pushing or creating a PR, list remotes and check for one that points at a non-`paperclipai` GitHub fork: + +```bash +git remote -v +``` + +Treat any remote whose URL points to `github.com:/paperclip` (or `github.com//paperclip.git`) as the user's fork. Common names are `fork`, ``, or `myfork`. The remote named `origin` or `upstream` that points at `paperclipai/paperclip` is the canonical upstream — do not push feature branches there if a fork exists. + +### Pushing to the fork + +```bash +# Push the current branch to the user's fork and set upstream +git push -u HEAD +``` + +Then create the PR from the fork branch: + +```bash +gh pr create --repo paperclipai/paperclip --head : ... +``` + +`gh pr create` usually figures out the head ref automatically when run from a branch tracking the fork; the explicit `--head :` form is the reliable fallback when it does not. + +### When no fork exists + +If `git remote -v` shows only `paperclipai/paperclip` remotes (no user fork), fall back to pushing branches to `origin` as before. Do NOT create a fork on the user's behalf — ask first. + +### Keeping the fork up to date + +The canonical remote that points at `paperclipai/paperclip` may be named `origin` **or** `upstream` depending on how the user set up the repo. Detect it the same way as in the "Detect a fork remote" step, then fetch and push from/with that remote so the sync works under either convention: + +```bash +UPSTREAM_REMOTE=$(git remote -v | awk '/paperclipai\/paperclip.*\(fetch\)/{print $1; exit}') +git fetch "$UPSTREAM_REMOTE" +git push "${UPSTREAM_REMOTE}/master:master" +``` + +## Pull Requests + +> **MANDATORY PRE-FLIGHT:** Before creating ANY pull request, you MUST read the canonical source files listed below. Do NOT run `gh pr create` until you have read these files and verified your PR body matches every required section. + +### Step 1 — Read the canonical files + +You MUST read all three of these files before creating a PR: + +1. **`.github/PULL_REQUEST_TEMPLATE.md`** — the required PR body structure +2. **`CONTRIBUTING.md`** — contribution conventions, PR requirements, and thinking-path examples +3. **`.github/workflows/pr.yml`** — CI checks that gate merge + +### Step 2 — Validate your PR body against this checklist + +After reading the template, verify your `--body` includes every one of these sections (names must match exactly): + +- [ ] `## Thinking Path` — blockquote style, 5-8 reasoning steps +- [ ] `## What Changed` — bullet list of concrete changes +- [ ] `## Verification` — how a reviewer confirms this works +- [ ] `## Risks` — what could go wrong +- [ ] `## Model Used` — provider, model ID, version, capabilities +- [ ] `## Checklist` — copied from the template, items checked off + +If any section is missing or empty, do NOT submit the PR. Go back and fill it in. + +### Step 3 — Create the PR + +Only after completing Steps 1 and 2, run `gh pr create`. Use the template contents as the structure for `--body` — do not write a freeform summary. + +## Hard Rules — Do NOT Bypass + +These rules exist because agents have caused real damage by improvising around CLI failures. Follow them exactly. + +1. **CLI is the only interface to worktrees and databases.** All worktree and database operations MUST go through `npx paperclipai` / `pnpm paperclipai` commands. You MUST NOT: + - Run `pg_dump`, `pg_restore`, `psql`, `createdb`, `dropdb`, or any raw postgres commands + - Manually set `DATABASE_URL` to point a worktree server at another instance's database + - Run `rm -rf` on any `.paperclip/`, `.paperclip-worktrees/`, or `db/` directory + - Directly manipulate embedded postgres data directories + - Kill postgres processes by PID + +2. **If a CLI command fails, stop and report.** Do NOT attempt workarounds. If `worktree:make`, `worktree reseed`, `worktree init`, `worktree:cleanup`, or any other `paperclipai` command fails: + - Report the exact error message in your task comment + - Set the task to `blocked` + - Suggest running `npx paperclipai doctor --repair` or recreating the worktree from scratch + - Do NOT try to manually replicate what the CLI does + +3. **Never share databases between instances.** Each worktree instance gets its own isolated database. Never override `DATABASE_URL` to point one instance at another's database. This destroys isolation and can corrupt production data. + +4. **Starting a dev server in a worktree requires setup first.** The correct sequence is: + ```bash + # If the worktree already exists but has no running instance: + cd + eval "$(npx paperclipai worktree env)" + pnpm install && pnpm build + npx paperclipai run # or pnpm dev + + # If the worktree needs a fresh database: + npx paperclipai worktree reseed --seed-mode full + + # If the worktree is broken beyond repair: + npx paperclipai worktree:cleanup + npx paperclipai worktree:make --seed-mode full + ``` + If any step fails, follow rule 2 — stop and report. + +5. **Seeding is a CLI operation.** When asked to seed a worktree database from the main instance, use `worktree reseed` or recreate with `worktree:make --seed-mode full`. Read `doc/DEVELOPING.md` for the full option tables. Never attempt manual database copying. + +## Persistent Dev Servers (for Manual Testing) + +When an agent needs to start a dev server that outlives the current heartbeat — for example, so a human or QA agent can manually test against it — the server process **must** be launched in a detached session. A process started directly from a heartbeat shell is killed when the heartbeat exits. + +### Use `tmux` for persistent servers + +```bash +# 1. cd into the worktree (or main repo) and source the environment +cd +eval "$(npx paperclipai worktree env)" # skip if using the primary instance + +# 2. Start the dev server in a named, detached tmux session +tmux new-session -d -s 'pnpm dev' + +# Example with a descriptive name: +tmux new-session -d -s auth-fix-3102 'pnpm dev' +``` + +### Managing the session + +| Task | Command | +|------|---------| +| Check if the session is alive | `tmux has-session -t 2>/dev/null && echo running` | +| View server output | `tmux capture-pane -t -p` | +| Kill the session | `tmux kill-session -t ` | +| List all tmux sessions | `tmux list-sessions` | + +### Verifying the server is reachable + +After launching, confirm the port is listening before reporting success: + +```bash +# Wait briefly for startup, then verify +sleep 3 +curl -sf http://127.0.0.1:/api/health && echo "Server is up" +lsof -nP -iTCP: -sTCP:LISTEN +``` + +### Key rules + +1. **Always use `tmux` (or equivalent)** when a dev server needs to stay running after the heartbeat ends. A server started directly from the agent shell will die when the heartbeat exits, even if it appeared healthy moments before. +2. **Name the session descriptively** — include the worktree name and port (e.g., `auth-fix-3102`). +3. **Verify the server is listening** before reporting the URL to anyone. +4. **Do not use `nohup` or `&` alone** — these are unreliable for agent shells that may have their entire process group killed. +5. **Clean up when done** — kill the tmux session when the testing is complete. + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Server won't start | Run `npx paperclipai doctor --repair` to diagnose and auto-fix | +| Forgetting to source worktree env | Run `eval "$(npx paperclipai worktree env)"` after cd-ing into the worktree | +| Stale dependencies after pull | Run `pnpm install && pnpm build` after pulling | +| Schema out of date after pull | Run `pnpm db:generate && pnpm db:migrate` | +| Reseeding while target DB is running | Stop the target server first, or use `--allow-live-target` | +| Cleaning up with unmerged commits | Merge or push first, or use `--force` if intentionally discarding | +| Running agents against wrong instance | Verify `PAPERCLIP_API_URL` points to the correct port | +| CLI command fails | Do NOT work around it — report the error and block (see Hard Rules above) | +| Agent tries manual postgres operations | NEVER do this — all DB ops go through the CLI (see Hard Rules above) | +| Dev server dies between heartbeats | Launch in a detached `tmux` session — see "Persistent Dev Servers" above | +| Pushed feature branch to `paperclipai/paperclip` when a fork exists | Push to the user's fork remote instead — see "Forks" above | -- 2.52.0 From 37e0aac9713f6ce0bafd05fa999d6be276ee75b3 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Sun, 3 May 2026 15:38:18 -0400 Subject: [PATCH 3/8] ci: build prod image from .farhoodlabs/Dockerfile Pulls the prod image up to the same toolset as the dev image (kubectl, kubeseal, uv/uvx, forgejo CLIs, nano, vim) without diverging the upstream root Dockerfile. Both build-dev.yml and build-prod.yml now share the same fork-overlay Dockerfile; only the image tag and trigger branch differ. --- .farhoodlabs/.github/workflows/build-prod.yml | 1 + .github/workflows/build-prod.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.farhoodlabs/.github/workflows/build-prod.yml b/.farhoodlabs/.github/workflows/build-prod.yml index 8e187759..468a0041 100644 --- a/.farhoodlabs/.github/workflows/build-prod.yml +++ b/.farhoodlabs/.github/workflows/build-prod.yml @@ -47,6 +47,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . + file: .farhoodlabs/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-prod.yml b/.github/workflows/build-prod.yml index 8e187759..468a0041 100644 --- a/.github/workflows/build-prod.yml +++ b/.github/workflows/build-prod.yml @@ -47,6 +47,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . + file: .farhoodlabs/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} -- 2.52.0 From fccbc7e39e58d0806fc451e2a9d3101492552d7a Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Thu, 14 May 2026 10:03:30 -0400 Subject: [PATCH 4/8] feat(ci): install gitea tea CLI in fork Dockerfile Adds the official Gitea 'tea' CLI (v0.14.0) alongside the existing forgejo CLIs (fj, fj-ex, fgj). Useful when interacting with Gitea instances whose API surface is covered by tea but not by the forgejo variants. --- .farhoodlabs/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.farhoodlabs/Dockerfile b/.farhoodlabs/Dockerfile index 46b63f84..2ce01007 100644 --- a/.farhoodlabs/Dockerfile +++ b/.farhoodlabs/Dockerfile @@ -52,7 +52,7 @@ ARG USER_UID=1000 ARG USER_GID=1000 WORKDIR /app COPY --chown=node:node --from=build /app /app -# Fork additions: kubectl, kubeseal, uv, forgejo CLIs, editor tools +# Fork additions: kubectl, kubeseal, uv, forgejo CLIs, gitea tea CLI, editor tools # Upstream installs: claude-code, codex, opencode-ai, openssh-client, jq RUN apt-get update \ && apt-get install -y --no-install-recommends openssh-client jq nano vim \ @@ -71,6 +71,8 @@ RUN apt-get update \ && chmod +x /usr/local/bin/fj-ex \ && curl -fsSL https://codeberg.org/romaintb/fgj/releases/download/v0.3.0/fgj_linux_amd64 -o /usr/local/bin/fgj \ && chmod +x /usr/local/bin/fgj \ + && curl -fsSL https://dl.gitea.com/tea/0.14.0/tea-0.14.0-linux-amd64 -o /usr/local/bin/tea \ + && chmod +x /usr/local/bin/tea \ && npm install --global --omit=dev @anthropic-ai/claude-code@latest @openai/codex@latest opencode-ai \ && mkdir -p /paperclip \ && chown node:node /paperclip -- 2.52.0 From 9e854e33d908257ad096edbfaa73a21c36d57fad Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Thu, 14 May 2026 11:41:39 -0400 Subject: [PATCH 5/8] fix(skills): drop GitHub-only regex gate on PAT input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PAT input on the skill import flow was hidden by a regex that matched github.com or org/repo shorthand. Self-hosted Gitea/Forgejo/GitLab sources got no auth field at all. Always show the input when a source is entered, and label it generically ('Personal access token') instead of 'GitHub PAT'. UI only — backend already accepts any token via /skills/:id/auth and /companies/:companyId/skills POST {source, authToken}. --- ui/src/pages/CompanySkills.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/pages/CompanySkills.tsx b/ui/src/pages/CompanySkills.tsx index 0ce0c91a..ed6429be 100644 --- a/ui/src/pages/CompanySkills.tsx +++ b/ui/src/pages/CompanySkills.tsx @@ -1345,13 +1345,13 @@ export function CompanySkills() { {importSkill.isPending ? : "Add"} - {source.trim().length > 0 && /github\.com|^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_.-]+/.test(source.trim()) && ( + {source.trim().length > 0 && (
setImportAuthToken(event.target.value)} - placeholder="GitHub PAT (optional, for private repos)" + placeholder="Personal access token (optional, for private repos)" className="w-full bg-transparent text-sm outline-none placeholder:text-muted-foreground" autoComplete="off" /> -- 2.52.0 From 8dbe99e32e63b2da4c3450820bed15d291e13485 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Thu, 14 May 2026 11:49:49 -0400 Subject: [PATCH 6/8] feat(skills): support Gitea/Forgejo git hosts end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skills source pipeline was hardcoded to GitHub conventions, so even though the UI now accepts non-GitHub URLs, the server couldn't actually fetch from anywhere else. - github-fetch.ts: dispatch by host family (github.com → GitHub API + raw.githubusercontent.com; everything else → Gitea/Forgejo API v1 + /api/v1/repos/.../media for raw content). - parseGitHubSourceUrl: also accept Gitea/Forgejo web URLs (/{owner}/{repo}/src/{branch|commit|tag}/{ref}/{path}). - routes/company-skills.ts: drop the hostname='github.com' gate in deriveTrackedSkillRef so non-GitHub skills are still tracked. - Generalize user-facing strings ('GitHub PAT' → 'PAT', 'GitHub source URL' → 'Source URL', etc.). GitHub Enterprise (was assumed by '/api/v3') is no longer a special case — non-github.com hosts are treated as Gitea/Forgejo. If GHE support is needed later, add a per-source host-family override. --- server/src/routes/company-skills.ts | 10 ------- server/src/services/company-skills.ts | 43 ++++++++++++++++++--------- server/src/services/github-fetch.ts | 23 +++++++++----- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/server/src/routes/company-skills.ts b/server/src/routes/company-skills.ts index 402ffcbc..2ed51987 100644 --- a/server/src/routes/company-skills.ts +++ b/server/src/routes/company-skills.ts @@ -33,12 +33,6 @@ export function companySkillRoutes(db: Db) { return Boolean((agent.permissions as Record).canCreateAgents); } - function asString(value: unknown): string | null { - if (typeof value !== "string") return null; - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : null; - } - function deriveTrackedSkillRef(skill: SkillTelemetryInput): string | null { if (skill.sourceType === "skills_sh") { return skill.key; @@ -46,10 +40,6 @@ export function companySkillRoutes(db: Db) { if (skill.sourceType !== "github") { return null; } - const hostname = asString(skill.metadata?.hostname); - if (hostname !== "github.com") { - return null; - } return skill.key; } diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index b9238e06..51c2ad0b 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -29,7 +29,7 @@ import type { import { normalizeAgentUrlKey } from "@paperclipai/shared"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { notFound, unprocessable } from "../errors.js"; -import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js"; +import { ghFetch, gitHubApiBase, inferGitHostFamily, resolveRawGitHubUrl } from "./github-fetch.js"; import { agentService } from "./agents.js"; import { projectService } from "./projects.js"; import { secretService } from "./secrets.js"; @@ -577,7 +577,7 @@ async function resolveGitHubCommitSha(owner: string, repo: string, ref: string, ); const sha = asString(response.sha); if (!sha) { - throw unprocessable(`Failed to resolve GitHub ref ${ref}`); + throw unprocessable(`Failed to resolve ref ${ref}`); } return sha; } @@ -585,26 +585,41 @@ async function resolveGitHubCommitSha(owner: string, repo: string, ref: string, function parseGitHubSourceUrl(rawUrl: string) { const url = new URL(rawUrl); if (url.protocol !== "https:") { - throw unprocessable("GitHub source URL must use HTTPS"); + throw unprocessable("Source URL must use HTTPS"); } const parts = url.pathname.split("/").filter(Boolean); if (parts.length < 2) { - throw unprocessable("Invalid GitHub URL"); + throw unprocessable("Invalid git source URL"); } const owner = parts[0]!; const repo = parts[1]!.replace(/\.git$/i, ""); + const family = inferGitHostFamily(url.hostname); let ref = "main"; let basePath = ""; let filePath: string | null = null; let explicitRef = false; - if (parts[2] === "tree") { - ref = parts[3] ?? "main"; - basePath = parts.slice(4).join("/"); - explicitRef = true; - } else if (parts[2] === "blob") { - ref = parts[3] ?? "main"; - filePath = parts.slice(4).join("/"); - basePath = filePath ? path.posix.dirname(filePath) : ""; + if (family === "github") { + if (parts[2] === "tree") { + ref = parts[3] ?? "main"; + basePath = parts.slice(4).join("/"); + explicitRef = true; + } else if (parts[2] === "blob") { + ref = parts[3] ?? "main"; + filePath = parts.slice(4).join("/"); + basePath = filePath ? path.posix.dirname(filePath) : ""; + explicitRef = true; + } + } else if (parts[2] === "src" && (parts[3] === "branch" || parts[3] === "commit" || parts[3] === "tag")) { + // Gitea/Forgejo web URLs: /{owner}/{repo}/src/{branch|commit|tag}/{ref}/{path} + ref = parts[4] ?? "main"; + const tail = parts.slice(5); + const tailJoined = tail.join("/"); + if (tail.length > 0 && /\.[A-Za-z0-9]+$/.test(tail[tail.length - 1]!)) { + filePath = tailJoined; + basePath = path.posix.dirname(tailJoined); + } else { + basePath = tailJoined; + } explicitRef = true; } return { hostname: url.hostname, owner, repo, ref, basePath, filePath, explicitRef }; @@ -2483,7 +2498,7 @@ export function companySkillService(db: Db) { name: secretName, provider: "local_encrypted", value: authToken, - description: `GitHub PAT for skill ${skill.slug}`, + description: `PAT for skill ${skill.slug}`, }); secretId = created.id; } @@ -2590,7 +2605,7 @@ export function companySkillService(db: Db) { name: secretName, provider: "local_encrypted", value: authToken, - description: `GitHub PAT for skill ${skill.slug}`, + description: `PAT for skill ${skill.slug}`, }); secretId = created.id; } diff --git a/server/src/services/github-fetch.ts b/server/src/services/github-fetch.ts index c279ace5..66ee992f 100644 --- a/server/src/services/github-fetch.ts +++ b/server/src/services/github-fetch.ts @@ -1,19 +1,25 @@ import { unprocessable } from "../errors.js"; -function isGitHubDotCom(hostname: string) { +export type GitHostFamily = "github" | "gitea"; + +export function inferGitHostFamily(hostname: string): GitHostFamily { const h = hostname.toLowerCase(); - return h === "github.com" || h === "www.github.com"; + if (h === "github.com" || h === "www.github.com") return "github"; + return "gitea"; } export function gitHubApiBase(hostname: string) { - return isGitHubDotCom(hostname) ? "https://api.github.com" : `https://${hostname}/api/v3`; + return inferGitHostFamily(hostname) === "github" + ? "https://api.github.com" + : `https://${hostname}/api/v1`; } export function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) { const p = filePath.replace(/^\/+/, ""); - return isGitHubDotCom(hostname) - ? `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}` - : `https://${hostname}/raw/${owner}/${repo}/${ref}/${p}`; + if (inferGitHostFamily(hostname) === "github") { + return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}`; + } + return `https://${hostname}/api/v1/repos/${owner}/${repo}/media/${p}?ref=${encodeURIComponent(ref)}`; } export async function ghFetch(url: string, init?: RequestInit, authToken?: string): Promise { @@ -24,6 +30,9 @@ export async function ghFetch(url: string, init?: RequestInit, authToken?: strin try { return await fetch(url, { ...init, headers, redirect: authToken ? "manual" : "follow" }); } catch { - throw unprocessable(`Could not connect to ${new URL(url).hostname} — ensure the URL points to a GitHub or GitHub Enterprise instance`); + const hostname = (() => { + try { return new URL(url).hostname; } catch { return url; } + })(); + throw unprocessable(`Could not connect to ${hostname}`); } } -- 2.52.0 From 80f7d8270c04355a3aca164e8b50258c244807ee Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Sat, 16 May 2026 10:28:22 -0400 Subject: [PATCH 7/8] refactor(portability): migrate to git-source; delete github-fetch.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the skills refactor: company-portability was the second user of the per-host REST shim (its own parallel parseGitHubSourceUrl + fetch helpers + raw.githubusercontent URL builder), so importing a company package from a non-github URL hit the same Gitea 404 the skills path did. - Extend git-source.ts: - parseGitSourceUrl: also recognises query-string shape (?ref=...&path=...) used by portability URLs, with precedence over path-style segments when both are present. - RepoSnapshot: add readBinary (Uint8Array for the company logo fetch) and readFileOptional (null on NotFoundError, for the COMPANY.md probe + main->master fallback). - Rewrite resolveSource in company-portability.ts to open a single in-memory snapshot per import and serve all reads (COMPANY.md, candidate tree, includes, logo) from it. Drops fetchText/fetchJson/ fetchBinary/fetchOptionalText. - parseGitHubSourceUrl stays exported with its original return shape ({hostname, owner, repo, ref, basePath, companyPath}) so the existing test suite passes unchanged. It now delegates URL parsing to parseGitSourceUrl and layers companyPath derivation on top. - Delete server/src/services/github-fetch.ts: zero remaining callers. Test coverage: - 7 new git-source tests (query-string parse variants, query-string precedence over path style, readBinary, readFileOptional NotFound null + non-NotFound rethrow) — 34/34 passing. - 52 existing company-portability tests still pass via the parseGitHubSourceUrl shim contract. - Smoke-tested end-to-end against https://git.farh.net/.../?ref=main: ref resolves, snapshot opens, readFile/readBinary/readFileOptional all return expected results. Note: two pre-existing failures in company-skills-routes.test.ts ("does not expose a skill reference...") exist on dev too and are unrelated to this change. Co-Authored-By: Claude Opus 4.7 (1M context) --- server/src/__tests__/git-source.test.ts | 74 +++++++++ server/src/services/company-portability.ts | 183 ++++++++------------- server/src/services/git-source.ts | 43 ++++- server/src/services/github-fetch.ts | 38 ----- 4 files changed, 180 insertions(+), 158 deletions(-) delete mode 100644 server/src/services/github-fetch.ts diff --git a/server/src/__tests__/git-source.test.ts b/server/src/__tests__/git-source.test.ts index 4ba1e335..ce257817 100644 --- a/server/src/__tests__/git-source.test.ts +++ b/server/src/__tests__/git-source.test.ts @@ -145,6 +145,43 @@ describe("parseGitSourceUrl", () => { it("rejects malformed URLs", () => { expect(() => parseGitSourceUrl("not a url")).toThrow(); }); + + it("parses a query-string URL with ?ref= and ?path=", () => { + expect( + parseGitSourceUrl("https://github.com/o/r?ref=feature%2Fdemo&path=subdir"), + ).toMatchObject({ + cloneUrl: "https://github.com/o/r.git", + ref: "feature/demo", + basePath: "subdir", + filePath: null, + explicitRef: true, + }); + }); + + it("parses a query-string URL with only ?ref=", () => { + expect(parseGitSourceUrl("https://github.com/o/r?ref=develop")).toMatchObject({ + ref: "develop", + basePath: "", + explicitRef: true, + }); + }); + + it("parses a query-string URL with only ?path=", () => { + expect(parseGitSourceUrl("https://github.com/o/r?path=sub")).toMatchObject({ + ref: null, + basePath: "sub", + explicitRef: false, + }); + }); + + it("query-string parsing takes precedence over path-style segments", () => { + expect( + parseGitSourceUrl("https://github.com/o/r/tree/main/old?ref=newref&path=newpath"), + ).toMatchObject({ + ref: "newref", + basePath: "newpath", + }); + }); }); describe("buildCloneUrl", () => { @@ -333,4 +370,41 @@ describe("openRepoSnapshot", () => { openRepoSnapshot(parsed, "main", "1111111111111111111111111111111111111111"), ).rejects.toThrow(/repository not found/i); }); + + it("readBinary returns the raw blob bytes", async () => { + cloneFn.mockResolvedValue(undefined); + resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff"); + walkFn.mockImplementation(async () => {}); + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); + readBlobFn.mockResolvedValue({ blob: bytes }); + + const parsed = parseGitSourceUrl("https://git.example.com/o/r"); + const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff"); + const result = await snap.readBinary("logo.png"); + expect(result).toBe(bytes); + }); + + it("readFileOptional returns null on NotFoundError", async () => { + cloneFn.mockResolvedValue(undefined); + resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff"); + walkFn.mockImplementation(async () => {}); + const err = Object.assign(new Error("missing"), { code: "NotFoundError" }); + readBlobFn.mockRejectedValue(err); + + const parsed = parseGitSourceUrl("https://git.example.com/o/r"); + const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff"); + const result = await snap.readFileOptional("missing.md"); + expect(result).toBeNull(); + }); + + it("readFileOptional rethrows non-NotFound errors", async () => { + cloneFn.mockResolvedValue(undefined); + resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff"); + walkFn.mockImplementation(async () => {}); + readBlobFn.mockRejectedValue(new Error("disk explosion")); + + const parsed = parseGitSourceUrl("https://git.example.com/o/r"); + const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff"); + await expect(snap.readFileOptional("any.md")).rejects.toThrow(/disk explosion/); + }); }); diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index 864f773a..30e4c14f 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -57,7 +57,7 @@ import { import { requireOpenCodeModelId } from "@paperclipai/adapter-opencode-local/server"; import { findServerAdapter } from "../adapters/index.js"; import { forbidden, HttpError, notFound, unprocessable } from "../errors.js"; -import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js"; +import { openRepoSnapshot, parseGitSourceUrl, resolveGitRef, type RepoSnapshot } from "./git-source.js"; import type { StorageService } from "../storage/types.js"; import { accessService } from "./access.js"; import { agentService } from "./agents.js"; @@ -2339,42 +2339,6 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc { }; } -async function fetchText(url: string) { - const response = await ghFetch(url); - if (!response.ok) { - throw unprocessable(`Failed to fetch ${url}: ${response.status}`); - } - return response.text(); -} - -async function fetchOptionalText(url: string) { - const response = await ghFetch(url); - if (response.status === 404) return null; - if (!response.ok) { - throw unprocessable(`Failed to fetch ${url}: ${response.status}`); - } - return response.text(); -} - -async function fetchBinary(url: string) { - const response = await ghFetch(url); - if (!response.ok) { - throw unprocessable(`Failed to fetch ${url}: ${response.status}`); - } - return Buffer.from(await response.arrayBuffer()); -} - -async function fetchJson(url: string): Promise { - const response = await ghFetch(url, { - headers: { - accept: "application/vnd.github+json", - }, - }); - if (!response.ok) { - throw unprocessable(`Failed to fetch ${url}: ${response.status}`); - } - return response.json() as Promise; -} function dedupeEnvInputs(values: CompanyPortabilityManifest["envInputs"]) { const seen = new Set(); @@ -2864,52 +2828,37 @@ function normalizeGitHubSourcePath(value: string | null | undefined) { export function parseGitHubSourceUrl(rawUrl: string) { const url = new URL(rawUrl); - if (url.protocol !== "https:") { - throw unprocessable("GitHub source URL must use HTTPS"); - } - const hostname = url.hostname; - const parts = url.pathname.split("/").filter(Boolean); - if (parts.length < 2) { - throw unprocessable("Invalid GitHub URL"); - } - const owner = parts[0]!; - const repo = parts[1]!.replace(/\.git$/i, ""); - const queryRef = url.searchParams.get("ref")?.trim(); - const queryPath = normalizeGitHubSourcePath(url.searchParams.get("path")); + // Handle the portability-specific companyPath query param before delegating, + // since git-source has no notion of it. const queryCompanyPath = normalizeGitHubSourcePath(url.searchParams.get("companyPath")); - if (queryRef || queryPath || queryCompanyPath) { - const companyPath = queryCompanyPath || [queryPath, "COMPANY.md"].filter(Boolean).join("/") || "COMPANY.md"; - let basePath = queryPath; - if (!basePath && companyPath !== "COMPANY.md") { - basePath = path.posix.dirname(companyPath); - if (basePath === ".") basePath = ""; + + const parsed = parseGitSourceUrl(rawUrl); + + let companyPath: string; + let basePath = parsed.basePath; + if (queryCompanyPath) { + companyPath = queryCompanyPath; + if (!basePath) { + const derived = path.posix.dirname(companyPath); + basePath = derived === "." ? "" : derived; } - return { - hostname, - owner, - repo, - ref: queryRef || "main", - basePath, - companyPath, - }; + } else if (parsed.filePath) { + // blob-style URL pointed directly at a file + companyPath = parsed.filePath; + } else if (basePath) { + companyPath = `${basePath}/COMPANY.md`; + } else { + companyPath = "COMPANY.md"; } - let ref = "main"; - let basePath = ""; - let companyPath = "COMPANY.md"; - if (parts[2] === "tree") { - ref = parts[3] ?? "main"; - basePath = parts.slice(4).join("/"); - } else if (parts[2] === "blob") { - ref = parts[3] ?? "main"; - const blobPath = parts.slice(4).join("/"); - if (!blobPath) { - throw unprocessable("Invalid GitHub blob URL"); - } - companyPath = blobPath; - basePath = path.posix.dirname(blobPath); - if (basePath === ".") basePath = ""; - } - return { hostname, owner, repo, ref, basePath, companyPath }; + + return { + hostname: parsed.hostname, + owner: parsed.owner, + repo: parsed.repo, + ref: parsed.ref ?? "main", + basePath, + companyPath, + }; } @@ -3013,30 +2962,38 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { ); } - const parsed = parseGitHubSourceUrl(source.url); - let ref = parsed.ref; + const sourceUrl = source.url; + const parsed = parseGitHubSourceUrl(sourceUrl); const warnings: string[] = []; const companyRelativePath = parsed.companyPath === "COMPANY.md" ? [parsed.basePath, "COMPANY.md"].filter(Boolean).join("/") : parsed.companyPath; + + async function openSnapshot(refName: string): Promise { + const ps = parseGitSourceUrl(sourceUrl); + const wanted = { ...ps, ref: refName, explicitRef: true }; + const resolved = await resolveGitRef(wanted); + return openRepoSnapshot(wanted, resolved.trackingRef, resolved.pinnedSha); + } + + let ref = parsed.ref; + let snapshot: RepoSnapshot; let companyMarkdown: string | null = null; try { - companyMarkdown = await fetchOptionalText( - resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath), - ); + snapshot = await openSnapshot(ref); + companyMarkdown = await snapshot.readFileOptional(companyRelativePath); } catch (err) { if (ref === "main") { ref = "master"; - warnings.push("GitHub ref main not found; falling back to master."); - companyMarkdown = await fetchOptionalText( - resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath), - ); + warnings.push("Git ref main not found; falling back to master."); + snapshot = await openSnapshot(ref); + companyMarkdown = await snapshot.readFileOptional(companyRelativePath); } else { throw err; } } if (!companyMarkdown) { - throw unprocessable("GitHub company package is missing COMPANY.md"); + throw unprocessable("Git company package is missing COMPANY.md"); } const companyPath = parsed.companyPath === "COMPANY.md" @@ -3045,31 +3002,22 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const files: Record = { [companyPath]: companyMarkdown, }; - const apiBase = gitHubApiBase(parsed.hostname); - const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>( - `${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`, - ).catch(() => ({ tree: [] })); const basePrefix = parsed.basePath ? `${parsed.basePath.replace(/^\/+|\/+$/g, "")}/` : ""; - const candidatePaths = (tree.tree ?? []) - .filter((entry) => entry.type === "blob") - .map((entry) => entry.path) - .filter((entry): entry is string => typeof entry === "string") - .filter((entry) => { - if (basePrefix && !entry.startsWith(basePrefix)) return false; - const relative = basePrefix ? entry.slice(basePrefix.length) : entry; - return ( - relative.endsWith(".md") || - relative.startsWith("skills/") || - relative === ".paperclip.yaml" || - relative === ".paperclip.yml" - ); - }); + const allPaths = await snapshot.listFiles(); + const candidatePaths = allPaths.filter((entry) => { + if (basePrefix && !entry.startsWith(basePrefix)) return false; + const relative = basePrefix ? entry.slice(basePrefix.length) : entry; + return ( + relative.endsWith(".md") || + relative.startsWith("skills/") || + relative === ".paperclip.yaml" || + relative === ".paperclip.yml" + ); + }); for (const repoPath of candidatePaths) { const relativePath = basePrefix ? repoPath.slice(basePrefix.length) : repoPath; if (files[relativePath] !== undefined) continue; - files[normalizePortablePath(relativePath)] = await fetchText( - resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), - ); + files[normalizePortablePath(relativePath)] = await snapshot.readFile(repoPath); } const companyDoc = parseFrontmatterMarkdown(companyMarkdown); const includeEntries = readIncludeEntries(companyDoc.frontmatter); @@ -3078,9 +3026,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const relativePath = normalizePortablePath(includeEntry.path); if (files[relativePath] !== undefined) continue; if (!(repoPath.endsWith(".md") || repoPath.endsWith(".yaml") || repoPath.endsWith(".yml"))) continue; - files[relativePath] = await fetchText( - resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), - ); + files[relativePath] = await snapshot.readFile(repoPath); } const resolved = buildManifestFromPackageFiles(files); @@ -3088,12 +3034,13 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (companyLogoPath && !resolved.files[companyLogoPath]) { const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/"); try { - const binary = await fetchBinary( - resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), + const binary = await snapshot.readBinary(repoPath); + resolved.files[companyLogoPath] = bufferToPortableBinaryFile( + Buffer.from(binary), + inferContentTypeFromPath(companyLogoPath), ); - resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath)); } catch (err) { - warnings.push(`Failed to fetch company logo ${companyLogoPath} from GitHub: ${err instanceof Error ? err.message : String(err)}`); + warnings.push(`Failed to fetch company logo ${companyLogoPath} from git: ${err instanceof Error ? err.message : String(err)}`); } } resolved.warnings.unshift(...warnings); diff --git a/server/src/services/git-source.ts b/server/src/services/git-source.ts index f52dce46..c67cadcd 100644 --- a/server/src/services/git-source.ts +++ b/server/src/services/git-source.ts @@ -25,6 +25,8 @@ export type RepoSnapshot = { sha: string; listFiles(): Promise; readFile(repoPath: string): Promise; + readFileOptional(repoPath: string): Promise; + readBinary(repoPath: string): Promise; }; const SHA_REGEX = /^[0-9a-f]{40}$/i; @@ -50,6 +52,25 @@ export function parseGitSourceUrl(rawUrl: string): ParsedGitSource { const owner = segments[0]!; const repo = segments[1]!.replace(/\.git$/i, ""); + // Query-string shape: /{owner}/{repo}?ref=...&path=... + // Used by company portability URLs. Takes precedence over path-based parsing + // so a URL with both shapes (rare) prefers the explicit query params. + const queryRef = url.searchParams.get("ref")?.trim() ?? null; + const queryPath = url.searchParams.get("path")?.trim() ?? null; + if (queryRef || queryPath) { + const normalizedPath = (queryPath ?? "").replace(/\\/g, "/").replace(/^\/+|\/+$/g, ""); + return { + cloneUrl: buildCloneUrl(url.hostname, owner, repo), + hostname: url.hostname, + owner, + repo, + ref: queryRef || null, + basePath: normalizedPath, + filePath: null, + explicitRef: Boolean(queryRef), + }; + } + let ref: string | null = null; let basePath = ""; let filePath: string | null = null; @@ -233,11 +254,29 @@ export async function openRepoSnapshot( return out; } - async function readFile(repoPath: string): Promise { + async function readBinary(repoPath: string): Promise { const normalized = repoPath.replace(/^\/+/, ""); const { blob } = await git.readBlob({ fs, dir, oid: sha, filepath: normalized }); + return blob; + } + + async function readFile(repoPath: string): Promise { + const blob = await readBinary(repoPath); return new TextDecoder("utf-8").decode(blob); } - return { sha, listFiles, readFile }; + async function readFileOptional(repoPath: string): Promise { + try { + return await readFile(repoPath); + } catch (err) { + // isomorphic-git throws NotFoundError when the path is missing from the tree. + const name = (err as { code?: string; name?: string } | null)?.code + ?? (err as { name?: string } | null)?.name + ?? ""; + if (/NotFound/i.test(name)) return null; + throw err; + } + } + + return { sha, listFiles, readFile, readFileOptional, readBinary }; } diff --git a/server/src/services/github-fetch.ts b/server/src/services/github-fetch.ts deleted file mode 100644 index 66ee992f..00000000 --- a/server/src/services/github-fetch.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { unprocessable } from "../errors.js"; - -export type GitHostFamily = "github" | "gitea"; - -export function inferGitHostFamily(hostname: string): GitHostFamily { - const h = hostname.toLowerCase(); - if (h === "github.com" || h === "www.github.com") return "github"; - return "gitea"; -} - -export function gitHubApiBase(hostname: string) { - return inferGitHostFamily(hostname) === "github" - ? "https://api.github.com" - : `https://${hostname}/api/v1`; -} - -export function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) { - const p = filePath.replace(/^\/+/, ""); - if (inferGitHostFamily(hostname) === "github") { - return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}`; - } - return `https://${hostname}/api/v1/repos/${owner}/${repo}/media/${p}?ref=${encodeURIComponent(ref)}`; -} - -export async function ghFetch(url: string, init?: RequestInit, authToken?: string): Promise { - const headers = new Headers(init?.headers); - if (authToken) { - headers.set("Authorization", `Bearer ${authToken}`); - } - try { - return await fetch(url, { ...init, headers, redirect: authToken ? "manual" : "follow" }); - } catch { - const hostname = (() => { - try { return new URL(url).hostname; } catch { return url; } - })(); - throw unprocessable(`Could not connect to ${hostname}`); - } -} -- 2.52.0 From bf251188dffa8989dc933279fd3c43f5cd40b231 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Sat, 16 May 2026 10:35:56 -0400 Subject: [PATCH 8/8] test(portability): cover resolveSource orchestration via previewImport Closes the coverage gap on the actual migrated function. Mocks the two network-touching git-source exports (resolveGitRef, openRepoSnapshot) while keeping parseGitSourceUrl real so the parseGitHubSourceUrl shim contract stays honest. Adds 5 cases: - happy path: opens one snapshot, calls listFiles, readFileOptional on COMPANY.md, readFile on candidate paths - ref fallback: when openRepoSnapshot('main') rejects, falls back to 'master' and emits the expected warning - COMPANY.md absent everywhere: throws "missing COMPANY.md" - referenced logo: readBinary is called for the logoPath from .paperclip.yaml - logo read failure: warning emitted, no throw 57/57 portability tests passing; existing 52 unchanged via shim. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/__tests__/company-portability.test.ts | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index 3a258263..bcdadea5 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -137,6 +137,25 @@ vi.mock("../routes/org-chart-svg.js", () => ({ renderOrgChartPng: vi.fn(async () => Buffer.from("png")), })); +const gitSourceMock = vi.hoisted(() => ({ + resolveGitRef: vi.fn(), + openRepoSnapshot: vi.fn(), +})); + +// parseGitSourceUrl stays real (the shim parseGitHubSourceUrl delegates to it +// and is asserted by existing tests). Only the network-touching functions are +// overridable per-test. +vi.mock("../services/git-source.js", async () => { + const actual = await vi.importActual( + "../services/git-source.js", + ); + return { + ...actual, + resolveGitRef: gitSourceMock.resolveGitRef, + openRepoSnapshot: gitSourceMock.openRepoSnapshot, + }; +}); + const { companyPortabilityService, parseGitHubSourceUrl } = await import("../services/company-portability.js"); function asTextFile(entry: CompanyPortabilityFileEntry | undefined) { @@ -3378,3 +3397,173 @@ describe("company portability", () => { expect(preview.plan.issuePlans).toHaveLength(0); }); }); + +describe("git source orchestration via resolveSource", () => { + const minimalCompanyMarkdown = "---\ncompany:\n name: Demo\n---\n# Demo\n"; + const githubUrl = "https://git.example.com/acme/co?ref=main&path="; + + function makeSnapshot(overrides: { + files?: string[]; + fileContents?: Record; + binaryContents?: Record; + readBinaryReject?: Error; + } = {}) { + const files = overrides.files ?? ["COMPANY.md"]; + const fileContents = overrides.fileContents ?? { "COMPANY.md": minimalCompanyMarkdown }; + const binaryContents = overrides.binaryContents ?? {}; + return { + sha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + listFiles: vi.fn(async () => files), + readFile: vi.fn(async (p: string) => { + if (p in fileContents) return fileContents[p]; + throw Object.assign(new Error(`not found: ${p}`), { code: "NotFoundError" }); + }), + readFileOptional: vi.fn(async (p: string) => fileContents[p] ?? null), + readBinary: vi.fn(async (p: string) => { + if (overrides.readBinaryReject) throw overrides.readBinaryReject; + if (p in binaryContents) return binaryContents[p]!; + throw Object.assign(new Error(`not found: ${p}`), { code: "NotFoundError" }); + }), + }; + } + + function setupResolveStub() { + gitSourceMock.resolveGitRef.mockResolvedValue({ + pinnedSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + trackingRef: "main", + }); + } + + beforeEach(() => { + gitSourceMock.resolveGitRef.mockReset(); + gitSourceMock.openRepoSnapshot.mockReset(); + companySvc.getById.mockResolvedValue(null); + agentSvc.list.mockResolvedValue([]); + projectSvc.list.mockResolvedValue([]); + issueSvc.list.mockResolvedValue([]); + issueSvc.listComments.mockResolvedValue([]); + companySkillSvc.list.mockResolvedValue([]); + }); + + it("opens a snapshot and walks the tree for a github source", async () => { + setupResolveStub(); + const snapshot = makeSnapshot({ + files: ["COMPANY.md", "README.md", "skills/x/SKILL.md"], + fileContents: { + "COMPANY.md": minimalCompanyMarkdown, + "README.md": "# readme", + "skills/x/SKILL.md": "---\nname: x\n---\n", + }, + }); + gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot); + + const portability = companyPortabilityService({} as any); + const preview = await portability.previewImport({ + source: { type: "github", url: githubUrl }, + include: { company: true, agents: false, projects: false, issues: false, skills: false }, + target: { mode: "new_company", newCompanyName: "Demo" }, + agents: "all", + collisionStrategy: "rename", + }); + + expect(gitSourceMock.resolveGitRef).toHaveBeenCalledTimes(1); + expect(gitSourceMock.openRepoSnapshot).toHaveBeenCalledTimes(1); + expect(snapshot.listFiles).toHaveBeenCalled(); + expect(snapshot.readFileOptional).toHaveBeenCalledWith("COMPANY.md"); + expect(snapshot.readFile).toHaveBeenCalledWith("README.md"); + expect(snapshot.readFile).toHaveBeenCalledWith("skills/x/SKILL.md"); + expect(preview.errors).toEqual([]); + }); + + it("falls back from main to master when the main ref does not exist", async () => { + setupResolveStub(); + const masterSnap = makeSnapshot(); + // First call (ref=main) rejects; second (ref=master) succeeds. + gitSourceMock.openRepoSnapshot + .mockRejectedValueOnce(new Error("ref not found")) + .mockResolvedValueOnce(masterSnap); + + const portability = companyPortabilityService({} as any); + const preview = await portability.previewImport({ + source: { type: "github", url: githubUrl }, + include: { company: true, agents: false, projects: false, issues: false, skills: false }, + target: { mode: "new_company", newCompanyName: "Demo" }, + agents: "all", + collisionStrategy: "rename", + }); + + expect(gitSourceMock.openRepoSnapshot).toHaveBeenCalledTimes(2); + expect(masterSnap.readFileOptional).toHaveBeenCalledWith("COMPANY.md"); + expect(preview.warnings).toContain("Git ref main not found; falling back to master."); + }); + + it("throws when COMPANY.md is missing on both main and master", async () => { + setupResolveStub(); + const emptySnap = makeSnapshot({ fileContents: {} }); + gitSourceMock.openRepoSnapshot.mockResolvedValue(emptySnap); + + const portability = companyPortabilityService({} as any); + await expect( + portability.previewImport({ + source: { type: "github", url: githubUrl }, + include: { company: true, agents: false, projects: false, issues: false, skills: false }, + target: { mode: "new_company", newCompanyName: "Demo" }, + agents: "all", + collisionStrategy: "rename", + }), + ).rejects.toThrow(/missing COMPANY.md/i); + }); + + it("fetches a referenced company logo as binary", async () => { + setupResolveStub(); + // logoPath lives in .paperclip.yaml (paperclip extension), not COMPANY.md. + const paperclipYaml = "company:\n logoPath: images/logo.png\n"; + const logoBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); + const snapshot = makeSnapshot({ + files: ["COMPANY.md", ".paperclip.yaml", "images/logo.png"], + fileContents: { + "COMPANY.md": minimalCompanyMarkdown, + ".paperclip.yaml": paperclipYaml, + }, + binaryContents: { "images/logo.png": logoBytes }, + }); + gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot); + + const portability = companyPortabilityService({} as any); + await portability.previewImport({ + source: { type: "github", url: githubUrl }, + include: { company: true, agents: false, projects: false, issues: false, skills: false }, + target: { mode: "new_company", newCompanyName: "Demo" }, + agents: "all", + collisionStrategy: "rename", + }); + + expect(snapshot.readBinary).toHaveBeenCalledWith("images/logo.png"); + }); + + it("warns instead of throwing when the logo blob can't be read", async () => { + setupResolveStub(); + const paperclipYaml = "company:\n logoPath: images/logo.png\n"; + const snapshot = makeSnapshot({ + files: ["COMPANY.md", ".paperclip.yaml"], + fileContents: { + "COMPANY.md": minimalCompanyMarkdown, + ".paperclip.yaml": paperclipYaml, + }, + readBinaryReject: new Error("blob missing"), + }); + gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot); + + const portability = companyPortabilityService({} as any); + const preview = await portability.previewImport({ + source: { type: "github", url: githubUrl }, + include: { company: true, agents: false, projects: false, issues: false, skills: false }, + target: { mode: "new_company", newCompanyName: "Demo" }, + agents: "all", + collisionStrategy: "rename", + }); + + expect(snapshot.readBinary).toHaveBeenCalled(); + expect(preview.warnings.some((w: string) => /Failed to fetch company logo/i.test(w))).toBe(true); + }); +}); -- 2.52.0