forked from farhoodlabs/paperclip
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Chris Farhood
|
5499a0b4a6 | ||
|
Chris Farhood
|
55faea456f | ||
|
Chris Farhood
|
329ba3fd2e | ||
|
Chris Farhood
|
bf251188df | ||
|
Chris Farhood
|
80f7d8270c | ||
|
Chris Farhood
|
5703fa225c | ||
|
Chris Farhood
|
4317d2a3b4 | ||
|
Chris Farhood
|
8dbe99e32e | ||
|
Chris Farhood
|
9e854e33d9 | ||
|
Chris Farhood
|
fccbc7e39e | ||
|
Chris Farhood
|
7a8afbb719 | ||
|
Chris Farhood
|
30ef61bb25 | ||
|
Chris Farhood
|
37e0aac971 | ||
|
Chris Farhood
|
cee1cd7f4e | ||
|
Chris Farhood
|
85cbbc9263 | ||
|
Chris Farhood
|
acbfcb7d00 | ||
|
Chris Farhood
|
3bbd632355 | ||
|
Chris Farhood
|
e37180d3e3 |
+11
-28
@@ -11,7 +11,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
image-tag: ${{ steps.tag.outputs.sha }}
|
||||
@@ -23,28 +23,21 @@ jobs:
|
||||
id: tag
|
||||
run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Login to Gitea Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
registry: git.farh.net
|
||||
username: ${{ gitea.repository_owner }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/farhoodlabs/paperclip-dev
|
||||
images: git.farh.net/farhoodlabs/paperclip-dev
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha,prefix=
|
||||
@@ -62,25 +55,16 @@ jobs:
|
||||
|
||||
update-infra:
|
||||
needs: build
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate app token
|
||||
id: app-token
|
||||
uses: actions/create-github-app-token@v1
|
||||
with:
|
||||
app-id: ${{ secrets.PAPERCLIP_APP_ID }}
|
||||
private-key: ${{ secrets.PAPERCLIP_APP_PRIVATE_KEY }}
|
||||
repositories: paperclip-infra
|
||||
|
||||
- name: Update dev image tag in infra repo
|
||||
run: |
|
||||
SHA="${{ needs.build.outputs.image-tag }}"
|
||||
FILE="overlays/dev/kustomization.yaml"
|
||||
|
||||
response=$(curl -sS \
|
||||
-H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/farhoodlabs/paperclip-infra/contents/$FILE")
|
||||
-H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
|
||||
"https://git.farh.net/api/v1/repos/farhoodlabs/paperclip-infra/contents/$FILE")
|
||||
|
||||
file_sha=$(echo "$response" | jq -r '.sha')
|
||||
content=$(echo "$response" | jq -r '.content' | base64 -d)
|
||||
@@ -88,7 +72,6 @@ jobs:
|
||||
encoded=$(printf '%s' "$new_content" | base64 -w 0)
|
||||
|
||||
curl -sS -X PUT \
|
||||
-H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/farhoodlabs/paperclip-infra/contents/$FILE" \
|
||||
-H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
|
||||
"https://git.farh.net/api/v1/repos/farhoodlabs/paperclip-infra/contents/$FILE" \
|
||||
-d "{\"message\":\"chore(cd): update paperclip-dev to $SHA\",\"content\":\"$encoded\",\"sha\":\"$file_sha\"}"
|
||||
|
||||
+7
-12
@@ -11,33 +11,27 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Login to Gitea Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
registry: git.farh.net
|
||||
username: ${{ gitea.repository_owner }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/farhoodlabs/paperclip
|
||||
images: git.farh.net/farhoodlabs/paperclip
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha,prefix=
|
||||
@@ -47,6 +41,7 @@ jobs:
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: .farhoodlabs/Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# Paperclip Fork — Project Context
|
||||
|
||||
This is a fork of [paperclipai/paperclip](https://github.com/paperclipai/paperclip).
|
||||
Fork repo: https://github.com/farhoodlabs/paperclip
|
||||
Fork repo: https://git.farh.net/farhoodlabs/paperclip
|
||||
|
||||
## Branch Model
|
||||
|
||||
| Branch | Purpose |
|
||||
|---|---|
|
||||
| `master` | Mirrors `upstream/master` exactly + `.farhoodlabs/` overlay directory + `assemble-local.yml` action. Never commit application code here. |
|
||||
| `local` | **Default branch.** Assembled automatically by `assemble-local.yml` on every `master` push. Contains: upstream + fork Dockerfile/workflows + all pending upstream PR cherry-picks. Builds `ghcr.io/farhoodlabs/paperclip`. |
|
||||
| `dev` | Development branch based on upstream/master. Builds `ghcr.io/farhoodlabs/paperclip-dev` on every push. |
|
||||
| `local` | **Default branch.** Assembled automatically by `assemble-local.yml` on every `master` push. Contains: upstream + fork Dockerfile/workflows + all pending upstream PR cherry-picks. Builds `git.farh.net/farhoodlabs/paperclip`. |
|
||||
| `dev` | Development branch based on upstream/master. Builds `git.farh.net/farhoodlabs/paperclip-dev` on every push. |
|
||||
| PR branches | `skill-pat-feature`, `skill-scan-refresh`, `feat/company-portability-complete` — open PRs to upstream, never rebase onto master/local. |
|
||||
|
||||
**Never commit directly to `local`** — it is fully regenerated by the assemble action and any direct commits will be overwritten.
|
||||
@@ -70,7 +70,7 @@ Edit `.farhoodlabs/Dockerfile` on `master`. Only modify the production stage —
|
||||
|
||||
## Deployment
|
||||
|
||||
Paperclip runs in Kubernetes, not locally. Use `kubectl` to access it. The production image is `ghcr.io/farhoodlabs/paperclip:latest`.
|
||||
Paperclip runs in Kubernetes, not locally. Use `kubectl` to access it. The production image is `git.farh.net/farhoodlabs/paperclip:latest`.
|
||||
|
||||
## Key Files
|
||||
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
name: Assemble local branch
|
||||
|
||||
# Triggers on every master push (i.e. after syncing upstream) and on demand.
|
||||
# Builds the `local` branch: master + fork overlay + cherry-picked pending upstream PRs.
|
||||
# Syncs build-dev.yml to the `dev` branch so every dev push triggers a build.
|
||||
#
|
||||
# PR entries support an optional "exclude:BRANCH" suffix to handle cases where
|
||||
# one PR branch was rebased onto another. The exclude branch's commits are subtracted
|
||||
# from the cherry-pick range so they aren't double-applied.
|
||||
#
|
||||
# When upstream merges a PR, remove its entry from PR_CHERRY_PICK or PR_SQUASH below.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
assemble:
|
||||
runs-on: runners-farhoodlabs
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Configure git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Fetch all remotes
|
||||
run: |
|
||||
git remote add upstream https://github.com/paperclipai/paperclip.git 2>/dev/null || true
|
||||
git fetch --all --quiet
|
||||
|
||||
- name: Assemble local branch
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Start local from master (which mirrors upstream)
|
||||
git checkout -B local origin/master
|
||||
|
||||
# Apply fork overlay: Dockerfile, build workflows, CLAUDE.md
|
||||
cp .farhoodlabs/Dockerfile Dockerfile
|
||||
cp .farhoodlabs/CLAUDE.md CLAUDE.md
|
||||
mkdir -p .github/workflows
|
||||
cp .farhoodlabs/.github/workflows/build-prod.yml .github/workflows/build-prod.yml
|
||||
cp .farhoodlabs/.github/workflows/build-dev.yml .github/workflows/build-dev.yml
|
||||
git add Dockerfile CLAUDE.md .github/workflows/build-prod.yml .github/workflows/build-dev.yml
|
||||
git commit -m "chore: apply fork overlay from .farhoodlabs"
|
||||
|
||||
# --- PRs to cherry-pick commit-by-commit (clean, no merge commits) ---
|
||||
# Format: "PR-number branch-name [exclude:base-branch]"
|
||||
# Use exclude: when a branch was rebased onto another PR branch to avoid double-applying commits.
|
||||
# Remove an entry here when upstream merges the PR.
|
||||
PR_CHERRY_PICK=(
|
||||
"3237 skill-pat-feature"
|
||||
"3351 skill-scan-refresh exclude:skill-pat-feature"
|
||||
"4162 fix/far-108-k8s-adapter-reaper-liveness"
|
||||
)
|
||||
|
||||
for entry in "${PR_CHERRY_PICK[@]}"; do
|
||||
# Parse: pr_num, branch, optional exclude branch
|
||||
pr_num=$(echo "$entry" | awk '{print $1}')
|
||||
branch=$(echo "$entry" | awk '{print $2}')
|
||||
exclude_branch=$(echo "$entry" | grep -oP '(?<=exclude:)\S+' || true)
|
||||
remote_branch="origin/$branch"
|
||||
exclude_arg=""
|
||||
if [ -n "$exclude_branch" ]; then
|
||||
exclude_arg="--not origin/$exclude_branch"
|
||||
fi
|
||||
|
||||
if ! git rev-parse "$remote_branch" &>/dev/null; then
|
||||
echo "WARNING: $remote_branch not found, skipping PR #$pr_num"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Exclude commits already on origin/master (fork-overlay/CI infra
|
||||
# that landed on master via the .farhoodlabs/ overlay path). PR
|
||||
# branches sometimes pull these in via `git merge origin/master`,
|
||||
# but cherry-picking them onto `local` (which is already master)
|
||||
# is redundant and produces conflicts on the assemble-local file.
|
||||
mapfile -t commits < <(git log --no-merges --reverse --format="%H" upstream/master.."$remote_branch" ^origin/master $exclude_arg)
|
||||
|
||||
if [ ${#commits[@]} -eq 0 ]; then
|
||||
echo "PR #$pr_num ($branch): no unique commits — likely merged upstream, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "PR #$pr_num ($branch): cherry-picking ${#commits[@]} commit(s)"
|
||||
for sha in "${commits[@]}"; do
|
||||
git cherry-pick "$sha" || {
|
||||
# If the cherry-pick produced an empty result (commit's content
|
||||
# is already in HEAD via auto-merge), skip it instead of failing.
|
||||
# State signature: CHERRY_PICK_HEAD set, no unmerged paths,
|
||||
# nothing staged.
|
||||
if [ -f .git/CHERRY_PICK_HEAD ] \
|
||||
&& [ -z "$(git diff --name-only --diff-filter=U)" ] \
|
||||
&& git diff --staged --quiet; then
|
||||
echo "PR #$pr_num: $sha became empty after merge, skipping"
|
||||
git cherry-pick --skip
|
||||
continue
|
||||
fi
|
||||
echo "::error::Cherry-pick conflict at $sha from PR #$pr_num ($branch)"
|
||||
echo "::error::Resolve the conflict, force-push the branch, then re-run this workflow"
|
||||
git cherry-pick --abort
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
done
|
||||
|
||||
# --- PRs to apply as a single squash (complex history with merge commits) ---
|
||||
# git merge --squash applies the net final diff of the branch, bypassing
|
||||
# intra-PR commit ordering issues. CI commits that cancel out are ignored.
|
||||
# Remove an entry here when upstream merges the PR.
|
||||
PR_SQUASH=(
|
||||
"3987 feat/company-portability-complete"
|
||||
)
|
||||
|
||||
for entry in "${PR_SQUASH[@]}"; do
|
||||
pr_num="${entry%% *}"
|
||||
branch="${entry#* }"
|
||||
remote_branch="origin/$branch"
|
||||
|
||||
if ! git rev-parse "$remote_branch" &>/dev/null; then
|
||||
echo "WARNING: $remote_branch not found, skipping PR #$pr_num"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check if the branch has any unique non-merge commits
|
||||
unique=$(git log --no-merges --oneline upstream/master.."$remote_branch" | wc -l)
|
||||
if [ "$unique" -eq 0 ]; then
|
||||
echo "PR #$pr_num ($branch): no unique commits — likely merged upstream, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "PR #$pr_num ($branch): applying as squash ($unique non-merge commits)"
|
||||
git merge --squash "$remote_branch" || {
|
||||
echo "::error::Squash conflict for PR #$pr_num ($branch)"
|
||||
git merge --abort 2>/dev/null || git reset --hard HEAD
|
||||
exit 1
|
||||
}
|
||||
# Only commit if there are staged changes
|
||||
git diff --staged --quiet || git commit -m "feat: apply PR #$pr_num ($branch)"
|
||||
done
|
||||
|
||||
git push origin local --force
|
||||
echo "local branch assembled and pushed"
|
||||
|
||||
- name: Trigger prod build
|
||||
run: |
|
||||
curl -sS -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
https://api.github.com/repos/${{ github.repository }}/actions/workflows/build-prod.yml/dispatches \
|
||||
-d '{"ref":"local"}'
|
||||
|
||||
- name: Sync build-dev.yml to dev branch
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! git rev-parse origin/dev &>/dev/null; then
|
||||
echo "dev branch not found on origin, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
canonical=".farhoodlabs/.github/workflows/build-dev.yml"
|
||||
target=".github/workflows/build-dev.yml"
|
||||
|
||||
if git show origin/dev:"$target" 2>/dev/null | diff --brief - "$canonical" &>/dev/null; then
|
||||
echo "build-dev.yml on dev is up to date, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Syncing build-dev.yml to dev branch..."
|
||||
# Save canonical content before switching branches (.farhoodlabs/ only exists on master)
|
||||
tmp=$(mktemp)
|
||||
cp "$canonical" "$tmp"
|
||||
git checkout -B dev-wf-sync origin/dev
|
||||
mkdir -p "$(dirname "$target")"
|
||||
cp "$tmp" "$target"
|
||||
rm "$tmp"
|
||||
git add "$target"
|
||||
git commit -m "chore(ci): sync build-dev.yml from .farhoodlabs"
|
||||
git push origin dev-wf-sync:dev
|
||||
echo "build-dev.yml synced to dev"
|
||||
@@ -11,7 +11,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
image-tag: ${{ steps.tag.outputs.sha }}
|
||||
@@ -23,28 +23,21 @@ jobs:
|
||||
id: tag
|
||||
run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Login to Gitea Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
registry: git.farh.net
|
||||
username: ${{ gitea.repository_owner }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/farhoodlabs/paperclip-dev
|
||||
images: git.farh.net/farhoodlabs/paperclip-dev
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha,prefix=
|
||||
@@ -62,25 +55,16 @@ jobs:
|
||||
|
||||
update-infra:
|
||||
needs: build
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate app token
|
||||
id: app-token
|
||||
uses: actions/create-github-app-token@v1
|
||||
with:
|
||||
app-id: ${{ secrets.PAPERCLIP_APP_ID }}
|
||||
private-key: ${{ secrets.PAPERCLIP_APP_PRIVATE_KEY }}
|
||||
repositories: paperclip-infra
|
||||
|
||||
- name: Update dev image tag in infra repo
|
||||
run: |
|
||||
SHA="${{ needs.build.outputs.image-tag }}"
|
||||
FILE="overlays/dev/kustomization.yaml"
|
||||
|
||||
response=$(curl -sS \
|
||||
-H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/farhoodlabs/paperclip-infra/contents/$FILE")
|
||||
-H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
|
||||
"https://git.farh.net/api/v1/repos/farhoodlabs/paperclip-infra/contents/$FILE")
|
||||
|
||||
file_sha=$(echo "$response" | jq -r '.sha')
|
||||
content=$(echo "$response" | jq -r '.content' | base64 -d)
|
||||
@@ -88,7 +72,6 @@ jobs:
|
||||
encoded=$(printf '%s' "$new_content" | base64 -w 0)
|
||||
|
||||
curl -sS -X PUT \
|
||||
-H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/farhoodlabs/paperclip-infra/contents/$FILE" \
|
||||
-H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
|
||||
"https://git.farh.net/api/v1/repos/farhoodlabs/paperclip-infra/contents/$FILE" \
|
||||
-d "{\"message\":\"chore(cd): update paperclip-dev to $SHA\",\"content\":\"$encoded\",\"sha\":\"$file_sha\"}"
|
||||
|
||||
@@ -11,33 +11,27 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Login to Gitea Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
registry: git.farh.net
|
||||
username: ${{ gitea.repository_owner }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/farhoodlabs/paperclip
|
||||
images: git.farh.net/farhoodlabs/paperclip
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha,prefix=
|
||||
@@ -47,6 +41,7 @@ jobs:
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: .farhoodlabs/Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
@@ -1,96 +1,16 @@
|
||||
# Disabled in fork — `gh` CLI and GitHub-specific commands are not available on Gitea.
|
||||
# Lockfile refreshes are managed directly in development workflows.
|
||||
#
|
||||
# NOTE: upstream may overwrite this file when master is synced. Re-apply if that happens.
|
||||
name: Refresh Lockfile
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: refresh-lockfile-master
|
||||
cancel-in-progress: false
|
||||
|
||||
inputs:
|
||||
note:
|
||||
description: "Disabled in fork. Uses GitHub-specific gh CLI."
|
||||
required: false
|
||||
jobs:
|
||||
refresh:
|
||||
disabled:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9.15.4
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: pnpm
|
||||
|
||||
- name: Refresh pnpm lockfile
|
||||
run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
- name: Fail on unexpected file changes
|
||||
run: |
|
||||
changed="$(git status --porcelain)"
|
||||
if [ -z "$changed" ]; then
|
||||
echo "Lockfile is already up to date."
|
||||
exit 0
|
||||
fi
|
||||
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
|
||||
echo "Unexpected files changed during lockfile refresh:"
|
||||
echo "$changed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Create or update pull request
|
||||
id: upsert-pr
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
run: |
|
||||
if git diff --quiet -- pnpm-lock.yaml; then
|
||||
echo "Lockfile unchanged, nothing to do."
|
||||
echo "pr_url=" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BRANCH="chore/refresh-lockfile"
|
||||
git config user.name "lockfile-bot"
|
||||
git config user.email "lockfile-bot@users.noreply.github.com"
|
||||
|
||||
git checkout -B "$BRANCH"
|
||||
git add pnpm-lock.yaml
|
||||
git commit -m "chore(lockfile): refresh pnpm-lock.yaml"
|
||||
git push --force origin "$BRANCH"
|
||||
|
||||
# Only reuse an open PR from this repository owner, not a fork with the same branch name.
|
||||
pr_url="$(
|
||||
gh pr list --state open --head "$BRANCH" --json url,headRepositoryOwner \
|
||||
--jq ".[] | select(.headRepositoryOwner.login == \"$REPO_OWNER\") | .url" |
|
||||
head -n 1
|
||||
)"
|
||||
if [ -z "$pr_url" ]; then
|
||||
pr_url="$(gh pr create \
|
||||
--head "$BRANCH" \
|
||||
--title "chore(lockfile): refresh pnpm-lock.yaml" \
|
||||
--body "Auto-generated lockfile refresh after dependencies changed on master. This PR only updates pnpm-lock.yaml.")"
|
||||
echo "Created new PR: $pr_url"
|
||||
else
|
||||
echo "PR already exists: $pr_url"
|
||||
fi
|
||||
echo "pr_url=$pr_url" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Enable auto-merge for lockfile PR
|
||||
if: steps.upsert-pr.outputs.pr_url != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh pr merge --auto --squash --delete-branch "${{ steps.upsert-pr.outputs.pr_url }}"
|
||||
- run: echo "Disabled. Lockfile management requires GitHub-specific tooling."
|
||||
|
||||
@@ -16,7 +16,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: runners-farhoodlabs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout master
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# Paperclip Fork — Project Context
|
||||
|
||||
This is a fork of [paperclipai/paperclip](https://github.com/paperclipai/paperclip).
|
||||
Fork repo: https://github.com/farhoodlabs/paperclip
|
||||
Fork repo: https://git.farh.net/farhoodlabs/paperclip
|
||||
|
||||
## Branch Model
|
||||
|
||||
| Branch | Purpose |
|
||||
|---|---|
|
||||
| `master` | Mirrors `upstream/master` exactly + `.farhoodlabs/` overlay directory + `assemble-local.yml` action. Never commit application code here. |
|
||||
| `local` | **Default branch.** Assembled automatically by `assemble-local.yml` on every `master` push. Contains: upstream + fork Dockerfile/workflows + all pending upstream PR cherry-picks. Builds `ghcr.io/farhoodlabs/paperclip`. |
|
||||
| `dev` | Development branch based on upstream/master. Builds `ghcr.io/farhoodlabs/paperclip-dev` on every push. |
|
||||
| `local` | **Default branch.** Assembled automatically by `assemble-local.yml` on every `master` push. Contains: upstream + fork Dockerfile/workflows + all pending upstream PR cherry-picks. Builds `git.farh.net/farhoodlabs/paperclip`. |
|
||||
| `dev` | Development branch based on upstream/master. Builds `git.farh.net/farhoodlabs/paperclip-dev` on every push. |
|
||||
| PR branches | `skill-pat-feature`, `skill-scan-refresh`, `feat/company-portability-complete` — open PRs to upstream, never rebase onto master/local. |
|
||||
|
||||
**Never commit directly to `local`** — it is fully regenerated by the assemble action and any direct commits will be overwritten.
|
||||
@@ -70,7 +70,7 @@ Edit `.farhoodlabs/Dockerfile` on `master`. Only modify the production stage —
|
||||
|
||||
## Deployment
|
||||
|
||||
Paperclip runs in Kubernetes, not locally. Use `kubectl` to access it. The production image is `ghcr.io/farhoodlabs/paperclip:latest`.
|
||||
Paperclip runs in Kubernetes, not locally. Use `kubectl` to access it. The production image is `git.farh.net/farhoodlabs/paperclip:latest`.
|
||||
|
||||
## Key Files
|
||||
|
||||
|
||||
@@ -137,6 +137,25 @@ vi.mock("../routes/org-chart-svg.js", () => ({
|
||||
renderOrgChartPng: vi.fn(async () => Buffer.from("png")),
|
||||
}));
|
||||
|
||||
const gitSourceMock = vi.hoisted(() => ({
|
||||
resolveGitRef: vi.fn(),
|
||||
openRepoSnapshot: vi.fn(),
|
||||
}));
|
||||
|
||||
// parseGitSourceUrl stays real (the shim parseGitHubSourceUrl delegates to it
|
||||
// and is asserted by existing tests). Only the network-touching functions are
|
||||
// overridable per-test.
|
||||
vi.mock("../services/git-source.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../services/git-source.js")>(
|
||||
"../services/git-source.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
resolveGitRef: gitSourceMock.resolveGitRef,
|
||||
openRepoSnapshot: gitSourceMock.openRepoSnapshot,
|
||||
};
|
||||
});
|
||||
|
||||
const { companyPortabilityService, parseGitHubSourceUrl } = await import("../services/company-portability.js");
|
||||
|
||||
function asTextFile(entry: CompanyPortabilityFileEntry | undefined) {
|
||||
@@ -3378,3 +3397,173 @@ describe("company portability", () => {
|
||||
expect(preview.plan.issuePlans).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("git source orchestration via resolveSource", () => {
|
||||
const minimalCompanyMarkdown = "---\ncompany:\n name: Demo\n---\n# Demo\n";
|
||||
const githubUrl = "https://git.example.com/acme/co?ref=main&path=";
|
||||
|
||||
function makeSnapshot(overrides: {
|
||||
files?: string[];
|
||||
fileContents?: Record<string, string>;
|
||||
binaryContents?: Record<string, Uint8Array>;
|
||||
readBinaryReject?: Error;
|
||||
} = {}) {
|
||||
const files = overrides.files ?? ["COMPANY.md"];
|
||||
const fileContents = overrides.fileContents ?? { "COMPANY.md": minimalCompanyMarkdown };
|
||||
const binaryContents = overrides.binaryContents ?? {};
|
||||
return {
|
||||
sha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
listFiles: vi.fn(async () => files),
|
||||
readFile: vi.fn(async (p: string) => {
|
||||
if (p in fileContents) return fileContents[p];
|
||||
throw Object.assign(new Error(`not found: ${p}`), { code: "NotFoundError" });
|
||||
}),
|
||||
readFileOptional: vi.fn(async (p: string) => fileContents[p] ?? null),
|
||||
readBinary: vi.fn(async (p: string) => {
|
||||
if (overrides.readBinaryReject) throw overrides.readBinaryReject;
|
||||
if (p in binaryContents) return binaryContents[p]!;
|
||||
throw Object.assign(new Error(`not found: ${p}`), { code: "NotFoundError" });
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function setupResolveStub() {
|
||||
gitSourceMock.resolveGitRef.mockResolvedValue({
|
||||
pinnedSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
trackingRef: "main",
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
gitSourceMock.resolveGitRef.mockReset();
|
||||
gitSourceMock.openRepoSnapshot.mockReset();
|
||||
companySvc.getById.mockResolvedValue(null);
|
||||
agentSvc.list.mockResolvedValue([]);
|
||||
projectSvc.list.mockResolvedValue([]);
|
||||
issueSvc.list.mockResolvedValue([]);
|
||||
issueSvc.listComments.mockResolvedValue([]);
|
||||
companySkillSvc.list.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it("opens a snapshot and walks the tree for a github source", async () => {
|
||||
setupResolveStub();
|
||||
const snapshot = makeSnapshot({
|
||||
files: ["COMPANY.md", "README.md", "skills/x/SKILL.md"],
|
||||
fileContents: {
|
||||
"COMPANY.md": minimalCompanyMarkdown,
|
||||
"README.md": "# readme",
|
||||
"skills/x/SKILL.md": "---\nname: x\n---\n",
|
||||
},
|
||||
});
|
||||
gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot);
|
||||
|
||||
const portability = companyPortabilityService({} as any);
|
||||
const preview = await portability.previewImport({
|
||||
source: { type: "github", url: githubUrl },
|
||||
include: { company: true, agents: false, projects: false, issues: false, skills: false },
|
||||
target: { mode: "new_company", newCompanyName: "Demo" },
|
||||
agents: "all",
|
||||
collisionStrategy: "rename",
|
||||
});
|
||||
|
||||
expect(gitSourceMock.resolveGitRef).toHaveBeenCalledTimes(1);
|
||||
expect(gitSourceMock.openRepoSnapshot).toHaveBeenCalledTimes(1);
|
||||
expect(snapshot.listFiles).toHaveBeenCalled();
|
||||
expect(snapshot.readFileOptional).toHaveBeenCalledWith("COMPANY.md");
|
||||
expect(snapshot.readFile).toHaveBeenCalledWith("README.md");
|
||||
expect(snapshot.readFile).toHaveBeenCalledWith("skills/x/SKILL.md");
|
||||
expect(preview.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("falls back from main to master when the main ref does not exist", async () => {
|
||||
setupResolveStub();
|
||||
const masterSnap = makeSnapshot();
|
||||
// First call (ref=main) rejects; second (ref=master) succeeds.
|
||||
gitSourceMock.openRepoSnapshot
|
||||
.mockRejectedValueOnce(new Error("ref not found"))
|
||||
.mockResolvedValueOnce(masterSnap);
|
||||
|
||||
const portability = companyPortabilityService({} as any);
|
||||
const preview = await portability.previewImport({
|
||||
source: { type: "github", url: githubUrl },
|
||||
include: { company: true, agents: false, projects: false, issues: false, skills: false },
|
||||
target: { mode: "new_company", newCompanyName: "Demo" },
|
||||
agents: "all",
|
||||
collisionStrategy: "rename",
|
||||
});
|
||||
|
||||
expect(gitSourceMock.openRepoSnapshot).toHaveBeenCalledTimes(2);
|
||||
expect(masterSnap.readFileOptional).toHaveBeenCalledWith("COMPANY.md");
|
||||
expect(preview.warnings).toContain("Git ref main not found; falling back to master.");
|
||||
});
|
||||
|
||||
it("throws when COMPANY.md is missing on both main and master", async () => {
|
||||
setupResolveStub();
|
||||
const emptySnap = makeSnapshot({ fileContents: {} });
|
||||
gitSourceMock.openRepoSnapshot.mockResolvedValue(emptySnap);
|
||||
|
||||
const portability = companyPortabilityService({} as any);
|
||||
await expect(
|
||||
portability.previewImport({
|
||||
source: { type: "github", url: githubUrl },
|
||||
include: { company: true, agents: false, projects: false, issues: false, skills: false },
|
||||
target: { mode: "new_company", newCompanyName: "Demo" },
|
||||
agents: "all",
|
||||
collisionStrategy: "rename",
|
||||
}),
|
||||
).rejects.toThrow(/missing COMPANY.md/i);
|
||||
});
|
||||
|
||||
it("fetches a referenced company logo as binary", async () => {
|
||||
setupResolveStub();
|
||||
// logoPath lives in .paperclip.yaml (paperclip extension), not COMPANY.md.
|
||||
const paperclipYaml = "company:\n logoPath: images/logo.png\n";
|
||||
const logoBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]);
|
||||
const snapshot = makeSnapshot({
|
||||
files: ["COMPANY.md", ".paperclip.yaml", "images/logo.png"],
|
||||
fileContents: {
|
||||
"COMPANY.md": minimalCompanyMarkdown,
|
||||
".paperclip.yaml": paperclipYaml,
|
||||
},
|
||||
binaryContents: { "images/logo.png": logoBytes },
|
||||
});
|
||||
gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot);
|
||||
|
||||
const portability = companyPortabilityService({} as any);
|
||||
await portability.previewImport({
|
||||
source: { type: "github", url: githubUrl },
|
||||
include: { company: true, agents: false, projects: false, issues: false, skills: false },
|
||||
target: { mode: "new_company", newCompanyName: "Demo" },
|
||||
agents: "all",
|
||||
collisionStrategy: "rename",
|
||||
});
|
||||
|
||||
expect(snapshot.readBinary).toHaveBeenCalledWith("images/logo.png");
|
||||
});
|
||||
|
||||
it("warns instead of throwing when the logo blob can't be read", async () => {
|
||||
setupResolveStub();
|
||||
const paperclipYaml = "company:\n logoPath: images/logo.png\n";
|
||||
const snapshot = makeSnapshot({
|
||||
files: ["COMPANY.md", ".paperclip.yaml"],
|
||||
fileContents: {
|
||||
"COMPANY.md": minimalCompanyMarkdown,
|
||||
".paperclip.yaml": paperclipYaml,
|
||||
},
|
||||
readBinaryReject: new Error("blob missing"),
|
||||
});
|
||||
gitSourceMock.openRepoSnapshot.mockResolvedValue(snapshot);
|
||||
|
||||
const portability = companyPortabilityService({} as any);
|
||||
const preview = await portability.previewImport({
|
||||
source: { type: "github", url: githubUrl },
|
||||
include: { company: true, agents: false, projects: false, issues: false, skills: false },
|
||||
target: { mode: "new_company", newCompanyName: "Demo" },
|
||||
agents: "all",
|
||||
collisionStrategy: "rename",
|
||||
});
|
||||
|
||||
expect(snapshot.readBinary).toHaveBeenCalled();
|
||||
expect(preview.warnings.some((w: string) => /Failed to fetch company logo/i.test(w))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -145,6 +145,43 @@ describe("parseGitSourceUrl", () => {
|
||||
it("rejects malformed URLs", () => {
|
||||
expect(() => parseGitSourceUrl("not a url")).toThrow();
|
||||
});
|
||||
|
||||
it("parses a query-string URL with ?ref= and ?path=", () => {
|
||||
expect(
|
||||
parseGitSourceUrl("https://github.com/o/r?ref=feature%2Fdemo&path=subdir"),
|
||||
).toMatchObject({
|
||||
cloneUrl: "https://github.com/o/r.git",
|
||||
ref: "feature/demo",
|
||||
basePath: "subdir",
|
||||
filePath: null,
|
||||
explicitRef: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses a query-string URL with only ?ref=", () => {
|
||||
expect(parseGitSourceUrl("https://github.com/o/r?ref=develop")).toMatchObject({
|
||||
ref: "develop",
|
||||
basePath: "",
|
||||
explicitRef: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses a query-string URL with only ?path=", () => {
|
||||
expect(parseGitSourceUrl("https://github.com/o/r?path=sub")).toMatchObject({
|
||||
ref: null,
|
||||
basePath: "sub",
|
||||
explicitRef: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("query-string parsing takes precedence over path-style segments", () => {
|
||||
expect(
|
||||
parseGitSourceUrl("https://github.com/o/r/tree/main/old?ref=newref&path=newpath"),
|
||||
).toMatchObject({
|
||||
ref: "newref",
|
||||
basePath: "newpath",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCloneUrl", () => {
|
||||
@@ -333,4 +370,41 @@ describe("openRepoSnapshot", () => {
|
||||
openRepoSnapshot(parsed, "main", "1111111111111111111111111111111111111111"),
|
||||
).rejects.toThrow(/repository not found/i);
|
||||
});
|
||||
|
||||
it("readBinary returns the raw blob bytes", async () => {
|
||||
cloneFn.mockResolvedValue(undefined);
|
||||
resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff");
|
||||
walkFn.mockImplementation(async () => {});
|
||||
const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]);
|
||||
readBlobFn.mockResolvedValue({ blob: bytes });
|
||||
|
||||
const parsed = parseGitSourceUrl("https://git.example.com/o/r");
|
||||
const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff");
|
||||
const result = await snap.readBinary("logo.png");
|
||||
expect(result).toBe(bytes);
|
||||
});
|
||||
|
||||
it("readFileOptional returns null on NotFoundError", async () => {
|
||||
cloneFn.mockResolvedValue(undefined);
|
||||
resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff");
|
||||
walkFn.mockImplementation(async () => {});
|
||||
const err = Object.assign(new Error("missing"), { code: "NotFoundError" });
|
||||
readBlobFn.mockRejectedValue(err);
|
||||
|
||||
const parsed = parseGitSourceUrl("https://git.example.com/o/r");
|
||||
const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff");
|
||||
const result = await snap.readFileOptional("missing.md");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("readFileOptional rethrows non-NotFound errors", async () => {
|
||||
cloneFn.mockResolvedValue(undefined);
|
||||
resolveRefFn.mockResolvedValue("ffffffffffffffffffffffffffffffffffffffff");
|
||||
walkFn.mockImplementation(async () => {});
|
||||
readBlobFn.mockRejectedValue(new Error("disk explosion"));
|
||||
|
||||
const parsed = parseGitSourceUrl("https://git.example.com/o/r");
|
||||
const snap = await openRepoSnapshot(parsed, "main", "ffffffffffffffffffffffffffffffffffffffff");
|
||||
await expect(snap.readFileOptional("any.md")).rejects.toThrow(/disk explosion/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -57,7 +57,7 @@ import {
|
||||
import { requireOpenCodeModelId } from "@paperclipai/adapter-opencode-local/server";
|
||||
import { findServerAdapter } from "../adapters/index.js";
|
||||
import { forbidden, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js";
|
||||
import { openRepoSnapshot, parseGitSourceUrl, resolveGitRef, type RepoSnapshot } from "./git-source.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { accessService } from "./access.js";
|
||||
import { agentService } from "./agents.js";
|
||||
@@ -2339,42 +2339,6 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchText(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function fetchOptionalText(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function fetchBinary(url: string) {
|
||||
const response = await ghFetch(url);
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
const response = await ghFetch(url, {
|
||||
headers: {
|
||||
accept: "application/vnd.github+json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.json() as Promise<T>;
|
||||
}
|
||||
|
||||
function dedupeEnvInputs(values: CompanyPortabilityManifest["envInputs"]) {
|
||||
const seen = new Set<string>();
|
||||
@@ -2864,52 +2828,37 @@ function normalizeGitHubSourcePath(value: string | null | undefined) {
|
||||
|
||||
export function parseGitHubSourceUrl(rawUrl: string) {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.protocol !== "https:") {
|
||||
throw unprocessable("GitHub source URL must use HTTPS");
|
||||
}
|
||||
const hostname = url.hostname;
|
||||
const parts = url.pathname.split("/").filter(Boolean);
|
||||
if (parts.length < 2) {
|
||||
throw unprocessable("Invalid GitHub URL");
|
||||
}
|
||||
const owner = parts[0]!;
|
||||
const repo = parts[1]!.replace(/\.git$/i, "");
|
||||
const queryRef = url.searchParams.get("ref")?.trim();
|
||||
const queryPath = normalizeGitHubSourcePath(url.searchParams.get("path"));
|
||||
// Handle the portability-specific companyPath query param before delegating,
|
||||
// since git-source has no notion of it.
|
||||
const queryCompanyPath = normalizeGitHubSourcePath(url.searchParams.get("companyPath"));
|
||||
if (queryRef || queryPath || queryCompanyPath) {
|
||||
const companyPath = queryCompanyPath || [queryPath, "COMPANY.md"].filter(Boolean).join("/") || "COMPANY.md";
|
||||
let basePath = queryPath;
|
||||
if (!basePath && companyPath !== "COMPANY.md") {
|
||||
basePath = path.posix.dirname(companyPath);
|
||||
if (basePath === ".") basePath = "";
|
||||
|
||||
const parsed = parseGitSourceUrl(rawUrl);
|
||||
|
||||
let companyPath: string;
|
||||
let basePath = parsed.basePath;
|
||||
if (queryCompanyPath) {
|
||||
companyPath = queryCompanyPath;
|
||||
if (!basePath) {
|
||||
const derived = path.posix.dirname(companyPath);
|
||||
basePath = derived === "." ? "" : derived;
|
||||
}
|
||||
return {
|
||||
hostname,
|
||||
owner,
|
||||
repo,
|
||||
ref: queryRef || "main",
|
||||
basePath,
|
||||
companyPath,
|
||||
};
|
||||
} else if (parsed.filePath) {
|
||||
// blob-style URL pointed directly at a file
|
||||
companyPath = parsed.filePath;
|
||||
} else if (basePath) {
|
||||
companyPath = `${basePath}/COMPANY.md`;
|
||||
} else {
|
||||
companyPath = "COMPANY.md";
|
||||
}
|
||||
let ref = "main";
|
||||
let basePath = "";
|
||||
let companyPath = "COMPANY.md";
|
||||
if (parts[2] === "tree") {
|
||||
ref = parts[3] ?? "main";
|
||||
basePath = parts.slice(4).join("/");
|
||||
} else if (parts[2] === "blob") {
|
||||
ref = parts[3] ?? "main";
|
||||
const blobPath = parts.slice(4).join("/");
|
||||
if (!blobPath) {
|
||||
throw unprocessable("Invalid GitHub blob URL");
|
||||
}
|
||||
companyPath = blobPath;
|
||||
basePath = path.posix.dirname(blobPath);
|
||||
if (basePath === ".") basePath = "";
|
||||
}
|
||||
return { hostname, owner, repo, ref, basePath, companyPath };
|
||||
|
||||
return {
|
||||
hostname: parsed.hostname,
|
||||
owner: parsed.owner,
|
||||
repo: parsed.repo,
|
||||
ref: parsed.ref ?? "main",
|
||||
basePath,
|
||||
companyPath,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -3013,30 +2962,38 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
);
|
||||
}
|
||||
|
||||
const parsed = parseGitHubSourceUrl(source.url);
|
||||
let ref = parsed.ref;
|
||||
const sourceUrl = source.url;
|
||||
const parsed = parseGitHubSourceUrl(sourceUrl);
|
||||
const warnings: string[] = [];
|
||||
const companyRelativePath = parsed.companyPath === "COMPANY.md"
|
||||
? [parsed.basePath, "COMPANY.md"].filter(Boolean).join("/")
|
||||
: parsed.companyPath;
|
||||
|
||||
async function openSnapshot(refName: string): Promise<RepoSnapshot> {
|
||||
const ps = parseGitSourceUrl(sourceUrl);
|
||||
const wanted = { ...ps, ref: refName, explicitRef: true };
|
||||
const resolved = await resolveGitRef(wanted);
|
||||
return openRepoSnapshot(wanted, resolved.trackingRef, resolved.pinnedSha);
|
||||
}
|
||||
|
||||
let ref = parsed.ref;
|
||||
let snapshot: RepoSnapshot;
|
||||
let companyMarkdown: string | null = null;
|
||||
try {
|
||||
companyMarkdown = await fetchOptionalText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
|
||||
);
|
||||
snapshot = await openSnapshot(ref);
|
||||
companyMarkdown = await snapshot.readFileOptional(companyRelativePath);
|
||||
} catch (err) {
|
||||
if (ref === "main") {
|
||||
ref = "master";
|
||||
warnings.push("GitHub ref main not found; falling back to master.");
|
||||
companyMarkdown = await fetchOptionalText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
|
||||
);
|
||||
warnings.push("Git ref main not found; falling back to master.");
|
||||
snapshot = await openSnapshot(ref);
|
||||
companyMarkdown = await snapshot.readFileOptional(companyRelativePath);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (!companyMarkdown) {
|
||||
throw unprocessable("GitHub company package is missing COMPANY.md");
|
||||
throw unprocessable("Git company package is missing COMPANY.md");
|
||||
}
|
||||
|
||||
const companyPath = parsed.companyPath === "COMPANY.md"
|
||||
@@ -3045,31 +3002,22 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const files: Record<string, CompanyPortabilityFileEntry> = {
|
||||
[companyPath]: companyMarkdown,
|
||||
};
|
||||
const apiBase = gitHubApiBase(parsed.hostname);
|
||||
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
|
||||
`${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
|
||||
).catch(() => ({ tree: [] }));
|
||||
const basePrefix = parsed.basePath ? `${parsed.basePath.replace(/^\/+|\/+$/g, "")}/` : "";
|
||||
const candidatePaths = (tree.tree ?? [])
|
||||
.filter((entry) => entry.type === "blob")
|
||||
.map((entry) => entry.path)
|
||||
.filter((entry): entry is string => typeof entry === "string")
|
||||
.filter((entry) => {
|
||||
if (basePrefix && !entry.startsWith(basePrefix)) return false;
|
||||
const relative = basePrefix ? entry.slice(basePrefix.length) : entry;
|
||||
return (
|
||||
relative.endsWith(".md") ||
|
||||
relative.startsWith("skills/") ||
|
||||
relative === ".paperclip.yaml" ||
|
||||
relative === ".paperclip.yml"
|
||||
);
|
||||
});
|
||||
const allPaths = await snapshot.listFiles();
|
||||
const candidatePaths = allPaths.filter((entry) => {
|
||||
if (basePrefix && !entry.startsWith(basePrefix)) return false;
|
||||
const relative = basePrefix ? entry.slice(basePrefix.length) : entry;
|
||||
return (
|
||||
relative.endsWith(".md") ||
|
||||
relative.startsWith("skills/") ||
|
||||
relative === ".paperclip.yaml" ||
|
||||
relative === ".paperclip.yml"
|
||||
);
|
||||
});
|
||||
for (const repoPath of candidatePaths) {
|
||||
const relativePath = basePrefix ? repoPath.slice(basePrefix.length) : repoPath;
|
||||
if (files[relativePath] !== undefined) continue;
|
||||
files[normalizePortablePath(relativePath)] = await fetchText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
);
|
||||
files[normalizePortablePath(relativePath)] = await snapshot.readFile(repoPath);
|
||||
}
|
||||
const companyDoc = parseFrontmatterMarkdown(companyMarkdown);
|
||||
const includeEntries = readIncludeEntries(companyDoc.frontmatter);
|
||||
@@ -3078,9 +3026,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
const relativePath = normalizePortablePath(includeEntry.path);
|
||||
if (files[relativePath] !== undefined) continue;
|
||||
if (!(repoPath.endsWith(".md") || repoPath.endsWith(".yaml") || repoPath.endsWith(".yml"))) continue;
|
||||
files[relativePath] = await fetchText(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
);
|
||||
files[relativePath] = await snapshot.readFile(repoPath);
|
||||
}
|
||||
|
||||
const resolved = buildManifestFromPackageFiles(files);
|
||||
@@ -3088,12 +3034,13 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
if (companyLogoPath && !resolved.files[companyLogoPath]) {
|
||||
const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/");
|
||||
try {
|
||||
const binary = await fetchBinary(
|
||||
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
|
||||
const binary = await snapshot.readBinary(repoPath);
|
||||
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(
|
||||
Buffer.from(binary),
|
||||
inferContentTypeFromPath(companyLogoPath),
|
||||
);
|
||||
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath));
|
||||
} catch (err) {
|
||||
warnings.push(`Failed to fetch company logo ${companyLogoPath} from GitHub: ${err instanceof Error ? err.message : String(err)}`);
|
||||
warnings.push(`Failed to fetch company logo ${companyLogoPath} from git: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
resolved.warnings.unshift(...warnings);
|
||||
|
||||
@@ -25,6 +25,8 @@ export type RepoSnapshot = {
|
||||
sha: string;
|
||||
listFiles(): Promise<string[]>;
|
||||
readFile(repoPath: string): Promise<string>;
|
||||
readFileOptional(repoPath: string): Promise<string | null>;
|
||||
readBinary(repoPath: string): Promise<Uint8Array>;
|
||||
};
|
||||
|
||||
const SHA_REGEX = /^[0-9a-f]{40}$/i;
|
||||
@@ -50,6 +52,25 @@ export function parseGitSourceUrl(rawUrl: string): ParsedGitSource {
|
||||
const owner = segments[0]!;
|
||||
const repo = segments[1]!.replace(/\.git$/i, "");
|
||||
|
||||
// Query-string shape: /{owner}/{repo}?ref=...&path=...
|
||||
// Used by company portability URLs. Takes precedence over path-based parsing
|
||||
// so a URL with both shapes (rare) prefers the explicit query params.
|
||||
const queryRef = url.searchParams.get("ref")?.trim() ?? null;
|
||||
const queryPath = url.searchParams.get("path")?.trim() ?? null;
|
||||
if (queryRef || queryPath) {
|
||||
const normalizedPath = (queryPath ?? "").replace(/\\/g, "/").replace(/^\/+|\/+$/g, "");
|
||||
return {
|
||||
cloneUrl: buildCloneUrl(url.hostname, owner, repo),
|
||||
hostname: url.hostname,
|
||||
owner,
|
||||
repo,
|
||||
ref: queryRef || null,
|
||||
basePath: normalizedPath,
|
||||
filePath: null,
|
||||
explicitRef: Boolean(queryRef),
|
||||
};
|
||||
}
|
||||
|
||||
let ref: string | null = null;
|
||||
let basePath = "";
|
||||
let filePath: string | null = null;
|
||||
@@ -233,11 +254,29 @@ export async function openRepoSnapshot(
|
||||
return out;
|
||||
}
|
||||
|
||||
async function readFile(repoPath: string): Promise<string> {
|
||||
async function readBinary(repoPath: string): Promise<Uint8Array> {
|
||||
const normalized = repoPath.replace(/^\/+/, "");
|
||||
const { blob } = await git.readBlob({ fs, dir, oid: sha, filepath: normalized });
|
||||
return blob;
|
||||
}
|
||||
|
||||
async function readFile(repoPath: string): Promise<string> {
|
||||
const blob = await readBinary(repoPath);
|
||||
return new TextDecoder("utf-8").decode(blob);
|
||||
}
|
||||
|
||||
return { sha, listFiles, readFile };
|
||||
async function readFileOptional(repoPath: string): Promise<string | null> {
|
||||
try {
|
||||
return await readFile(repoPath);
|
||||
} catch (err) {
|
||||
// isomorphic-git throws NotFoundError when the path is missing from the tree.
|
||||
const name = (err as { code?: string; name?: string } | null)?.code
|
||||
?? (err as { name?: string } | null)?.name
|
||||
?? "";
|
||||
if (/NotFound/i.test(name)) return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
return { sha, listFiles, readFile, readFileOptional, readBinary };
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
import { unprocessable } from "../errors.js";
|
||||
|
||||
export type GitHostFamily = "github" | "gitea";
|
||||
|
||||
export function inferGitHostFamily(hostname: string): GitHostFamily {
|
||||
const h = hostname.toLowerCase();
|
||||
if (h === "github.com" || h === "www.github.com") return "github";
|
||||
return "gitea";
|
||||
}
|
||||
|
||||
export function gitHubApiBase(hostname: string) {
|
||||
return inferGitHostFamily(hostname) === "github"
|
||||
? "https://api.github.com"
|
||||
: `https://${hostname}/api/v1`;
|
||||
}
|
||||
|
||||
export function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) {
|
||||
const p = filePath.replace(/^\/+/, "");
|
||||
if (inferGitHostFamily(hostname) === "github") {
|
||||
return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}`;
|
||||
}
|
||||
return `https://${hostname}/api/v1/repos/${owner}/${repo}/media/${p}?ref=${encodeURIComponent(ref)}`;
|
||||
}
|
||||
|
||||
export async function ghFetch(url: string, init?: RequestInit, authToken?: string): Promise<Response> {
|
||||
const headers = new Headers(init?.headers);
|
||||
if (authToken) {
|
||||
headers.set("Authorization", `Bearer ${authToken}`);
|
||||
}
|
||||
try {
|
||||
return await fetch(url, { ...init, headers, redirect: authToken ? "manual" : "follow" });
|
||||
} catch {
|
||||
const hostname = (() => {
|
||||
try { return new URL(url).hostname; } catch { return url; }
|
||||
})();
|
||||
throw unprocessable(`Could not connect to ${hostname}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user