Files
paperclip/scripts/verify-release-registry-state.test.mjs
Devin Foley a72731f118 fix: harden release registry verification against npm lag (#4816)
## Thinking Path

> - Paperclip orchestrates AI agents for zero-human companies
> - Its release automation publishes canary packages to npm and then
validates the published registry state before considering the release
healthy
> - The failing canary run `25139465018` showed that npm can expose a
newly published version through version-specific endpoints before the
root package document has fully converged
> - That made a successful canary publish look like a failed release
because the verifier trusted stale root metadata too early
> - This pull request hardens the registry verification path by
preferring version-specific manifest checks, retrying
convergence-sensitive failures, and distinguishing permanent failures
from propagation lag
> - While validating that change in CI, a separate teardown race in
`heartbeat-stale-queue-invalidation.test.ts` surfaced and was hardened
so the PR could pass reliably
> - The benefit is that transient npm propagation lag no longer fails a
successful canary publish, while genuine registry-state and
dependency-integrity failures still stop the release flow promptly

## What Changed

- Hardened `scripts/verify-release-registry-state.mjs` so it prefers
version-specific manifest resolution over stale root metadata, adds
bounded registry-fetch timeouts, and classifies failures as retriable vs
non-retriable.
- Updated `scripts/release-lib.sh` and `scripts/release.sh` so
post-publish registry verification retries only convergence-sensitive
failures and reports immediate permanent failures clearly.
- Expanded `scripts/verify-release-registry-state.test.mjs` with
regression coverage for stale root metadata, fetch timeout behavior,
peer dependency range handling, non-retriable canary-latest cases, and
related verifier edge cases.
- Hardened
`server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts`
teardown to tolerate the late-comment foreign-key race that CI exposed
while validating this branch.

## Verification

- `pnpm run test:release-registry`
- `node --check scripts/verify-release-registry-state.mjs`
- `bash -n scripts/release.sh && bash -n scripts/release-lib.sh`
- PR checks passed on head `5c422600fc12acac61f6b7c267a4dc915df622b1`:
`policy`, `verify`, `e2e`, `security/snyk`, and `Greptile Review`

## Risks

- Low risk. The main behavioral changes are limited to release
automation and verifier retry semantics, plus a test-only teardown
hardening for a CI race.

> I checked [`ROADMAP.md`](ROADMAP.md). This is a narrow release bugfix
and does not overlap planned core feature work.

## Model Used

- OpenAI Codex via Paperclip `codex_local` with tool use and local code
execution enabled. This agent session runs on a GPT-5-class coding
model; the exact backend model ID/context window is not exposed by the
local adapter runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots
- [ ] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I have addressed all Greptile and reviewer comments before
requesting merge
2026-05-09 22:18:12 -07:00

364 lines
9.2 KiB
JavaScript

import assert from "node:assert/strict";
import test from "node:test";
import {
collectInternalDependencyProblems,
createManifestLookupKey,
fetchRegistryJson,
isCanaryVersion,
verifyPackageRegistryProblems,
verifyPackageRegistryState,
} from "./verify-release-registry-state.mjs";
test("isCanaryVersion matches release canaries", () => {
assert.equal(isCanaryVersion("2026.427.0-canary.3"), true);
assert.equal(isCanaryVersion("2026.427.0"), false);
});
test("collectInternalDependencyProblems flags missing internal versions", () => {
const manifest = {
dependencies: {
"@paperclipai/plugin-sdk": "2026.425.0-canary.5",
e2b: "^2.19.0",
},
};
const packageDocsByName = new Map([
[
"@paperclipai/plugin-sdk",
{
versions: {
"2026.427.0-canary.3": {},
},
},
],
]);
assert.deepEqual(
collectInternalDependencyProblems(manifest, packageDocsByName),
["dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version"],
);
});
test("collectInternalDependencyProblems accepts version-specific manifests when the root document is stale", () => {
const manifest = {
dependencies: {
"@paperclipai/plugin-sdk": "2026.425.0-canary.5",
},
};
const packageDocsByName = new Map([
[
"@paperclipai/plugin-sdk",
{
versions: {},
},
],
]);
const packageManifestsByKey = new Map([
[
createManifestLookupKey("@paperclipai/plugin-sdk", "2026.425.0-canary.5"),
{ name: "@paperclipai/plugin-sdk", version: "2026.425.0-canary.5" },
],
]);
assert.deepEqual(
collectInternalDependencyProblems(manifest, packageDocsByName, packageManifestsByKey),
[],
);
});
test("collectInternalDependencyProblems ignores peer dependency range specifiers", () => {
const manifest = {
peerDependencies: {
"@paperclipai/server": "^2026.430.0-canary.0",
},
};
assert.deepEqual(
collectInternalDependencyProblems(manifest, new Map()),
[],
);
});
test("collectInternalDependencyProblems reports unfetched transitive dependency metadata neutrally", () => {
const manifest = {
optionalDependencies: {
"@paperclipai/browser": "2026.430.0-canary.0",
},
};
assert.deepEqual(
collectInternalDependencyProblems(manifest, new Map()),
[
"optionalDependencies requires @paperclipai/browser@2026.430.0-canary.0, but npm publication metadata was not fetched for that dependency",
],
);
});
test("verifyPackageRegistryState tolerates a stale root versions map when dist-tags and direct manifests are correct", () => {
const packageDocsByName = new Map([
[
"@paperclipai/ui",
{
"dist-tags": {
canary: "2026.430.0-canary.0",
latest: "2026.430.0",
},
versions: {},
},
],
[
"@paperclipai/shared",
{
versions: {},
},
],
]);
const packageManifestsByKey = new Map([
[
createManifestLookupKey("@paperclipai/ui", "2026.430.0-canary.0"),
{
name: "@paperclipai/ui",
version: "2026.430.0-canary.0",
dependencies: {
"@paperclipai/shared": "2026.430.0-canary.0",
},
},
],
[
createManifestLookupKey("@paperclipai/shared", "2026.430.0-canary.0"),
{
name: "@paperclipai/shared",
version: "2026.430.0-canary.0",
},
],
]);
assert.deepEqual(
verifyPackageRegistryState({
packageName: "@paperclipai/ui",
packageDoc: packageDocsByName.get("@paperclipai/ui"),
packageDocsByName,
packageManifestsByKey,
channel: "canary",
distTag: "canary",
targetVersion: "2026.430.0-canary.0",
allowCanaryLatest: false,
}),
[],
);
});
test("verifyPackageRegistryState fails when canary latest is left in place by default", () => {
const packageDocsByName = new Map([
[
"@paperclipai/plugin-e2b",
{
"dist-tags": {
latest: "2026.425.0-canary.5",
canary: "2026.427.0-canary.3",
},
versions: {
"2026.425.0-canary.5": {
dependencies: {
"@paperclipai/plugin-sdk": "2026.425.0-canary.5",
},
},
"2026.427.0-canary.3": {
dependencies: {
"@paperclipai/plugin-sdk": "2026.427.0-canary.3",
},
},
},
},
],
[
"@paperclipai/plugin-sdk",
{
versions: {
"2026.427.0-canary.3": {},
},
},
],
]);
assert.deepEqual(
verifyPackageRegistryState({
packageName: "@paperclipai/plugin-e2b",
packageDoc: packageDocsByName.get("@paperclipai/plugin-e2b"),
packageDocsByName,
channel: "canary",
distTag: "canary",
targetVersion: "2026.427.0-canary.3",
allowCanaryLatest: false,
}),
[
"@paperclipai/plugin-e2b: latest dist-tag still resolves to canary 2026.425.0-canary.5; if that state is intentional, rerun the verification script directly with --allow-canary-latest",
"@paperclipai/plugin-e2b@2026.425.0-canary.5 via latest: dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version",
],
);
});
test("verifyPackageRegistryProblems marks canary latest drift as non-retriable", () => {
const packageDocsByName = new Map([
[
"@paperclipai/plugin-e2b",
{
"dist-tags": {
latest: "2026.425.0-canary.5",
canary: "2026.427.0-canary.3",
},
versions: {
"2026.427.0-canary.3": {},
},
},
],
]);
const problems = verifyPackageRegistryProblems({
packageName: "@paperclipai/plugin-e2b",
packageDoc: packageDocsByName.get("@paperclipai/plugin-e2b"),
packageDocsByName,
channel: "canary",
distTag: "canary",
targetVersion: "2026.427.0-canary.3",
allowCanaryLatest: false,
});
assert.equal(problems[0]?.retriable, false);
assert.match(problems[0]?.message ?? "", /latest dist-tag still resolves to canary/);
});
test("verifyPackageRegistryState allows intentional canary latest but still checks dependencies", () => {
const packageDocsByName = new Map([
[
"paperclipai",
{
"dist-tags": {
latest: "2026.427.0-canary.3",
canary: "2026.427.0-canary.3",
},
versions: {
"2026.427.0-canary.3": {
dependencies: {
"@paperclipai/server": "2026.427.0-canary.3",
},
},
},
},
],
[
"@paperclipai/server",
{
versions: {
"2026.427.0-canary.3": {},
},
},
],
]);
assert.deepEqual(
verifyPackageRegistryState({
packageName: "paperclipai",
packageDoc: packageDocsByName.get("paperclipai"),
packageDocsByName,
channel: "canary",
distTag: "canary",
targetVersion: "2026.427.0-canary.3",
allowCanaryLatest: true,
}),
[],
);
});
test("verifyPackageRegistryState still fails when the dist-tag is stale", () => {
const packageDocsByName = new Map([
[
"@paperclipai/ui",
{
"dist-tags": {
canary: "2026.429.0-canary.2",
},
versions: {},
},
],
]);
const packageManifestsByKey = new Map([
[
createManifestLookupKey("@paperclipai/ui", "2026.430.0-canary.0"),
{
name: "@paperclipai/ui",
version: "2026.430.0-canary.0",
},
],
]);
assert.deepEqual(
verifyPackageRegistryState({
packageName: "@paperclipai/ui",
packageDoc: packageDocsByName.get("@paperclipai/ui"),
packageDocsByName,
packageManifestsByKey,
channel: "canary",
distTag: "canary",
targetVersion: "2026.430.0-canary.0",
allowCanaryLatest: false,
}),
["@paperclipai/ui: dist-tag canary resolves to 2026.429.0-canary.2, expected 2026.430.0-canary.0"],
);
});
test("verifyPackageRegistryState ignores internal peer dependency ranges", () => {
const packageDocsByName = new Map([
[
"@paperclipai/plugin-sdk",
{
"dist-tags": {
canary: "2026.430.0-canary.0",
},
versions: {
"2026.430.0-canary.0": {
peerDependencies: {
"@paperclipai/server": "^2026.430.0-canary.0",
},
},
},
},
],
]);
assert.deepEqual(
verifyPackageRegistryState({
packageName: "@paperclipai/plugin-sdk",
packageDoc: packageDocsByName.get("@paperclipai/plugin-sdk"),
packageDocsByName,
channel: "canary",
distTag: "canary",
targetVersion: "2026.430.0-canary.0",
allowCanaryLatest: false,
}),
[],
);
});
test("fetchRegistryJson times out hung requests", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (_url, { signal }) =>
new Promise((_resolve, reject) => {
signal.addEventListener(
"abort",
() => reject(new DOMException("The operation was aborted.", "AbortError")),
{ once: true },
);
});
try {
await assert.rejects(
fetchRegistryJson(new URL("https://registry.npmjs.org/@paperclipai%2Fui"), { timeoutMs: 1 }),
/timed out/,
);
} finally {
globalThis.fetch = originalFetch;
}
});