fix: harden release registry verification against npm lag (#4816)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Its release automation publishes canary packages to npm and then validates the published registry state before considering the release healthy > - The failing canary run `25139465018` showed that npm can expose a newly published version through version-specific endpoints before the root package document has fully converged > - That made a successful canary publish look like a failed release because the verifier trusted stale root metadata too early > - This pull request hardens the registry verification path by preferring version-specific manifest checks, retrying convergence-sensitive failures, and distinguishing permanent failures from propagation lag > - While validating that change in CI, a separate teardown race in `heartbeat-stale-queue-invalidation.test.ts` surfaced and was hardened so the PR could pass reliably > - The benefit is that transient npm propagation lag no longer fails a successful canary publish, while genuine registry-state and dependency-integrity failures still stop the release flow promptly ## What Changed - Hardened `scripts/verify-release-registry-state.mjs` so it prefers version-specific manifest resolution over stale root metadata, adds bounded registry-fetch timeouts, and classifies failures as retriable vs non-retriable. - Updated `scripts/release-lib.sh` and `scripts/release.sh` so post-publish registry verification retries only convergence-sensitive failures and reports immediate permanent failures clearly. - Expanded `scripts/verify-release-registry-state.test.mjs` with regression coverage for stale root metadata, fetch timeout behavior, peer dependency range handling, non-retriable canary-latest cases, and related verifier edge cases. - Hardened `server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts` teardown to tolerate the late-comment foreign-key race that CI exposed while validating this branch. ## Verification - `pnpm run test:release-registry` - `node --check scripts/verify-release-registry-state.mjs` - `bash -n scripts/release.sh && bash -n scripts/release-lib.sh` - PR checks passed on head `5c422600fc12acac61f6b7c267a4dc915df622b1`: `policy`, `verify`, `e2e`, `security/snyk`, and `Greptile Review` ## Risks - Low risk. The main behavioral changes are limited to release automation and verifier retry semantics, plus a test-only teardown hardening for a CI race. > I checked [`ROADMAP.md`](ROADMAP.md). This is a narrow release bugfix and does not overlap planned core feature work. ## Model Used - OpenAI Codex via Paperclip `codex_local` with tool use and local code execution enabled. This agent session runs on a GPT-5-class coding model; the exact backend model ID/context window is not exposed by the local adapter runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I have addressed all Greptile and reviewer comments before requesting merge
This commit is contained in:
Devin Foley
@@ -263,6 +263,38 @@ wait_for_npm_package_version() {
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_release_registry_state() {
|
||||
local attempts="${1:-12}"
|
||||
local delay_seconds="${2:-5}"
|
||||
shift 2
|
||||
local attempt=1
|
||||
local output
|
||||
local status
|
||||
|
||||
while [ "$attempt" -le "$attempts" ]; do
|
||||
if output="$(node "$REPO_ROOT/scripts/verify-release-registry-state.mjs" "$@" 2>&1)"; then
|
||||
[ -n "$output" ] && printf '%s\n' "$output"
|
||||
return 0
|
||||
fi
|
||||
status=$?
|
||||
|
||||
printf '%s\n' "$output" >&2
|
||||
|
||||
if [ "$status" -eq 2 ]; then
|
||||
return "$status"
|
||||
fi
|
||||
|
||||
if [ "$attempt" -lt "$attempts" ]; then
|
||||
release_warn "npm registry metadata has not converged yet (attempt ${attempt}/${attempts}); retrying in ${delay_seconds}s."
|
||||
sleep "$delay_seconds"
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
return "${status:-1}"
|
||||
}
|
||||
|
||||
require_clean_worktree() {
|
||||
if [ -n "$(git -C "$REPO_ROOT" status --porcelain)" ]; then
|
||||
release_fail "working tree is not clean. Commit, stash, or remove changes before releasing."
|
||||
|
||||
+14
-33
@@ -11,7 +11,6 @@ release_date=""
|
||||
dry_run=false
|
||||
skip_verify=false
|
||||
print_version_only=false
|
||||
allow_canary_latest=false
|
||||
tag_name=""
|
||||
|
||||
cleanup_on_exit=false
|
||||
@@ -19,12 +18,11 @@ cleanup_on_exit=false
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/release.sh <canary|stable> [--date YYYY-MM-DD] [--dry-run] [--skip-verify] [--print-version] [--allow-canary-latest]
|
||||
./scripts/release.sh <canary|stable> [--date YYYY-MM-DD] [--dry-run] [--skip-verify] [--print-version]
|
||||
|
||||
Examples:
|
||||
./scripts/release.sh canary
|
||||
./scripts/release.sh canary --date 2026-03-17 --dry-run
|
||||
./scripts/release.sh canary --allow-canary-latest
|
||||
./scripts/release.sh stable
|
||||
./scripts/release.sh stable --date 2026-03-17 --dry-run
|
||||
./scripts/release.sh stable --date 2026-03-18 --print-version
|
||||
@@ -34,9 +32,6 @@ Notes:
|
||||
zero-padded UTC day, and P is the same-day stable patch slot.
|
||||
- Canary releases publish YYYY.MDD.P-canary.N under the npm dist-tag
|
||||
"canary" and create the git tag canary/vYYYY.MDD.P-canary.N.
|
||||
- Canary releases fail by default if npm leaves the "latest" dist-tag
|
||||
pointing at any canary. Pass --allow-canary-latest only when that is an
|
||||
intentional first-publish or migration state.
|
||||
- Stable releases publish YYYY.MDD.P under the npm dist-tag "latest" and
|
||||
create the git tag vYYYY.MDD.P.
|
||||
- Stable release notes must already exist at releases/vYYYY.MDD.P.md.
|
||||
@@ -104,7 +99,6 @@ while [ $# -gt 0 ]; do
|
||||
--dry-run) dry_run=true ;;
|
||||
--skip-verify) skip_verify=true ;;
|
||||
--print-version) print_version_only=true ;;
|
||||
--allow-canary-latest) allow_canary_latest=true ;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -121,10 +115,6 @@ done
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$allow_canary_latest" = true ] && [ "$channel" != "canary" ]; then
|
||||
release_fail "--allow-canary-latest can only be used with the canary channel."
|
||||
fi
|
||||
|
||||
PUBLISH_REMOTE="$(resolve_release_remote)"
|
||||
fetch_release_remote "$PUBLISH_REMOTE"
|
||||
|
||||
@@ -197,11 +187,6 @@ release_info " Release date (UTC): $RELEASE_DATE"
|
||||
release_info " Target stable version: $TARGET_STABLE_VERSION"
|
||||
if [ "$channel" = "canary" ]; then
|
||||
release_info " Canary version: $TARGET_PUBLISH_VERSION"
|
||||
if [ "$allow_canary_latest" = true ]; then
|
||||
release_info " latest dist-tag policy: allow canary"
|
||||
else
|
||||
release_info " latest dist-tag policy: fail if npm leaves latest on a canary"
|
||||
fi
|
||||
else
|
||||
release_info " Stable version: $TARGET_PUBLISH_VERSION"
|
||||
fi
|
||||
@@ -281,6 +266,8 @@ else
|
||||
release_info "==> Step 6/7: Confirming npm package availability and dist-tag integrity..."
|
||||
VERIFY_ATTEMPTS="${NPM_PUBLISH_VERIFY_ATTEMPTS:-12}"
|
||||
VERIFY_DELAY_SECONDS="${NPM_PUBLISH_VERIFY_DELAY_SECONDS:-5}"
|
||||
REGISTRY_STATE_VERIFY_ATTEMPTS="${NPM_REGISTRY_STATE_VERIFY_ATTEMPTS:-12}"
|
||||
REGISTRY_STATE_VERIFY_DELAY_SECONDS="${NPM_REGISTRY_STATE_VERIFY_DELAY_SECONDS:-5}"
|
||||
MISSING_PUBLISHED_PACKAGES=""
|
||||
|
||||
while IFS=$'\t' read -r _pkg_dir pkg_name pkg_version; do
|
||||
@@ -306,31 +293,25 @@ else
|
||||
--dist-tag "$DIST_TAG"
|
||||
--target-version "$TARGET_PUBLISH_VERSION"
|
||||
)
|
||||
if [ "$allow_canary_latest" = true ]; then
|
||||
verify_args+=(--allow-canary-latest)
|
||||
fi
|
||||
while IFS=$'\t' read -r _pkg_dir pkg_name _pkg_version; do
|
||||
[ -z "$pkg_name" ] && continue
|
||||
verify_args+=(--package "$pkg_name")
|
||||
done <<< "$VERSIONED_PACKAGE_INFO"
|
||||
|
||||
VERIFY_REGISTRY_STATE_ATTEMPTS="${NPM_REGISTRY_STATE_VERIFY_ATTEMPTS:-$VERIFY_ATTEMPTS}"
|
||||
VERIFY_REGISTRY_STATE_DELAY_SECONDS="${NPM_REGISTRY_STATE_VERIFY_DELAY_SECONDS:-$VERIFY_DELAY_SECONDS}"
|
||||
verify_registry_state_attempt=1
|
||||
|
||||
while true; do
|
||||
if node "$REPO_ROOT/scripts/verify-release-registry-state.mjs" "${verify_args[@]}"; then
|
||||
break
|
||||
release_info " Waiting for npm dist-tags and package metadata to converge..."
|
||||
if wait_for_release_registry_state \
|
||||
"$REGISTRY_STATE_VERIFY_ATTEMPTS" \
|
||||
"$REGISTRY_STATE_VERIFY_DELAY_SECONDS" \
|
||||
"${verify_args[@]}"; then
|
||||
:
|
||||
else
|
||||
verify_status=$?
|
||||
if [ "$verify_status" -eq 2 ]; then
|
||||
release_fail "publish completed, but registry verification failed immediately for ${TARGET_PUBLISH_VERSION}; dist-tag state is wrong or requires operator intervention"
|
||||
fi
|
||||
|
||||
if [ "$verify_registry_state_attempt" -ge "$VERIFY_REGISTRY_STATE_ATTEMPTS" ]; then
|
||||
release_fail "npm registry dist-tag verification never converged after ${VERIFY_REGISTRY_STATE_ATTEMPTS} attempt(s)."
|
||||
release_fail "publish completed, but npm dist-tags or registry metadata never converged for ${TARGET_PUBLISH_VERSION}"
|
||||
fi
|
||||
|
||||
release_warn "npm registry metadata is not fully propagated yet; retrying dist-tag verification in ${VERIFY_REGISTRY_STATE_DELAY_SECONDS}s (attempt ${verify_registry_state_attempt}/${VERIFY_REGISTRY_STATE_ATTEMPTS})..."
|
||||
sleep "$VERIFY_REGISTRY_STATE_DELAY_SECONDS"
|
||||
verify_registry_state_attempt=$((verify_registry_state_attempt + 1))
|
||||
done
|
||||
fi
|
||||
|
||||
release_info ""
|
||||
|
||||
@@ -3,11 +3,21 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const CANARY_VERSION_RE = /-canary\.\d+$/;
|
||||
const EXIT_RETRIABLE_FAILURE = 1;
|
||||
const EXIT_NON_RETRIABLE_FAILURE = 2;
|
||||
|
||||
export function isCanaryVersion(version) {
|
||||
return CANARY_VERSION_RE.test(version);
|
||||
}
|
||||
|
||||
function createExitError(message, exitCode = EXIT_RETRIABLE_FAILURE) {
|
||||
return Object.assign(new Error(message), { exitCode });
|
||||
}
|
||||
|
||||
function createProblem(message, { retriable = true } = {}) {
|
||||
return { message, retriable };
|
||||
}
|
||||
|
||||
function usage() {
|
||||
process.stderr.write(
|
||||
[
|
||||
@@ -55,58 +65,111 @@ function parseArgs(argv) {
|
||||
usage();
|
||||
process.exit(0);
|
||||
default:
|
||||
throw new Error(`unexpected argument: ${arg}`);
|
||||
throw createExitError(`unexpected argument: ${arg}`, EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.channel !== "canary" && options.channel !== "stable") {
|
||||
throw new Error("--channel must be canary or stable");
|
||||
throw createExitError("--channel must be canary or stable", EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
|
||||
if (!options.distTag) {
|
||||
throw new Error("--dist-tag is required");
|
||||
throw createExitError("--dist-tag is required", EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
|
||||
if (!options.targetVersion) {
|
||||
throw new Error("--target-version is required");
|
||||
throw createExitError("--target-version is required", EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
|
||||
if (options.packages.length === 0 || options.packages.some((name) => !name)) {
|
||||
throw new Error("at least one non-empty --package value is required");
|
||||
throw createExitError("at least one non-empty --package value is required", EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
|
||||
if (options.allowCanaryLatest && options.channel !== "canary") {
|
||||
throw new Error("--allow-canary-latest only applies to canary releases");
|
||||
throw createExitError("--allow-canary-latest only applies to canary releases", EXIT_NON_RETRIABLE_FAILURE);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function createRegistryUrl(packageName) {
|
||||
function createRegistryUrl(packageName, version = "") {
|
||||
const registry = process.env.npm_config_registry ?? process.env.NPM_CONFIG_REGISTRY ?? "https://registry.npmjs.org/";
|
||||
return new URL(encodeURIComponent(packageName), registry.endsWith("/") ? registry : `${registry}/`);
|
||||
const baseUrl = registry.endsWith("/") ? registry : `${registry}/`;
|
||||
const encodedPackage = encodeURIComponent(packageName);
|
||||
|
||||
if (!version) {
|
||||
return new URL(encodedPackage, baseUrl);
|
||||
}
|
||||
|
||||
return new URL(`${encodedPackage}/${encodeURIComponent(version)}`, baseUrl);
|
||||
}
|
||||
|
||||
async function fetchPackageDocument(packageName, { allowMissing = false } = {}) {
|
||||
const url = createRegistryUrl(packageName);
|
||||
const response = await fetch(url, {
|
||||
export async function fetchRegistryJson(url, { allowMissing = false, timeoutMs = 30_000 } = {}) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let response;
|
||||
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
accept: "application/vnd.npm.install-v1+json, application/json;q=0.9",
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new Error(`npm registry request timed out for ${url} after ${timeoutMs}ms`);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
if (response.status === 404 && allowMissing) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`npm registry request failed for ${packageName}: ${response.status} ${response.statusText}`);
|
||||
throw new Error(`npm registry request failed for ${url}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export function collectInternalDependencyProblems(manifest, packageDocsByName) {
|
||||
async function fetchPackageDocument(packageName, { allowMissing = false } = {}) {
|
||||
return fetchRegistryJson(createRegistryUrl(packageName), { allowMissing });
|
||||
}
|
||||
|
||||
async function fetchPackageManifest(packageName, version, { allowMissing = false } = {}) {
|
||||
return fetchRegistryJson(createRegistryUrl(packageName, version), { allowMissing });
|
||||
}
|
||||
|
||||
export function createManifestLookupKey(packageName, version) {
|
||||
return `${packageName}@${version}`;
|
||||
}
|
||||
|
||||
function isRangeVersionSpecifier(version) {
|
||||
return /[\^~*xX><| ]/.test(version);
|
||||
}
|
||||
|
||||
function resolvePublishedManifest(packageName, version, packageDoc, packageManifestsByKey = new Map()) {
|
||||
const directManifest = packageManifestsByKey.get(createManifestLookupKey(packageName, version));
|
||||
if (directManifest) {
|
||||
return directManifest;
|
||||
}
|
||||
|
||||
if (directManifest === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return packageDoc?.versions?.[version] ?? null;
|
||||
}
|
||||
|
||||
function collectInternalDependencyProblemEntries(
|
||||
manifest,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey = new Map(),
|
||||
) {
|
||||
const problems = [];
|
||||
const sections = [
|
||||
["dependencies", manifest.dependencies ?? {}],
|
||||
@@ -122,20 +185,41 @@ export function collectInternalDependencyProblems(manifest, packageDocsByName) {
|
||||
|
||||
if (typeof dependencyVersion !== "string" || !dependencyVersion) {
|
||||
problems.push(
|
||||
createProblem(
|
||||
`${sectionName} declares ${dependencyName} with a non-string version: ${JSON.stringify(dependencyVersion)}`,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const dependencyDoc = packageDocsByName.get(dependencyName);
|
||||
if (!dependencyDoc) {
|
||||
problems.push(`${sectionName} requires ${dependencyName}@${dependencyVersion}, but that package is not published`);
|
||||
// Peer dependency ranges express compatibility, not a manifest that can be fetched directly.
|
||||
if (sectionName === "peerDependencies" && isRangeVersionSpecifier(dependencyVersion)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(dependencyVersion in (dependencyDoc.versions ?? {}))) {
|
||||
const dependencyManifest = resolvePublishedManifest(
|
||||
dependencyName,
|
||||
dependencyVersion,
|
||||
packageDocsByName.get(dependencyName),
|
||||
packageManifestsByKey,
|
||||
);
|
||||
const dependencyLookupKey = createManifestLookupKey(dependencyName, dependencyVersion);
|
||||
|
||||
if (!dependencyManifest) {
|
||||
const dependencyDoc = packageDocsByName.get(dependencyName);
|
||||
if (!dependencyDoc && !packageManifestsByKey.has(dependencyLookupKey)) {
|
||||
problems.push(
|
||||
createProblem(
|
||||
`${sectionName} requires ${dependencyName}@${dependencyVersion}, but npm publication metadata was not fetched for that dependency`,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
problems.push(
|
||||
createProblem(
|
||||
`${sectionName} requires ${dependencyName}@${dependencyVersion}, but npm does not expose that version`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -144,21 +228,34 @@ export function collectInternalDependencyProblems(manifest, packageDocsByName) {
|
||||
return problems;
|
||||
}
|
||||
|
||||
function requireManifest(packageName, version, packageDoc, problems) {
|
||||
const manifest = packageDoc.versions?.[version];
|
||||
export function collectInternalDependencyProblems(
|
||||
manifest,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey = new Map(),
|
||||
) {
|
||||
return collectInternalDependencyProblemEntries(
|
||||
manifest,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
).map((problem) => problem.message);
|
||||
}
|
||||
|
||||
function requireManifest(packageName, version, packageDoc, packageManifestsByKey, problems) {
|
||||
const manifest = resolvePublishedManifest(packageName, version, packageDoc, packageManifestsByKey);
|
||||
if (!manifest) {
|
||||
if (problems) {
|
||||
problems.push(`${packageName}: npm registry is missing manifest data for ${version}`);
|
||||
problems.push(createProblem(`${packageName}: npm registry is missing manifest data for ${version}`));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function verifyPackageRegistryState({
|
||||
export function verifyPackageRegistryProblems({
|
||||
packageName,
|
||||
packageDoc,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey = new Map(),
|
||||
channel,
|
||||
distTag,
|
||||
targetVersion,
|
||||
@@ -170,14 +267,20 @@ export function verifyPackageRegistryState({
|
||||
|
||||
if (taggedVersion !== targetVersion) {
|
||||
problems.push(
|
||||
createProblem(
|
||||
`${packageName}: dist-tag ${distTag} resolves to ${taggedVersion ?? "<missing>"}, expected ${targetVersion}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const targetManifest = requireManifest(packageName, targetVersion, packageDoc, problems);
|
||||
const targetManifest = requireManifest(packageName, targetVersion, packageDoc, packageManifestsByKey, problems);
|
||||
if (targetManifest) {
|
||||
for (const problem of collectInternalDependencyProblems(targetManifest, packageDocsByName)) {
|
||||
problems.push(`${packageName}@${targetVersion}: ${problem}`);
|
||||
for (const problem of collectInternalDependencyProblemEntries(
|
||||
targetManifest,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
)) {
|
||||
problems.push(createProblem(`${packageName}@${targetVersion}: ${problem.message}`, problem));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,15 +289,28 @@ export function verifyPackageRegistryState({
|
||||
|
||||
if (latestVersion && isCanaryVersion(latestVersion) && !allowCanaryLatest) {
|
||||
problems.push(
|
||||
`${packageName}: latest dist-tag still resolves to canary ${latestVersion}; rerun with --allow-canary-latest only when that state is intentional`,
|
||||
createProblem(
|
||||
`${packageName}: latest dist-tag still resolves to canary ${latestVersion}; if that state is intentional, rerun the verification script directly with --allow-canary-latest`,
|
||||
{ retriable: false },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (latestVersion && isCanaryVersion(latestVersion)) {
|
||||
const latestManifest = requireManifest(packageName, latestVersion, packageDoc, problems);
|
||||
const latestManifest = requireManifest(
|
||||
packageName,
|
||||
latestVersion,
|
||||
packageDoc,
|
||||
packageManifestsByKey,
|
||||
problems,
|
||||
);
|
||||
if (latestManifest) {
|
||||
for (const problem of collectInternalDependencyProblems(latestManifest, packageDocsByName)) {
|
||||
problems.push(`${packageName}@${latestVersion} via latest: ${problem}`);
|
||||
for (const problem of collectInternalDependencyProblemEntries(
|
||||
latestManifest,
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
)) {
|
||||
problems.push(createProblem(`${packageName}@${latestVersion} via latest: ${problem.message}`, problem));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -203,10 +319,46 @@ export function verifyPackageRegistryState({
|
||||
return problems;
|
||||
}
|
||||
|
||||
export function verifyPackageRegistryState(options) {
|
||||
return verifyPackageRegistryProblems(options).map((problem) => problem.message);
|
||||
}
|
||||
|
||||
function collectInternalDependencyVersions(manifest) {
|
||||
const dependencyVersions = [];
|
||||
|
||||
for (const [sectionName, deps] of [
|
||||
["dependencies", manifest.dependencies ?? {}],
|
||||
["optionalDependencies", manifest.optionalDependencies ?? {}],
|
||||
["peerDependencies", manifest.peerDependencies ?? {}],
|
||||
]) {
|
||||
for (const [dependencyName, dependencyVersion] of Object.entries(deps)) {
|
||||
if (!dependencyName.startsWith("@paperclipai/")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeof dependencyVersion !== "string" || !dependencyVersion) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sectionName === "peerDependencies" && isRangeVersionSpecifier(dependencyVersion)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
dependencyVersions.push({
|
||||
packageName: dependencyName,
|
||||
version: dependencyVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return dependencyVersions;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const packageNames = [...new Set(options.packages)];
|
||||
const packageDocsByName = new Map();
|
||||
const packageManifestsByKey = new Map();
|
||||
|
||||
await Promise.all(
|
||||
packageNames.map(async (packageName) => {
|
||||
@@ -214,40 +366,60 @@ async function main() {
|
||||
}),
|
||||
);
|
||||
|
||||
const additionalInternalDeps = new Set();
|
||||
for (const packageDoc of packageDocsByName.values()) {
|
||||
const versionsToCheck = new Set([options.targetVersion]);
|
||||
const latestVersion = packageDoc["dist-tags"]?.latest;
|
||||
const versionsToFetchByPackage = new Map();
|
||||
for (const packageName of packageNames) {
|
||||
const packageDoc = packageDocsByName.get(packageName);
|
||||
const versionsToFetch = new Set([options.targetVersion]);
|
||||
const latestVersion = packageDoc?.["dist-tags"]?.latest;
|
||||
if (latestVersion && isCanaryVersion(latestVersion)) {
|
||||
versionsToCheck.add(latestVersion);
|
||||
versionsToFetch.add(latestVersion);
|
||||
}
|
||||
versionsToFetchByPackage.set(packageName, versionsToFetch);
|
||||
}
|
||||
|
||||
for (const version of versionsToCheck) {
|
||||
const manifest = packageDoc.versions?.[version];
|
||||
await Promise.all(
|
||||
[...versionsToFetchByPackage.entries()].flatMap(([packageName, versionsToFetch]) =>
|
||||
[...versionsToFetch].map(async (version) => {
|
||||
packageManifestsByKey.set(
|
||||
createManifestLookupKey(packageName, version),
|
||||
await fetchPackageManifest(packageName, version, { allowMissing: true }),
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const dependencyVersionsByKey = new Map();
|
||||
for (const [packageName, versionsToFetch] of versionsToFetchByPackage.entries()) {
|
||||
for (const version of versionsToFetch) {
|
||||
const manifest = resolvePublishedManifest(
|
||||
packageName,
|
||||
version,
|
||||
packageDocsByName.get(packageName),
|
||||
packageManifestsByKey,
|
||||
);
|
||||
if (!manifest) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const deps of [
|
||||
manifest.dependencies ?? {},
|
||||
manifest.optionalDependencies ?? {},
|
||||
manifest.peerDependencies ?? {},
|
||||
]) {
|
||||
for (const dependencyName of Object.keys(deps)) {
|
||||
if (dependencyName.startsWith("@paperclipai/")) {
|
||||
additionalInternalDeps.add(dependencyName);
|
||||
}
|
||||
}
|
||||
for (const dependencyVersion of collectInternalDependencyVersions(manifest)) {
|
||||
dependencyVersionsByKey.set(
|
||||
createManifestLookupKey(dependencyVersion.packageName, dependencyVersion.version),
|
||||
dependencyVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missingDeps = [...additionalInternalDeps].filter((dep) => !packageDocsByName.has(dep));
|
||||
await Promise.all(
|
||||
missingDeps.map(async (dependencyName) => {
|
||||
packageDocsByName.set(
|
||||
dependencyName,
|
||||
await fetchPackageDocument(dependencyName, { allowMissing: true }),
|
||||
[...dependencyVersionsByKey.values()].map(async ({ packageName, version }) => {
|
||||
const lookupKey = createManifestLookupKey(packageName, version);
|
||||
if (packageManifestsByKey.has(lookupKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
packageManifestsByKey.set(
|
||||
lookupKey,
|
||||
await fetchPackageManifest(packageName, version, { allowMissing: true }),
|
||||
);
|
||||
}),
|
||||
);
|
||||
@@ -256,10 +428,11 @@ async function main() {
|
||||
|
||||
for (const packageName of packageNames) {
|
||||
process.stdout.write(` Verifying ${packageName} on dist-tag ${options.distTag}\n`);
|
||||
const packageProblems = verifyPackageRegistryState({
|
||||
const packageProblems = verifyPackageRegistryProblems({
|
||||
packageName,
|
||||
packageDoc: packageDocsByName.get(packageName),
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
channel: options.channel,
|
||||
distTag: options.distTag,
|
||||
targetVersion: options.targetVersion,
|
||||
@@ -272,13 +445,16 @@ async function main() {
|
||||
}
|
||||
|
||||
for (const problem of packageProblems) {
|
||||
process.stderr.write(` ✗ ${problem}\n`);
|
||||
process.stderr.write(` ✗ ${problem.message}\n`);
|
||||
problems.push(problem);
|
||||
}
|
||||
}
|
||||
|
||||
if (problems.length > 0) {
|
||||
throw new Error(`npm registry verification failed for ${problems.length} problem(s)`);
|
||||
const exitCode = problems.some((problem) => !problem.retriable)
|
||||
? EXIT_NON_RETRIABLE_FAILURE
|
||||
: EXIT_RETRIABLE_FAILURE;
|
||||
throw createExitError(`npm registry verification failed for ${problems.length} problem(s)`, exitCode);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,6 +463,6 @@ const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process
|
||||
if (isDirectRun) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`Error: ${error.message}\n`);
|
||||
process.exit(1);
|
||||
process.exit(error.exitCode ?? EXIT_RETRIABLE_FAILURE);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ import test from "node:test";
|
||||
|
||||
import {
|
||||
collectInternalDependencyProblems,
|
||||
createManifestLookupKey,
|
||||
fetchRegistryJson,
|
||||
isCanaryVersion,
|
||||
verifyPackageRegistryProblems,
|
||||
verifyPackageRegistryState,
|
||||
} from "./verify-release-registry-state.mjs";
|
||||
|
||||
@@ -30,9 +33,119 @@ test("collectInternalDependencyProblems flags missing internal versions", () =>
|
||||
],
|
||||
]);
|
||||
|
||||
assert.deepEqual(collectInternalDependencyProblems(manifest, packageDocsByName), [
|
||||
"dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version",
|
||||
assert.deepEqual(
|
||||
collectInternalDependencyProblems(manifest, packageDocsByName),
|
||||
["dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version"],
|
||||
);
|
||||
});
|
||||
|
||||
test("collectInternalDependencyProblems accepts version-specific manifests when the root document is stale", () => {
|
||||
const manifest = {
|
||||
dependencies: {
|
||||
"@paperclipai/plugin-sdk": "2026.425.0-canary.5",
|
||||
},
|
||||
};
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
"@paperclipai/plugin-sdk",
|
||||
{
|
||||
versions: {},
|
||||
},
|
||||
],
|
||||
]);
|
||||
const packageManifestsByKey = new Map([
|
||||
[
|
||||
createManifestLookupKey("@paperclipai/plugin-sdk", "2026.425.0-canary.5"),
|
||||
{ name: "@paperclipai/plugin-sdk", version: "2026.425.0-canary.5" },
|
||||
],
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
collectInternalDependencyProblems(manifest, packageDocsByName, packageManifestsByKey),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("collectInternalDependencyProblems ignores peer dependency range specifiers", () => {
|
||||
const manifest = {
|
||||
peerDependencies: {
|
||||
"@paperclipai/server": "^2026.430.0-canary.0",
|
||||
},
|
||||
};
|
||||
|
||||
assert.deepEqual(
|
||||
collectInternalDependencyProblems(manifest, new Map()),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("collectInternalDependencyProblems reports unfetched transitive dependency metadata neutrally", () => {
|
||||
const manifest = {
|
||||
optionalDependencies: {
|
||||
"@paperclipai/browser": "2026.430.0-canary.0",
|
||||
},
|
||||
};
|
||||
|
||||
assert.deepEqual(
|
||||
collectInternalDependencyProblems(manifest, new Map()),
|
||||
[
|
||||
"optionalDependencies requires @paperclipai/browser@2026.430.0-canary.0, but npm publication metadata was not fetched for that dependency",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryState tolerates a stale root versions map when dist-tags and direct manifests are correct", () => {
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
"@paperclipai/ui",
|
||||
{
|
||||
"dist-tags": {
|
||||
canary: "2026.430.0-canary.0",
|
||||
latest: "2026.430.0",
|
||||
},
|
||||
versions: {},
|
||||
},
|
||||
],
|
||||
[
|
||||
"@paperclipai/shared",
|
||||
{
|
||||
versions: {},
|
||||
},
|
||||
],
|
||||
]);
|
||||
const packageManifestsByKey = new Map([
|
||||
[
|
||||
createManifestLookupKey("@paperclipai/ui", "2026.430.0-canary.0"),
|
||||
{
|
||||
name: "@paperclipai/ui",
|
||||
version: "2026.430.0-canary.0",
|
||||
dependencies: {
|
||||
"@paperclipai/shared": "2026.430.0-canary.0",
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
createManifestLookupKey("@paperclipai/shared", "2026.430.0-canary.0"),
|
||||
{
|
||||
name: "@paperclipai/shared",
|
||||
version: "2026.430.0-canary.0",
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
verifyPackageRegistryState({
|
||||
packageName: "@paperclipai/ui",
|
||||
packageDoc: packageDocsByName.get("@paperclipai/ui"),
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
channel: "canary",
|
||||
distTag: "canary",
|
||||
targetVersion: "2026.430.0-canary.0",
|
||||
allowCanaryLatest: false,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryState fails when canary latest is left in place by default", () => {
|
||||
@@ -79,12 +192,42 @@ test("verifyPackageRegistryState fails when canary latest is left in place by de
|
||||
allowCanaryLatest: false,
|
||||
}),
|
||||
[
|
||||
"@paperclipai/plugin-e2b: latest dist-tag still resolves to canary 2026.425.0-canary.5; rerun with --allow-canary-latest only when that state is intentional",
|
||||
"@paperclipai/plugin-e2b: latest dist-tag still resolves to canary 2026.425.0-canary.5; if that state is intentional, rerun the verification script directly with --allow-canary-latest",
|
||||
"@paperclipai/plugin-e2b@2026.425.0-canary.5 via latest: dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryProblems marks canary latest drift as non-retriable", () => {
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
"@paperclipai/plugin-e2b",
|
||||
{
|
||||
"dist-tags": {
|
||||
latest: "2026.425.0-canary.5",
|
||||
canary: "2026.427.0-canary.3",
|
||||
},
|
||||
versions: {
|
||||
"2026.427.0-canary.3": {},
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const problems = verifyPackageRegistryProblems({
|
||||
packageName: "@paperclipai/plugin-e2b",
|
||||
packageDoc: packageDocsByName.get("@paperclipai/plugin-e2b"),
|
||||
packageDocsByName,
|
||||
channel: "canary",
|
||||
distTag: "canary",
|
||||
targetVersion: "2026.427.0-canary.3",
|
||||
allowCanaryLatest: false,
|
||||
});
|
||||
|
||||
assert.equal(problems[0]?.retriable, false);
|
||||
assert.match(problems[0]?.message ?? "", /latest dist-tag still resolves to canary/);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryState allows intentional canary latest but still checks dependencies", () => {
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
@@ -126,3 +269,95 @@ test("verifyPackageRegistryState allows intentional canary latest but still chec
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryState still fails when the dist-tag is stale", () => {
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
"@paperclipai/ui",
|
||||
{
|
||||
"dist-tags": {
|
||||
canary: "2026.429.0-canary.2",
|
||||
},
|
||||
versions: {},
|
||||
},
|
||||
],
|
||||
]);
|
||||
const packageManifestsByKey = new Map([
|
||||
[
|
||||
createManifestLookupKey("@paperclipai/ui", "2026.430.0-canary.0"),
|
||||
{
|
||||
name: "@paperclipai/ui",
|
||||
version: "2026.430.0-canary.0",
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
verifyPackageRegistryState({
|
||||
packageName: "@paperclipai/ui",
|
||||
packageDoc: packageDocsByName.get("@paperclipai/ui"),
|
||||
packageDocsByName,
|
||||
packageManifestsByKey,
|
||||
channel: "canary",
|
||||
distTag: "canary",
|
||||
targetVersion: "2026.430.0-canary.0",
|
||||
allowCanaryLatest: false,
|
||||
}),
|
||||
["@paperclipai/ui: dist-tag canary resolves to 2026.429.0-canary.2, expected 2026.430.0-canary.0"],
|
||||
);
|
||||
});
|
||||
|
||||
test("verifyPackageRegistryState ignores internal peer dependency ranges", () => {
|
||||
const packageDocsByName = new Map([
|
||||
[
|
||||
"@paperclipai/plugin-sdk",
|
||||
{
|
||||
"dist-tags": {
|
||||
canary: "2026.430.0-canary.0",
|
||||
},
|
||||
versions: {
|
||||
"2026.430.0-canary.0": {
|
||||
peerDependencies: {
|
||||
"@paperclipai/server": "^2026.430.0-canary.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
verifyPackageRegistryState({
|
||||
packageName: "@paperclipai/plugin-sdk",
|
||||
packageDoc: packageDocsByName.get("@paperclipai/plugin-sdk"),
|
||||
packageDocsByName,
|
||||
channel: "canary",
|
||||
distTag: "canary",
|
||||
targetVersion: "2026.430.0-canary.0",
|
||||
allowCanaryLatest: false,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("fetchRegistryJson times out hung requests", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = (_url, { signal }) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("The operation was aborted.", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
fetchRegistryJson(new URL("https://registry.npmjs.org/@paperclipai%2Fui"), { timeoutMs: 1 }),
|
||||
/timed out/,
|
||||
);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -87,6 +88,40 @@ async function waitForCondition(fn: () => Promise<boolean>, timeoutMs = 3_000) {
|
||||
return fn();
|
||||
}
|
||||
|
||||
async function cleanupHeartbeatInvalidationFixture(db: ReturnType<typeof createDb>) {
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
await db.delete(companySkills);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agentRuntimeState);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
return;
|
||||
} catch (error) {
|
||||
const isLateCommentRace =
|
||||
error instanceof Error &&
|
||||
error.message.includes("issue_comments_issue_id_issues_id_fk");
|
||||
if (!isLateCommentRace || attempt === 4) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Heartbeat completion can write issue-thread comments shortly after the
|
||||
// run leaves queued/running. Retry the dependent deletes once those land.
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type SeedOptions = {
|
||||
agentName?: string;
|
||||
agentRole?: string;
|
||||
@@ -103,6 +138,9 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
let heartbeat!: ReturnType<typeof heartbeatService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
const countExecuteCallsForRun = (runId: string) =>
|
||||
mockAdapterExecute.mock.calls.filter(([context]) => context?.runId === runId).length;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-stale-queue-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
@@ -137,22 +175,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await db.delete(companySkills);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agentRuntimeState);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
await cleanupHeartbeatInvalidationFixture(db);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -230,6 +253,43 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
return { runId, wakeupRequestId };
|
||||
}
|
||||
|
||||
async function seedContinuationSummary(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
body: string;
|
||||
}) {
|
||||
const documentId = randomUUID();
|
||||
const revisionId = randomUUID();
|
||||
await db.insert(documents).values({
|
||||
id: documentId,
|
||||
companyId: input.companyId,
|
||||
title: "Continuation Summary",
|
||||
format: "markdown",
|
||||
latestBody: input.body,
|
||||
latestRevisionId: revisionId,
|
||||
latestRevisionNumber: 1,
|
||||
createdByAgentId: input.agentId,
|
||||
updatedByAgentId: input.agentId,
|
||||
});
|
||||
await db.insert(documentRevisions).values({
|
||||
id: revisionId,
|
||||
companyId: input.companyId,
|
||||
documentId,
|
||||
revisionNumber: 1,
|
||||
title: "Continuation Summary",
|
||||
format: "markdown",
|
||||
body: input.body,
|
||||
createdByAgentId: input.agentId,
|
||||
});
|
||||
await db.insert(issueDocuments).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.issueId,
|
||||
documentId,
|
||||
key: ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
|
||||
});
|
||||
}
|
||||
|
||||
it("cancels queued runs when the issue assignee changes before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent({ agentName: "OriginalCoder" });
|
||||
const replacementAgentId = randomUUID();
|
||||
@@ -300,7 +360,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_assignee_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("assignee changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels queued runs when the issue reaches a terminal status before the run starts", async () => {
|
||||
@@ -349,7 +409,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_terminal_status");
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels queued max-turn continuations when the issue is no longer in_progress before the run starts", async () => {
|
||||
@@ -409,7 +469,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_not_in_progress" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("no longer in_progress");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels queued max-turn continuations when another continuation owns the issue lock", async () => {
|
||||
@@ -497,7 +557,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("execution lock");
|
||||
expect(issue?.executionRunId).toBe(lockOwnerRunId);
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels queued in_review runs when the current participant changes before the run starts", async () => {
|
||||
@@ -577,7 +637,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_review_participant_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("in-review participant changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
|
||||
it("still runs comment-driven wakes on in_review issues even when the agent is no longer the current participant", async () => {
|
||||
@@ -695,6 +755,77 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(run?.status).toBe("succeeded");
|
||||
expect(run?.errorCode).toBeNull();
|
||||
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
|
||||
expect(countExecuteCallsForRun(runId)).toBe(1);
|
||||
});
|
||||
|
||||
it("cancels queued continuation recovery when the continuation summary parks executor work for review", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Implementation parked for review",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await seedContinuationSummary({
|
||||
companyId,
|
||||
issueId,
|
||||
agentId,
|
||||
body: [
|
||||
"# Continuation Summary",
|
||||
"",
|
||||
"## Next Action",
|
||||
"",
|
||||
"- Wait for reviewer feedback or approval before continuing executor work.",
|
||||
].join("\n"),
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_continuation_needed",
|
||||
invocationSource: "automation",
|
||||
contextExtras: {
|
||||
retryReason: "issue_continuation_needed",
|
||||
},
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_continuation_waiting_on_review");
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_continuation_waiting_on_review" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("continuation summary says the executor should wait");
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
ISSUE_CONTINUATION_SUMMARY_MAX_BODY_CHARS,
|
||||
buildContinuationSummaryMarkdown,
|
||||
continuationSummaryParksExecutor,
|
||||
extractContinuationSummaryNextAction,
|
||||
} from "../services/issue-continuation-summary.js";
|
||||
|
||||
describe("issue continuation summaries", () => {
|
||||
@@ -83,4 +85,31 @@ describe("issue continuation summaries", () => {
|
||||
expect(body).toContain("Latest run error (adapter_failed): adapter failed");
|
||||
expect(body).toContain("Inspect the failed run, fix the cause");
|
||||
});
|
||||
|
||||
it("detects continuation summaries that explicitly park executor work for review", () => {
|
||||
const body = [
|
||||
"# Continuation Summary",
|
||||
"",
|
||||
"## Next Action",
|
||||
"",
|
||||
"- Wait for reviewer feedback or approval before continuing executor work.",
|
||||
].join("\n");
|
||||
|
||||
expect(extractContinuationSummaryNextAction(body)).toBe(
|
||||
"Wait for reviewer feedback or approval before continuing executor work.",
|
||||
);
|
||||
expect(continuationSummaryParksExecutor(body)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not park executor work when the next action is still runnable", () => {
|
||||
const body = [
|
||||
"# Continuation Summary",
|
||||
"",
|
||||
"## Next Action",
|
||||
"",
|
||||
"- Re-check run `25145432006`, then move the issue to `in_review` if the final step is green.",
|
||||
].join("\n");
|
||||
|
||||
expect(continuationSummaryParksExecutor(body)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -104,6 +104,7 @@ import {
|
||||
issueTreeControlService,
|
||||
} from "./issue-tree-control.js";
|
||||
import {
|
||||
continuationSummaryParksExecutor,
|
||||
getIssueContinuationSummaryDocument,
|
||||
refreshIssueContinuationSummary,
|
||||
} from "./issue-continuation-summary.js";
|
||||
@@ -5977,7 +5978,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
| "issue_terminal_status"
|
||||
| "issue_not_in_progress"
|
||||
| "issue_execution_lock_changed"
|
||||
| "issue_review_participant_changed";
|
||||
| "issue_review_participant_changed"
|
||||
| "issue_continuation_waiting_on_review";
|
||||
details: Record<string, unknown>;
|
||||
};
|
||||
|
||||
@@ -6010,8 +6012,38 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const isInteractionWake = allowsIssueInteractionWake(context);
|
||||
const resumeIntent = context.resumeIntent === true || context.followUpRequested === true;
|
||||
const wakeReason = readNonEmptyString(context.wakeReason);
|
||||
const retryReason = readNonEmptyString(context.retryReason) ?? run.scheduledRetryReason ?? null;
|
||||
|
||||
if (
|
||||
issue.status === "in_progress" &&
|
||||
!wakeCommentId &&
|
||||
(wakeReason === "issue_continuation_needed" || retryReason === "issue_continuation_needed")
|
||||
) {
|
||||
const queuedWake = parseObject(context.paperclipWake);
|
||||
const queuedContinuationSummary =
|
||||
readNonEmptyString(parseObject(context.paperclipContinuationSummary).body) ??
|
||||
readNonEmptyString(parseObject(queuedWake.continuationSummary).body);
|
||||
const currentContinuationSummary = queuedContinuationSummary
|
||||
? null
|
||||
: await getIssueContinuationSummaryDocument(db, issueId);
|
||||
const continuationSummaryBody = queuedContinuationSummary ?? currentContinuationSummary?.body ?? null;
|
||||
if (continuationSummaryParksExecutor(continuationSummaryBody)) {
|
||||
return {
|
||||
stale: true,
|
||||
errorCode: "issue_continuation_waiting_on_review",
|
||||
reason:
|
||||
"Cancelled because the continuation summary says the executor should wait for reviewer feedback or approval before more work starts",
|
||||
details: {
|
||||
issueId,
|
||||
wakeReason,
|
||||
retryReason,
|
||||
nextAction: continuationSummaryBody,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (issue.assigneeAgentId !== run.agentId && !isInteractionWake) {
|
||||
return {
|
||||
stale: true,
|
||||
|
||||
@@ -9,6 +9,8 @@ export const ISSUE_CONTINUATION_SUMMARY_TITLE = "Continuation Summary";
|
||||
export const ISSUE_CONTINUATION_SUMMARY_MAX_BODY_CHARS = 8_000;
|
||||
const SUMMARY_SECTION_MAX_CHARS = 1_200;
|
||||
const PATH_CANDIDATE_RE = /(?:^|[\s`"'(])((?:server|ui|packages|doc|scripts|\.github)\/[A-Za-z0-9._/-]+)/g;
|
||||
const WAITING_FOR_REVIEW_OR_APPROVAL_RE =
|
||||
/\bwait(?:ing)? for\b.{0,160}\b(?:review(?:er)?(?: feedback)?|approval|board|human|user|operator)\b/i;
|
||||
|
||||
type IssueSummaryInput = {
|
||||
id: string;
|
||||
@@ -120,6 +122,16 @@ function extractPreviousNextAction(previousBody: string | null | undefined) {
|
||||
.find(Boolean) ?? null;
|
||||
}
|
||||
|
||||
export function extractContinuationSummaryNextAction(body: string | null | undefined) {
|
||||
return extractPreviousNextAction(body);
|
||||
}
|
||||
|
||||
export function continuationSummaryParksExecutor(body: string | null | undefined) {
|
||||
const nextAction = extractContinuationSummaryNextAction(body);
|
||||
if (!nextAction) return false;
|
||||
return WAITING_FOR_REVIEW_OR_APPROVAL_RE.test(nextAction);
|
||||
}
|
||||
|
||||
export function buildContinuationSummaryMarkdown(input: {
|
||||
issue: IssueSummaryInput;
|
||||
run: RunSummaryInput;
|
||||
|
||||
@@ -231,6 +231,36 @@ function formatIssueLinksForComment(relations: Array<{ identifier?: string | nul
|
||||
.join(", ");
|
||||
}
|
||||
|
||||
function unwrapDatabaseConflictError(error: unknown) {
|
||||
if (!error || typeof error !== "object") return null;
|
||||
|
||||
const candidate = error as {
|
||||
code?: string;
|
||||
constraint?: string;
|
||||
constraint_name?: string;
|
||||
message?: string;
|
||||
cause?: unknown;
|
||||
};
|
||||
|
||||
if (
|
||||
typeof candidate.code === "string" ||
|
||||
typeof candidate.constraint === "string" ||
|
||||
typeof candidate.constraint_name === "string"
|
||||
) {
|
||||
return candidate;
|
||||
}
|
||||
|
||||
const cause = candidate.cause;
|
||||
if (!cause || typeof cause !== "object") return candidate;
|
||||
|
||||
return cause as {
|
||||
code?: string;
|
||||
constraint?: string;
|
||||
constraint_name?: string;
|
||||
message?: string;
|
||||
};
|
||||
}
|
||||
|
||||
function isAgentInvokable(agent: typeof agents.$inferSelect | null | undefined) {
|
||||
return Boolean(agent && !["paused", "terminated", "pending_approval"].includes(agent.status));
|
||||
}
|
||||
@@ -928,21 +958,23 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
}
|
||||
|
||||
function isUniqueStaleRunEvaluationConflict(error: unknown) {
|
||||
if (!error || typeof error !== "object") return false;
|
||||
const maybe = error as { code?: string; constraint?: string; message?: string };
|
||||
const maybe = unwrapDatabaseConflictError(error);
|
||||
if (!maybe) return false;
|
||||
return maybe.code === "23505" &&
|
||||
(
|
||||
maybe.constraint === "issues_active_stale_run_evaluation_uq" ||
|
||||
maybe.constraint_name === "issues_active_stale_run_evaluation_uq" ||
|
||||
typeof maybe.message === "string" && maybe.message.includes("issues_active_stale_run_evaluation_uq")
|
||||
);
|
||||
}
|
||||
|
||||
function isUniqueStrandedIssueRecoveryConflict(error: unknown) {
|
||||
if (!error || typeof error !== "object") return false;
|
||||
const maybe = error as { code?: string; constraint?: string; message?: string };
|
||||
const maybe = unwrapDatabaseConflictError(error);
|
||||
if (!maybe) return false;
|
||||
return maybe.code === "23505" &&
|
||||
(
|
||||
maybe.constraint === "issues_active_stranded_issue_recovery_uq" ||
|
||||
maybe.constraint_name === "issues_active_stranded_issue_recovery_uq" ||
|
||||
typeof maybe.message === "string" && maybe.message.includes("issues_active_stranded_issue_recovery_uq")
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user