diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8ac8f710..fabaab1b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -85,11 +85,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Typecheck - run: pnpm -r typecheck + - name: Typecheck workspaces whose build scripts skip TypeScript + run: pnpm run typecheck:build-gaps - - name: Run tests - run: pnpm test:run + - name: Run general test suites + run: pnpm test:run:general - name: Verify release registry test coverage run: pnpm run test:release-registry @@ -97,7 +97,76 @@ jobs: - name: Build run: pnpm build - - name: Release canary dry run + verify_serialized_server: + name: Verify serialized server suites (${{ matrix.shard_label }}) + needs: [policy] + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - shard_index: 0 + shard_count: 4 + shard_label: 1/4 + - shard_index: 1 + shard_count: 4 + shard_label: 2/4 + - shard_index: 2 + shard_count: 4 + shard_label: 3/4 + - shard_index: 3 + shard_count: 4 + shard_label: 4/4 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run serialized server test shard + run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} + + canary_dry_run: + name: Canary Dry Run + needs: [policy] + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # `release.sh` always executes its Step 2/7 workspace build, even when + # `--skip-verify` bypasses the initial verification gate. + - name: Release canary dry run via release.sh internal build run: | git checkout -B master HEAD git checkout -- pnpm-lock.yaml @@ -126,9 +195,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Build - run: pnpm build - - name: Install Playwright run: npx playwright install --with-deps chromium diff --git a/package.json b/package.json index 9d6b5bec..152fe471 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,12 @@ "build-storybook": "pnpm --filter @paperclipai/ui build-storybook", "build": "pnpm run preflight:workspace-links && pnpm -r build", "typecheck": "pnpm run preflight:workspace-links && pnpm -r typecheck", + "typecheck:build-gaps": "pnpm run preflight:workspace-links && node scripts/run-typecheck-build-gaps.mjs", "test": "pnpm run test:run", "test:watch": "pnpm run preflight:workspace-links && vitest", "test:run": "pnpm run preflight:workspace-links && node scripts/run-vitest-stable.mjs", + "test:run:general": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk build && node scripts/run-vitest-stable.mjs --mode general", + "test:run:serialized": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk build && node scripts/run-vitest-stable.mjs --mode serialized", "db:generate": "pnpm --filter @paperclipai/db generate", "db:migrate": "pnpm --filter @paperclipai/db migrate", "issue-references:backfill": "pnpm run preflight:workspace-links && tsx scripts/backfill-issue-reference-mentions.ts", diff --git a/scripts/run-typecheck-build-gaps.mjs b/scripts/run-typecheck-build-gaps.mjs new file mode 100644 index 00000000..6210ab2a --- /dev/null +++ b/scripts/run-typecheck-build-gaps.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const repoRoot = process.cwd(); + +function fail(message) { + console.error(`[typecheck:build-gaps] ${message}`); + process.exit(1); +} + +function run(command, args) { + const result = spawnSync(command, args, { + cwd: repoRoot, + stdio: "inherit", + }); + + if (result.error) { + console.error(`[typecheck:build-gaps] Failed to spawn ${command}: ${result.error.message}`); + process.exit(1); + } + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +function readJson(filePath) { + return JSON.parse(readFileSync(filePath, "utf8")); +} + +function listWorkspacePackages() { + const result = spawnSync("pnpm", ["ls", "-r", "--depth", "-1", "--json"], { + cwd: repoRoot, + encoding: "utf8", + }); + + if (result.error) { + fail(`Unable to spawn pnpm to list workspace packages: ${result.error.message}`); + } + + if (result.status !== 0) { + fail("Unable to list pnpm workspace packages."); + } + + return JSON.parse(result.stdout); +} + +function buildSkipsTypeScript(pkg) { + const buildScript = pkg.scripts?.build; + if (typeof buildScript !== "string") { + return false; + } + + return !/\btsc\b/.test(buildScript); +} + +const workspacePackages = listWorkspacePackages(); +const buildGapCandidates = workspacePackages + .filter((workspacePkg) => workspacePkg.path !== repoRoot) + .map((workspacePkg) => ({ + name: workspacePkg.name, + path: workspacePkg.path, + pkg: readJson(path.join(workspacePkg.path, "package.json")), + })) + .filter(({ pkg }) => buildSkipsTypeScript(pkg)); +const packagesMissingTypecheck = buildGapCandidates.filter( + ({ pkg }) => typeof pkg.scripts?.typecheck !== "string", +); +if (packagesMissingTypecheck.length > 0) { + const missingNames = packagesMissingTypecheck.map((workspacePkg) => workspacePkg.name).join(", "); + fail( + `Workspace packages with build scripts that skip tsc must define a typecheck script. Missing: ${missingNames}`, + ); +} +const buildGapPackages = buildGapCandidates.filter( + ({ pkg }) => typeof pkg.scripts?.typecheck === "string", +); + +console.log( + `[typecheck:build-gaps] typechecking ${buildGapPackages.length} workspace(s): ${buildGapPackages.map(({ name }) => name).join(", ") || "(none)"}`, +); + +if (buildGapPackages.length === 0) { + process.exit(0); +} + +run("pnpm", ["--filter", "@paperclipai/plugin-sdk", "build"]); + +for (const workspacePkg of buildGapPackages) { + run("pnpm", ["--filter", workspacePkg.name, "typecheck"]); +} diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index e016719f..4ded9794 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -45,6 +45,9 @@ const additionalSerializedServerTests = new Set([ "server/src/__tests__/routines-e2e.test.ts", ]); let invocationIndex = 0; +const serializedModeName = "serialized"; +const generalModeName = "general"; +const allModeName = "all"; function walk(dir) { const entries = readdirSync(dir); @@ -77,6 +80,130 @@ function isRouteOrAuthzTest(file) { return additionalSerializedServerTests.has(file); } +function fail(message) { + console.error(`[test:run] ${message}`); + process.exit(1); +} + +function readOptionValue(argv, index, argName) { + const value = argv[index + 1]; + if (value === undefined) { + fail(`Missing value for ${argName}`); + } + + return value; +} + +function parseNonNegativeInteger(value, argName) { + const parsed = Number(value); + if (value.trim() === "" || !Number.isInteger(parsed) || parsed < 0) { + fail(`${argName} must be a non-negative integer. Received "${value}".`); + } + + return parsed; +} + +function parsePositiveInteger(value, argName) { + const parsed = Number(value); + if (value.trim() === "" || !Number.isInteger(parsed) || parsed < 1) { + fail(`${argName} must be a positive integer. Received "${value}".`); + } + + return parsed; +} + +function parseCliOptions(argv) { + let mode = allModeName; + let shardIndex = null; + let shardCount = null; + let dryRun = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + continue; + } + + if (arg === "--mode") { + mode = readOptionValue(argv, index, arg); + index += 1; + continue; + } + + if (arg.startsWith("--mode=")) { + mode = arg.slice("--mode=".length); + continue; + } + + if (arg === "--shard-index") { + shardIndex = parseNonNegativeInteger(readOptionValue(argv, index, arg), arg); + index += 1; + continue; + } + + if (arg.startsWith("--shard-index=")) { + shardIndex = parseNonNegativeInteger(arg.slice("--shard-index=".length), "--shard-index"); + continue; + } + + if (arg === "--shard-count") { + shardCount = parsePositiveInteger(readOptionValue(argv, index, arg), arg); + index += 1; + continue; + } + + if (arg.startsWith("--shard-count=")) { + shardCount = parsePositiveInteger(arg.slice("--shard-count=".length), "--shard-count"); + continue; + } + + if (arg === "--dry-run") { + dryRun = true; + continue; + } + + fail(`Unknown argument "${arg}".`); + } + + if (!new Set([allModeName, generalModeName, serializedModeName]).has(mode)) { + fail(`Unknown mode "${mode}". Expected one of: ${allModeName}, ${generalModeName}, ${serializedModeName}.`); + } + + if ((shardIndex === null) !== (shardCount === null)) { + fail("--shard-index and --shard-count must be provided together."); + } + + if (mode !== serializedModeName && shardIndex !== null) { + fail("--shard-index/--shard-count are only valid with --mode serialized."); + } + + if (mode === serializedModeName) { + const resolvedShardCount = shardCount ?? 1; + const resolvedShardIndex = shardIndex ?? 0; + if (resolvedShardIndex >= resolvedShardCount) { + fail(`--shard-index must be less than --shard-count. Received ${resolvedShardIndex} of ${resolvedShardCount}.`); + } + + return { + mode, + shardIndex: resolvedShardIndex, + shardCount: resolvedShardCount, + dryRun, + }; + } + + return { + mode, + shardIndex: null, + shardCount: null, + dryRun, + }; +} + +function selectSerializedSuites(routeTests, shardIndex, shardCount) { + return routeTests.filter((_, index) => index % shardCount === shardIndex); +} + function runVitest(args, label) { console.log(`\n[test:run] ${label}`); invocationIndex += 1; @@ -103,6 +230,38 @@ function runVitest(args, label) { } } +function runGeneralSuites(routeTests) { + const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]); + for (const project of nonServerProjects) { + runVitest(["--project", project], `non-server project ${project}`); + } + + runVitest( + ["--project", "@paperclipai/server", ...excludeRouteArgs], + `server suites excluding ${routeTests.length} serialized suites`, + ); +} + +function runSerializedSuites(routeTests, shardIndex, shardCount) { + const shardTests = selectSerializedSuites(routeTests, shardIndex, shardCount); + console.log( + `\n[test:run] serialized shard ${shardIndex + 1}/${shardCount} running ${shardTests.length} of ${routeTests.length} suites`, + ); + + for (const routeTest of shardTests) { + runVitest( + [ + "--project", + "@paperclipai/server", + routeTest.repoPath, + "--pool=forks", + "--poolOptions.forks.isolate=true", + ], + routeTest.repoPath, + ); + } +} + const routeTests = walk(serverTestsDir) .filter((file) => isRouteOrAuthzTest(toRepoPath(file))) .map((file) => ({ @@ -111,25 +270,32 @@ const routeTests = walk(serverTestsDir) })) .sort((a, b) => a.repoPath.localeCompare(b.repoPath)); -const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]); -for (const project of nonServerProjects) { - runVitest(["--project", project], `non-server project ${project}`); -} - -runVitest( - ["--project", "@paperclipai/server", ...excludeRouteArgs], - `server suites excluding ${routeTests.length} serialized suites`, -); - -for (const routeTest of routeTests) { - runVitest( - [ - "--project", - "@paperclipai/server", - routeTest.repoPath, - "--pool=forks", - "--poolOptions.forks.isolate=true", - ], - routeTest.repoPath, +const options = parseCliOptions(process.argv.slice(2)); +if (options.dryRun) { + const serializedSuites = + options.mode === serializedModeName + ? selectSerializedSuites(routeTests, options.shardIndex, options.shardCount) + : routeTests; + console.log( + JSON.stringify( + { + mode: options.mode, + shardIndex: options.shardIndex, + shardCount: options.shardCount, + serializedSuiteCount: routeTests.length, + selectedSerializedSuites: serializedSuites.map((routeTest) => routeTest.repoPath), + }, + null, + 2, + ), ); + process.exit(0); +} + +if (options.mode === generalModeName || options.mode === allModeName) { + runGeneralSuites(routeTests); +} + +if (options.mode === serializedModeName || options.mode === allModeName) { + runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1); }