diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index 33022502..572b012a 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -1,6 +1,8 @@ import { defineConfig } from "@playwright/test"; -const PORT = Number(process.env.PAPERCLIP_E2E_PORT ?? 3100); +// Use a dedicated port so e2e tests always start their own server in local_trusted mode, +// even when the dev server is running on :3100 in authenticated mode. +const PORT = Number(process.env.PAPERCLIP_E2E_PORT ?? 3199); const BASE_URL = `http://127.0.0.1:${PORT}`; export default defineConfig({ @@ -29,6 +31,11 @@ export default defineConfig({ timeout: 120_000, stdout: "pipe", stderr: "pipe", + env: { + ...process.env, + PORT: String(PORT), + PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", + }, }, outputDir: "./test-results", reporter: [["list"], ["html", { open: "never", outputFolder: "./playwright-report" }]], diff --git a/tests/e2e/signoff-policy.spec.ts b/tests/e2e/signoff-policy.spec.ts index a534415e..97e67746 100644 --- a/tests/e2e/signoff-policy.spec.ts +++ b/tests/e2e/signoff-policy.spec.ts @@ -1,4 +1,4 @@ -import { test, expect, type APIRequestContext } from "@playwright/test"; +import { test, expect, request as pwRequest, type APIRequestContext } from "@playwright/test"; /** * E2E: Signoff execution policy flow. @@ -11,119 +11,219 @@ import { test, expect, type APIRequestContext } from "@playwright/test"; * 5. Approver approves → execution completes, issue marked done * 6. Verify "changes requested" flow returns to executor * - * This test is API-driven with UI verification of execution state labels. + * Requires local_trusted deployment mode (set in playwright.config.ts webServer env). + * + * Agent auth flow: + * - Board request (local_trusted auto-auth) handles setup/teardown. + * - Agent-specific actions use API keys + heartbeat run IDs. + * - Reviewers/approvers invoke heartbeat runs (gets run IDs) then PATCH + * directly without checkout (checkout would force in_progress, breaking + * the in_review state the signoff policy requires). */ +const PORT = Number(process.env.PAPERCLIP_E2E_PORT ?? 3199); +const BASE_URL = `http://127.0.0.1:${PORT}`; const COMPANY_NAME = `E2E-Signoff-${Date.now()}`; -interface TestContext { - baseUrl: string; - companyId: string; - companyPrefix: string; - executorAgentId: string; - reviewerAgentId: string; - approverAgentId: string; +interface AgentAuth { + agentId: string; + token: string; + keyId: string; + request: APIRequestContext; } -async function setupCompany(request: APIRequestContext, baseUrl: string): Promise { +interface TestContext { + companyId: string; + companyPrefix: string; + executor: AgentAuth; + reviewer: AgentAuth; + approver: AgentAuth; + boardRequest: APIRequestContext; + issueIds: string[]; +} + +/** Create an authenticated APIRequestContext for an agent (token set, no run ID yet). */ +async function createAgentRequest(token: string): Promise { + return pwRequest.newContext({ + baseURL: BASE_URL, + extraHTTPHeaders: { Authorization: `Bearer ${token}` }, + }); +} + +/** Invoke a heartbeat run for an agent, returning the run ID. */ +async function invokeHeartbeat(board: APIRequestContext, agentId: string): Promise { + const res = await board.post(`${BASE_URL}/api/agents/${agentId}/heartbeat/invoke`); + expect(res.ok()).toBe(true); + const run = await res.json(); + return run.id; +} + +/** PATCH an issue as an agent with a fresh heartbeat run ID. */ +async function agentPatch( + board: APIRequestContext, + agent: AgentAuth, + issueId: string, + data: Record, +) { + const runId = await invokeHeartbeat(board, agent.agentId); + const res = await agent.request.patch(`${BASE_URL}/api/issues/${issueId}`, { + headers: { "X-Paperclip-Run-Id": runId }, + data, + }); + return res; +} + +/** Checkout an issue as an agent, then PATCH it. Used for executor mark-done. */ +async function agentCheckoutAndPatch( + board: APIRequestContext, + agent: AgentAuth, + issueId: string, + expectedStatuses: string[], + patchData: Record, +) { + const runId = await invokeHeartbeat(board, agent.agentId); + // Checkout (sets executionRunId so PATCH is allowed) + const checkoutRes = await agent.request.post(`${BASE_URL}/api/issues/${issueId}/checkout`, { + headers: { "X-Paperclip-Run-Id": runId }, + data: { agentId: agent.agentId, expectedStatuses }, + }); + if (!checkoutRes.ok()) { + // If agent checkout fails (e.g. run expired), fall back to board checkout + // then PATCH with the agent's identity + const boardCheckout = await board.post(`${BASE_URL}/api/issues/${issueId}/checkout`, { + data: { agentId: agent.agentId, expectedStatuses }, + }); + if (!boardCheckout.ok()) { + throw new Error(`Board checkout failed: ${await boardCheckout.text()}`); + } + // Board PATCH (executor mark-done triggers signoff regardless of actor) + const res = await board.patch(`${BASE_URL}/api/issues/${issueId}`, { + data: patchData, + }); + return res; + } + // PATCH with agent identity + const res = await agent.request.patch(`${BASE_URL}/api/issues/${issueId}`, { + headers: { "X-Paperclip-Run-Id": runId }, + data: patchData, + }); + return res; +} + +async function setupCompany(boardRequest: APIRequestContext): Promise { + // Verify server is in local_trusted mode + const healthRes = await boardRequest.get(`${BASE_URL}/api/health`); + expect(healthRes.ok()).toBe(true); + const health = await healthRes.json(); + if (health.deploymentMode !== "local_trusted") { + throw new Error( + `Signoff e2e tests require local_trusted deployment mode, ` + + `but server is in "${health.deploymentMode}" mode. ` + + `Set PAPERCLIP_DEPLOYMENT_MODE=local_trusted or use the webServer config.`, + ); + } + // Create company - const companyRes = await request.post(`${baseUrl}/api/companies`, { + const companyRes = await boardRequest.post(`${BASE_URL}/api/companies`, { data: { name: COMPANY_NAME }, }); - expect(companyRes.ok()).toBe(true); + if (!companyRes.ok()) { + const errBody = await companyRes.text(); + throw new Error(`POST /api/companies → ${companyRes.status()}: ${errBody}`); + } const company = await companyRes.json(); const companyId = company.id; + const companyPrefix = company.issuePrefix ?? company.prefix ?? company.urlKey ?? "E2E"; - // Fetch company prefix from the company object - const companyPrefix = company.prefix ?? company.urlKey ?? "E2E"; + // Helper: create agent + API key + request context + async function createAgent(name: string, role: string, title: string): Promise { + const agentRes = await boardRequest.post(`${BASE_URL}/api/companies/${companyId}/agents`, { + data: { name, role, title, adapterType: "process", adapterConfig: { command: "echo done" } }, + }); + expect(agentRes.ok()).toBe(true); + const agent = await agentRes.json(); - // Create executor agent (engineer) - const executorRes = await request.post(`${baseUrl}/api/companies/${companyId}/agents`, { - data: { - name: "Executor", - role: "engineer", - title: "Software Engineer", - adapterType: "process", - adapterConfig: { command: "echo done" }, - }, - }); - expect(executorRes.ok()).toBe(true); - const executor = await executorRes.json(); + const keyRes = await boardRequest.post(`${BASE_URL}/api/agents/${agent.id}/keys`, { + data: { name: `e2e-${name.toLowerCase()}` }, + }); + expect(keyRes.ok()).toBe(true); + const keyData = await keyRes.json(); - // Create reviewer agent (QA) - const reviewerRes = await request.post(`${baseUrl}/api/companies/${companyId}/agents`, { - data: { - name: "Reviewer", - role: "qa", - title: "QA Engineer", - adapterType: "process", - adapterConfig: { command: "echo done" }, - }, - }); - expect(reviewerRes.ok()).toBe(true); - const reviewer = await reviewerRes.json(); + return { + agentId: agent.id, + token: keyData.token, + keyId: keyData.id, + request: await createAgentRequest(keyData.token), + }; + } - // Create approver agent (CTO) - const approverRes = await request.post(`${baseUrl}/api/companies/${companyId}/agents`, { - data: { - name: "Approver", - role: "cto", - title: "CTO", - adapterType: "process", - adapterConfig: { command: "echo done" }, - }, - }); - expect(approverRes.ok()).toBe(true); - const approver = await approverRes.json(); + const executor = await createAgent("Executor", "engineer", "Software Engineer"); + const reviewer = await createAgent("Reviewer", "qa", "QA Engineer"); + const approver = await createAgent("Approver", "cto", "CTO"); return { - baseUrl, companyId, companyPrefix, - executorAgentId: executor.id, - reviewerAgentId: reviewer.id, - approverAgentId: approver.id, + executor, + reviewer, + approver, + boardRequest, + issueIds: [], }; } -async function createIssueWithPolicy( - request: APIRequestContext, - ctx: TestContext, - title: string, -) { - const res = await request.post(`${ctx.baseUrl}/api/companies/${ctx.companyId}/issues`, { +async function createIssueWithPolicy(ctx: TestContext, title: string, stages?: unknown[]) { + const defaultStages = [ + { type: "review", participants: [{ type: "agent", agentId: ctx.reviewer.agentId }] }, + { type: "approval", participants: [{ type: "agent", agentId: ctx.approver.agentId }] }, + ]; + const res = await ctx.boardRequest.post(`${BASE_URL}/api/companies/${ctx.companyId}/issues`, { data: { title, status: "in_progress", - assigneeAgentId: ctx.executorAgentId, - executionPolicy: { - stages: [ - { - type: "review", - participants: [{ type: "agent", agentId: ctx.reviewerAgentId }], - }, - { - type: "approval", - participants: [{ type: "agent", agentId: ctx.approverAgentId }], - }, - ], - }, + assigneeAgentId: ctx.executor.agentId, + executionPolicy: { stages: stages ?? defaultStages }, }, }); expect(res.ok()).toBe(true); - return res.json(); + const issue = await res.json(); + ctx.issueIds.push(issue.id); + return issue; } test.describe("Signoff execution policy", () => { let ctx: TestContext; - test.beforeAll(async ({ request }) => { - const baseUrl = (test.info().project.use as { baseURL?: string }).baseURL ?? "http://127.0.0.1:3100"; - ctx = await setupCompany(request, baseUrl); + test.beforeAll(async () => { + const boardRequest = await pwRequest.newContext({ baseURL: BASE_URL }); + ctx = await setupCompany(boardRequest); }); - test("happy path: executor → review → approval → done", async ({ request, page }) => { - const issue = await createIssueWithPolicy(request, ctx, "Signoff happy path"); + test.afterAll(async () => { + if (!ctx) return; + const board = ctx.boardRequest; + + // Dispose agent request contexts + for (const agent of [ctx.executor, ctx.reviewer, ctx.approver]) { + await agent.request.dispose(); + } + + // Clean up issues, keys, agents, company (best-effort) + for (const issueId of ctx.issueIds) { + await board.patch(`${BASE_URL}/api/issues/${issueId}`, { + data: { status: "cancelled", comment: "E2E test cleanup." }, + }).catch(() => {}); + } + for (const agent of [ctx.executor, ctx.reviewer, ctx.approver]) { + await board.delete(`${BASE_URL}/api/agents/${agent.agentId}/keys/${agent.keyId}`).catch(() => {}); + await board.delete(`${BASE_URL}/api/agents/${agent.agentId}`).catch(() => {}); + } + await board.delete(`${BASE_URL}/api/companies/${ctx.companyId}`).catch(() => {}); + await board.dispose(); + }); + + test("happy path: executor → review → approval → done", async ({ page }) => { + const issue = await createIssueWithPolicy(ctx, "Signoff happy path"); const issueId = issue.id; // Verify policy was saved @@ -133,23 +233,21 @@ test.describe("Signoff execution policy", () => { expect(issue.executionPolicy.stages[1].type).toBe("approval"); // Step 1: Executor marks done → should route to reviewer - const step1Res = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { - status: "done", - comment: "Implemented the feature, ready for review.", - }, - }); + const step1Res = await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issueId, ["in_progress"], + { status: "done", comment: "Implemented the feature, ready for review." }, + ); expect(step1Res.ok()).toBe(true); const step1Issue = await step1Res.json(); expect(step1Issue.status).toBe("in_review"); - expect(step1Issue.assigneeAgentId).toBe(ctx.reviewerAgentId); + expect(step1Issue.assigneeAgentId).toBe(ctx.reviewer.agentId); expect(step1Issue.executionState).toBeTruthy(); expect(step1Issue.executionState.status).toBe("pending"); expect(step1Issue.executionState.currentStageType).toBe("review"); expect(step1Issue.executionState.returnAssignee).toMatchObject({ type: "agent", - agentId: ctx.executorAgentId, + agentId: ctx.executor.agentId, }); // Step 2: Navigate to issue in UI and verify execution label @@ -157,17 +255,15 @@ test.describe("Signoff execution policy", () => { await expect(page.locator("text=Review pending")).toBeVisible({ timeout: 10_000 }); // Step 3: Reviewer approves → should route to approver - const step3Res = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { - status: "done", - comment: "QA signoff complete. Looks good.", - }, - }); + const step3Res = await agentPatch( + ctx.boardRequest, ctx.reviewer, issueId, + { status: "done", comment: "QA signoff complete. Looks good." }, + ); expect(step3Res.ok()).toBe(true); const step3Issue = await step3Res.json(); expect(step3Issue.status).toBe("in_review"); - expect(step3Issue.assigneeAgentId).toBe(ctx.approverAgentId); + expect(step3Issue.assigneeAgentId).toBe(ctx.approver.agentId); expect(step3Issue.executionState.status).toBe("pending"); expect(step3Issue.executionState.currentStageType).toBe("approval"); expect(step3Issue.executionState.completedStageIds).toHaveLength(1); @@ -177,12 +273,10 @@ test.describe("Signoff execution policy", () => { await expect(page.locator("text=Approval pending")).toBeVisible({ timeout: 10_000 }); // Step 5: Approver approves → should complete - const step5Res = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { - status: "done", - comment: "Approved. Ship it.", - }, - }); + const step5Res = await agentPatch( + ctx.boardRequest, ctx.approver, issueId, + { status: "done", comment: "Approved. Ship it." }, + ); expect(step5Res.ok()).toBe(true); const step5Issue = await step5Res.json(); @@ -192,115 +286,110 @@ test.describe("Signoff execution policy", () => { expect(step5Issue.executionState.lastDecisionOutcome).toBe("approved"); }); - test("changes requested: reviewer bounces back to executor", async ({ request }) => { - const issue = await createIssueWithPolicy(request, ctx, "Signoff changes requested"); + test("changes requested: reviewer bounces back to executor", async () => { + const issue = await createIssueWithPolicy(ctx, "Signoff changes requested"); const issueId = issue.id; // Executor marks done → routes to reviewer - const doneRes = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { status: "done", comment: "Ready for review." }, - }); + const doneRes = await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issueId, ["in_progress"], + { status: "done", comment: "Ready for review." }, + ); expect(doneRes.ok()).toBe(true); - const reviewIssue = await doneRes.json(); - expect(reviewIssue.status).toBe("in_review"); - expect(reviewIssue.assigneeAgentId).toBe(ctx.reviewerAgentId); + expect((await doneRes.json()).status).toBe("in_review"); // Reviewer requests changes → returns to executor - const changesRes = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { - status: "in_progress", - comment: "Needs another pass on edge cases.", - }, - }); + const changesRes = await agentPatch( + ctx.boardRequest, ctx.reviewer, issueId, + { status: "in_progress", comment: "Needs another pass on edge cases." }, + ); expect(changesRes.ok()).toBe(true); const changesIssue = await changesRes.json(); expect(changesIssue.status).toBe("in_progress"); - expect(changesIssue.assigneeAgentId).toBe(ctx.executorAgentId); + expect(changesIssue.assigneeAgentId).toBe(ctx.executor.agentId); expect(changesIssue.executionState.status).toBe("changes_requested"); expect(changesIssue.executionState.lastDecisionOutcome).toBe("changes_requested"); // Executor re-submits → goes back to reviewer (same stage) - const resubmitRes = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { status: "done", comment: "Fixed the edge cases." }, - }); + const resubmitRes = await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issueId, ["in_progress"], + { status: "done", comment: "Fixed the edge cases." }, + ); expect(resubmitRes.ok()).toBe(true); const resubmitIssue = await resubmitRes.json(); expect(resubmitIssue.status).toBe("in_review"); - expect(resubmitIssue.assigneeAgentId).toBe(ctx.reviewerAgentId); + expect(resubmitIssue.assigneeAgentId).toBe(ctx.reviewer.agentId); expect(resubmitIssue.executionState.status).toBe("pending"); expect(resubmitIssue.executionState.currentStageType).toBe("review"); }); - test("comment required: approval without comment fails", async ({ request }) => { - const issue = await createIssueWithPolicy(request, ctx, "Signoff comment required"); + test("comment required: approval without comment fails", async () => { + const issue = await createIssueWithPolicy(ctx, "Signoff comment required"); const issueId = issue.id; - // Executor marks done - await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { status: "done", comment: "Done." }, - }); + // Executor marks done → routes to reviewer + await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issueId, ["in_progress"], + { status: "done", comment: "Done." }, + ); // Reviewer tries to approve without comment → should fail - const noCommentRes = await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { status: "done" }, - }); - // Server should reject: 422 or similar + const noCommentRes = await agentPatch( + ctx.boardRequest, ctx.reviewer, issueId, + { status: "done" }, + ); expect(noCommentRes.ok()).toBe(false); const errorBody = await noCommentRes.json(); expect(JSON.stringify(errorBody)).toContain("comment"); }); - test("non-participant cannot advance stage", async ({ request }) => { - const issue = await createIssueWithPolicy(request, ctx, "Signoff access control"); + test("non-participant cannot advance stage", async () => { + const issue = await createIssueWithPolicy(ctx, "Signoff access control"); const issueId = issue.id; // Executor marks done → routes to reviewer - await request.patch(`${ctx.baseUrl}/api/issues/${issueId}`, { - data: { status: "done", comment: "Done." }, - }); + const doneRes = await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issueId, ["in_progress"], + { status: "done", comment: "Done." }, + ); + expect(doneRes.ok()).toBe(true); // Verify issue is in_review with reviewer - const issueRes = await request.get(`${ctx.baseUrl}/api/issues/${issueId}`); + const issueRes = await ctx.boardRequest.get(`${BASE_URL}/api/issues/${issueId}`); const inReviewIssue = await issueRes.json(); expect(inReviewIssue.status).toBe("in_review"); - expect(inReviewIssue.assigneeAgentId).toBe(ctx.reviewerAgentId); + expect(inReviewIssue.assigneeAgentId).toBe(ctx.reviewer.agentId); expect(inReviewIssue.executionState.currentStageType).toBe("review"); + + // Non-participant (approver at this stage) tries to advance → should be rejected + const advanceRes = await agentPatch( + ctx.boardRequest, ctx.approver, issueId, + { status: "done", comment: "I'm the approver, not the reviewer." }, + ); + expect(advanceRes.ok()).toBe(false); + expect(advanceRes.status()).toBeGreaterThanOrEqual(400); }); - test("review-only policy: reviewer approval completes execution", async ({ request }) => { - // Create issue with review-only policy (no approval stage) - const res = await request.post(`${ctx.baseUrl}/api/companies/${ctx.companyId}/issues`, { - data: { - title: "Signoff review-only", - status: "in_progress", - assigneeAgentId: ctx.executorAgentId, - executionPolicy: { - stages: [ - { - type: "review", - participants: [{ type: "agent", agentId: ctx.reviewerAgentId }], - }, - ], - }, - }, - }); - expect(res.ok()).toBe(true); - const issue = await res.json(); + test("review-only policy: reviewer approval completes execution", async () => { + const issue = await createIssueWithPolicy(ctx, "Signoff review-only", [ + { type: "review", participants: [{ type: "agent", agentId: ctx.reviewer.agentId }] }, + ]); // Executor marks done → routes to reviewer - const doneRes = await request.patch(`${ctx.baseUrl}/api/issues/${issue.id}`, { - data: { status: "done", comment: "Ready for review." }, - }); + const doneRes = await agentCheckoutAndPatch( + ctx.boardRequest, ctx.executor, issue.id, ["in_progress"], + { status: "done", comment: "Ready for review." }, + ); expect(doneRes.ok()).toBe(true); - const reviewIssue = await doneRes.json(); - expect(reviewIssue.status).toBe("in_review"); + expect((await doneRes.json()).status).toBe("in_review"); // Reviewer approves → should complete immediately (no approval stage) - const approveRes = await request.patch(`${ctx.baseUrl}/api/issues/${issue.id}`, { - data: { status: "done", comment: "LGTM." }, - }); + const approveRes = await agentPatch( + ctx.boardRequest, ctx.reviewer, issue.id, + { status: "done", comment: "LGTM." }, + ); expect(approveRes.ok()).toBe(true); const doneIssue = await approveRes.json(); expect(doneIssue.status).toBe("done");