import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { activityLog, agentRuntimeState, agentWakeupRequests, agents, companies, companySkills, createDb, documentRevisions, documents, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issueDocuments, issues, workspaceRuntimeServices, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.ts"; import { normalizeIssueExecutionPolicy, parseIssueExecutionState } from "../services/issue-execution-policy.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres issue monitor scheduler tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } describeEmbeddedPostgres("issue monitor scheduler", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; const seededAgentIds = new Set(); beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-monitor-"); db = createDb(tempDb.connectionString); }, 20_000); async function waitForHeartbeatIdle(timeoutMs = 3_000) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const active = await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(sql`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`); if (active.length === 0) return; await new Promise((resolve) => setTimeout(resolve, 50)); } throw new Error("Timed out waiting for issue monitor heartbeat runs to settle"); } async function heartbeatSideEffectFingerprint() { const [active, events, activity, leases, runtimeServices] = await Promise.all([ db .select({ count: sql`count(*)` }) .from(heartbeatRuns) .where(sql`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`), db.select({ count: sql`count(*)` }).from(heartbeatRunEvents), db.select({ count: sql`count(*)` }).from(activityLog), db.select({ count: sql`count(*)` }).from(environmentLeases), db.select({ count: sql`count(*)` }).from(workspaceRuntimeServices), ]); return [ active[0]?.count ?? 0, events[0]?.count ?? 0, activity[0]?.count ?? 0, leases[0]?.count ?? 0, runtimeServices[0]?.count ?? 0, ].join(":"); } async function waitForHeartbeatSideEffectsSettled(timeoutMs = 5_000, quietMs = 500) { const deadline = Date.now() + timeoutMs; let previous = ""; let stableSince = Date.now(); while (Date.now() < deadline) { const current = await heartbeatSideEffectFingerprint(); const activeCount = Number(current.split(":")[0] ?? 0); if (current !== previous || activeCount > 0) { previous = current; stableSince = Date.now(); } else if (Date.now() - stableSince >= quietMs) { return; } await new Promise((resolve) => setTimeout(resolve, 50)); } throw new Error("Timed out waiting for issue monitor heartbeat side effects to settle"); } async function cleanupRows() { await waitForHeartbeatSideEffectsSettled(); await db.delete(heartbeatRunEvents); await db.delete(issueComments); await db.delete(documentRevisions); await db.delete(issueDocuments); await db.delete(documents); await db.delete(activityLog); await db.delete(environmentLeases); await db.delete(workspaceRuntimeServices); await db.delete(issues); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); await db.delete(agents); await db.delete(companySkills); await db.delete(companies); } afterEach(async () => { seededAgentIds.clear(); let lastError: unknown = null; for (let attempt = 0; attempt < 3; attempt += 1) { try { await cleanupRows(); return; } catch (error) { lastError = error; await new Promise((resolve) => setTimeout(resolve, 100)); } } throw lastError; }); afterAll(async () => { await tempDb?.cleanup(); }); async function seedFixture(input?: { agentStatus?: "active" | "paused"; issueStatus?: "in_progress" | "in_review"; monitorAttemptCount?: number; monitor?: Record; }) { const companyId = randomUUID(); const agentId = randomUUID(); const issueId = randomUUID(); const nextCheckAt = new Date("2026-04-11T12:30:00.000Z"); const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; const monitorAttemptCount = input?.monitorAttemptCount ?? 0; const monitor = { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deploy", scheduledBy: "assignee", ...(input?.monitor ?? {}), }; await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix, requireBoardApprovalForNewAgents: false, }); await db.insert(agents).values({ id: agentId, companyId, name: "Monitor Bot", role: "engineer", status: input?.agentStatus ?? "active", adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", ""], cwd: process.cwd(), }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true, }, }, permissions: {}, }); seededAgentIds.add(agentId); await db.insert(issues).values({ id: issueId, companyId, title: "Watch external deploy", status: input?.issueStatus ?? "in_progress", priority: "medium", assigneeAgentId: agentId, issueNumber: 1, identifier: `${issuePrefix}-1`, executionPolicy: { mode: "normal", commentRequired: true, stages: [], monitor, }, executionState: { status: "idle", currentStageId: null, currentStageIndex: null, currentStageType: null, currentParticipant: null, returnAssignee: null, completedStageIds: [], lastDecisionId: null, lastDecisionOutcome: null, monitor: { status: "scheduled", nextCheckAt: nextCheckAt.toISOString(), lastTriggeredAt: null, attemptCount: monitorAttemptCount, notes: "Check deploy", scheduledBy: "assignee", serviceName: typeof monitor.serviceName === "string" ? monitor.serviceName : null, externalRef: typeof monitor.externalRef === "string" ? monitor.externalRef : null, timeoutAt: typeof monitor.timeoutAt === "string" ? monitor.timeoutAt : null, maxAttempts: typeof monitor.maxAttempts === "number" ? monitor.maxAttempts : null, recoveryPolicy: typeof monitor.recoveryPolicy === "string" ? monitor.recoveryPolicy : null, clearedAt: null, clearReason: null, }, }, monitorNextCheckAt: nextCheckAt, monitorAttemptCount, monitorNotes: "Check deploy", monitorScheduledBy: "assignee", }); return { companyId, agentId, issueId, nextCheckAt }; } it("triggers due issue monitors once and clears the one-shot schedule", async () => { const { issueId, agentId } = await seedFixture(); const heartbeat = heartbeatService(db); const tickAt = new Date("2026-04-11T12:31:00.000Z"); const result = await heartbeat.tickTimers(tickAt); expect(result.enqueued).toBe(1); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); expect(issue.monitorNextCheckAt).toBeNull(); expect(issue.monitorAttemptCount).toBe(1); expect(issue.monitorLastTriggeredAt?.toISOString()).toBe(tickAt.toISOString()); expect(normalizeIssueExecutionPolicy(issue.executionPolicy ?? null)?.monitor ?? null).toBeNull(); expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({ status: "triggered", lastTriggeredAt: tickAt.toISOString(), attemptCount: 1, }); const wakeup = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)) .then((rows) => rows[0] ?? null); expect(wakeup?.reason).toBe("issue_monitor_due"); const activity = await db .select() .from(activityLog) .where(eq(activityLog.entityId, issueId)) .then((rows) => rows.map((row) => row.action)); expect(activity).toContain("issue.monitor_triggered"); }); it("lets the board trigger a scheduled issue monitor immediately", async () => { const { issueId, agentId, nextCheckAt } = await seedFixture(); const heartbeat = heartbeatService(db); const triggeredAt = new Date("2026-04-11T12:00:00.000Z"); const result = await heartbeat.triggerIssueMonitor(issueId, { now: triggeredAt, actorType: "user", actorId: "local-board", }); expect(result.outcome).toBe("triggered"); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); expect(issue.monitorNextCheckAt).toBeNull(); expect(issue.monitorLastTriggeredAt?.toISOString()).toBe(triggeredAt.toISOString()); expect(issue.monitorAttemptCount).toBe(1); expect(normalizeIssueExecutionPolicy(issue.executionPolicy ?? null)?.monitor ?? null).toBeNull(); const wakeup = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)) .then((rows) => rows[0] ?? null); expect(wakeup?.reason).toBe("issue_monitor_due"); expect(wakeup?.payload).toMatchObject({ issueId, nextCheckAt: nextCheckAt.toISOString(), source: "manual", }); const activity = await db .select() .from(activityLog) .where(eq(activityLog.entityId, issueId)) .orderBy(activityLog.createdAt); expect(activity.map((row) => row.action)).toContain("issue.monitor_triggered"); const triggerEvent = activity.find((row) => row.action === "issue.monitor_triggered"); expect(triggerEvent?.actorType).toBe("user"); expect(triggerEvent?.actorId).toBe("local-board"); expect(triggerEvent?.details).toMatchObject({ nextCheckAt: nextCheckAt.toISOString(), source: "manual", }); }); it("clears due monitors that cannot be dispatched and records a skip", async () => { const { issueId } = await seedFixture({ agentStatus: "paused" }); const heartbeat = heartbeatService(db); const tickAt = new Date("2026-04-11T12:31:00.000Z"); const result = await heartbeat.tickTimers(tickAt); expect(result.skipped).toBe(1); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); expect(issue.monitorNextCheckAt).toBeNull(); expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({ status: "cleared", clearReason: "dispatch_skipped", }); const activity = await db .select() .from(activityLog) .where(eq(activityLog.entityId, issueId)) .then((rows) => rows.map((row) => row.action)); expect(activity).toContain("issue.monitor_skipped"); }); it("clears exhausted monitors and queues bounded owner recovery instead of another due check", async () => { const { issueId, agentId } = await seedFixture({ monitorAttemptCount: 1, monitor: { maxAttempts: 1, recoveryPolicy: "wake_owner", }, }); const heartbeat = heartbeatService(db); const tickAt = new Date("2026-04-11T12:31:00.000Z"); const result = await heartbeat.tickTimers(tickAt); expect(result.enqueued).toBe(0); expect(result.skipped).toBe(1); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); expect(issue.monitorNextCheckAt).toBeNull(); expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({ status: "cleared", clearReason: "max_attempts_exhausted", }); const wakeup = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)) .then((rows) => rows[0] ?? null); expect(wakeup?.reason).toBe("issue_monitor_recovery"); expect(wakeup?.payload).toMatchObject({ issueId, clearReason: "max_attempts_exhausted", maxAttempts: 1, modelProfile: "cheap", }); const activity = await db .select() .from(activityLog) .where(eq(activityLog.entityId, issueId)) .then((rows) => rows.map((row) => row.action)); expect(activity).toContain("issue.monitor_exhausted"); expect(activity).toContain("issue.monitor_recovery_wake_queued"); expect(activity).not.toContain("issue.monitor_triggered"); }); it("clears timed-out monitors and creates a visible recovery issue when requested", async () => { const { issueId, companyId } = await seedFixture({ monitor: { timeoutAt: "2026-04-11T12:00:00.000Z", recoveryPolicy: "create_recovery_issue", }, }); const heartbeat = heartbeatService(db); const tickAt = new Date("2026-04-11T12:31:00.000Z"); const result = await heartbeat.tickTimers(tickAt); expect(result.enqueued).toBe(0); expect(result.skipped).toBe(1); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); expect(issue.monitorNextCheckAt).toBeNull(); expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({ status: "cleared", clearReason: "timeout_exceeded", }); const recoveryIssue = await db .select() .from(issues) .where(eq(issues.originId, issueId)) .then((rows) => rows.find((row) => row.companyId === companyId && row.originKind === "stranded_issue_recovery") ?? null); expect(recoveryIssue).toMatchObject({ parentId: issueId, priority: "high", assigneeAdapterOverrides: { modelProfile: "cheap" }, }); expect(["todo", "in_progress"]).toContain(recoveryIssue?.status); }); it("omits external monitor refs from wake payloads and activity details", async () => { const { issueId, agentId } = await seedFixture({ monitor: { serviceName: "Deploy provider", externalRef: "https://provider.example/deploy/123?token=secret", }, }); const heartbeat = heartbeatService(db); const tickAt = new Date("2026-04-11T12:31:00.000Z"); await heartbeat.tickTimers(tickAt); const wakeup = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)) .then((rows) => rows[0] ?? null); expect(JSON.stringify(wakeup?.payload)).not.toContain("provider.example"); expect(wakeup?.payload).not.toHaveProperty("externalRef"); const activity = await db .select() .from(activityLog) .where(eq(activityLog.entityId, issueId)); expect(JSON.stringify(activity.map((row) => row.details))).not.toContain("provider.example"); expect(activity.find((row) => row.action === "issue.monitor_triggered")?.details).not.toHaveProperty("externalRef"); }); });