diff --git a/package-lock.json b/package-lock.json index 4a6bbf6..0d37196 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "paperclip-adapter-opencode-k8s", - "version": "0.1.39", + "version": "0.1.40", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "paperclip-adapter-opencode-k8s", - "version": "0.1.39", + "version": "0.1.40", "license": "MIT", "dependencies": { "@kubernetes/client-node": "^1.0.0", diff --git a/src/server/execute.test.ts b/src/server/execute.test.ts index 40f4fab..74ea9bf 100644 --- a/src/server/execute.test.ts +++ b/src/server/execute.test.ts @@ -4,33 +4,47 @@ import { execute, ensureAgentDbPvc, tailPodLogFile } from "./execute.js"; import { getSelfPodInfo, getBatchApi, getCoreApi, getPvc, createPvc } from "./k8s-client.js"; import { buildJobManifest, buildPodLogPath } from "./job-manifest.js"; -// Mock node:fs/promises to prevent tailPodLogFile (used by execute()) from -// hanging on unmocked fs.stat calls in test environment. -// vi.hoisted creates shared module-level state; beforeEach resets it so every -// test gets a clean first-read-success. -const { readMock, resetFsMocks } = vi.hoisted(() => { +// Mock node:fs/promises so tailPodLogFile (used by execute()) reads a +// configurable JSONL payload and returns. Individual tests override the +// payload via setMockJsonl(...) before calling execute(). +const { readMock, statMock, fhStatMock, resetFsMocks, setMockJsonl } = vi.hoisted(() => { + const HAPPY = [ + JSON.stringify({ type: "text", part: { text: "Task complete" }, sessionID: "ses_happy" }), + JSON.stringify({ type: "step_finish", part: { tokens: { input: 100, output: 50, cache: { read: 20 } }, cost: 0.002 } }), + ].join("\n"); + let payload = HAPPY; + let buffer = Buffer.from(payload); let readOffset = 0; + const apply = (next: string) => { payload = next; buffer = Buffer.from(payload); readOffset = 0; }; return { - readMock: vi.fn().mockImplementation(async () => { - if (readOffset === 0) { - readOffset = 17; - return { bytesRead: 17, buffer: Buffer.from('{"type":"text"}\n') }; - } - return { bytesRead: 0, buffer: Buffer.alloc(0) }; + readMock: vi.fn().mockImplementation(async (buf: Buffer, off: number, len: number, _pos: number) => { + if (readOffset >= buffer.byteLength) return { bytesRead: 0, buffer: buf }; + const remaining = buffer.byteLength - readOffset; + const toRead = Math.min(len, remaining); + buffer.copy(buf, off, readOffset, readOffset + toRead); + readOffset += toRead; + return { bytesRead: toRead, buffer: buf }; }), - resetFsMocks: () => { readOffset = 0; }, + statMock: vi.fn().mockImplementation(async () => ({ size: buffer.byteLength })), + fhStatMock: vi.fn().mockImplementation(async () => ({ size: buffer.byteLength })), + resetFsMocks: () => { apply(HAPPY); }, + setMockJsonl: (jsonl: string) => { apply(jsonl); }, }; }); -vi.mock("node:fs/promises", () => ({ - stat: vi.fn().mockResolvedValue({ size: 17 }), - open: vi.fn().mockResolvedValue({ - stat: vi.fn().mockResolvedValue({ size: 17 }), - read: readMock, - close: vi.fn().mockResolvedValue(undefined), - }), - unlink: vi.fn().mockResolvedValue(undefined), -})); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + open: vi.fn().mockResolvedValue({ + stat: fhStatMock, + read: readMock, + close: vi.fn().mockResolvedValue(undefined), + }), + unlink: vi.fn().mockResolvedValue(undefined), + }; +}); vi.mock("./k8s-client.js", () => ({ getSelfPodInfo: vi.fn(), @@ -43,7 +57,7 @@ vi.mock("./k8s-client.js", () => ({ vi.mock("./job-manifest.js", () => ({ buildJobManifest: vi.fn(), buildPodLogPath: vi.fn((companyId: string, agentId: string, runId: string) => - `/paperclip/instances/default/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson` + `/paperclip/instances/default/data/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson` ), LARGE_PROMPT_THRESHOLD_BYTES: 256 * 1024, })); @@ -169,7 +183,7 @@ beforeEach(() => { prompt: "Test prompt", opencodeArgs: [], promptMetrics: null, - podLogPath: `/paperclip/instances/default/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, + podLogPath: `/paperclip/instances/default/data/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, } as unknown as ReturnType); const batchApi = makeBatchApi(); @@ -590,6 +604,7 @@ describe("execute — happy path", () => { describe("execute — session unavailable (reattach classification)", () => { it("returns clearSession=true and session_unavailable code for unknown session error", async () => { + setMockJsonl(JSON.stringify({ type: "error", error: { message: "Unknown session ses_xxx" } })); const coreApi = makeCoreApi(1); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -601,6 +616,7 @@ describe("execute — session unavailable (reattach classification)", () => { }); it("returns clearSession=true for 'session not found' error", async () => { + setMockJsonl(JSON.stringify({ type: "error", error: { message: "Session ses_xxx not found" } })); const coreApi = makeCoreApi(1); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -672,6 +688,7 @@ describe("execute — exit code handling", () => { }); it("synthesizes exitCode=1 when error message exists but pod reported exitCode=0", async () => { + setMockJsonl(JSON.stringify({ type: "error", error: { message: "something went wrong" } })); const coreApi = makeCoreApi(0); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -735,6 +752,7 @@ describe("execute — llm_api_error signal", () => { it("returns llm_api_error when session exists but LLM produced no output tokens", async () => { // JSONL has a sessionID but no step_finish tokens and no text messages const emptyOutputJsonl = JSON.stringify({ sessionID: "ses_empty", type: "step_finish", part: { tokens: { input: 100, output: 0, cache: {} }, cost: 0 } }); + setMockJsonl(emptyOutputJsonl); const coreApi = makeCoreApi(0); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -757,6 +775,7 @@ describe("execute — llm_api_error signal", () => { const errorJsonl = [ JSON.stringify({ sessionID: "ses_err", type: "error", error: { message: "API quota exceeded" } }), ].join("\n"); + setMockJsonl(errorJsonl); const coreApi = makeCoreApi(1); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -939,7 +958,7 @@ describe("execute — large-prompt Secret path", () => { prompt: LARGE_PROMPT, opencodeArgs: [], promptMetrics: null, - podLogPath: `/paperclip/instances/default/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, + podLogPath: `/paperclip/instances/default/data/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, } as unknown as ReturnType); } @@ -1271,7 +1290,7 @@ describe("execute — large-prompt Secret create failure", () => { prompt: LARGE_PROMPT, opencodeArgs: [], promptMetrics: null, - podLogPath: `/paperclip/instances/default/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, + podLogPath: `/paperclip/instances/default/data/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, } as unknown as ReturnType); const coreApi = makeCoreApi(); @@ -1305,6 +1324,7 @@ describe("execute — step limit detection", () => { JSON.stringify({ type: "text", part: { text: "partial" }, sessionID: "ses_step" }), JSON.stringify({ type: "step_finish", part: { reason: "max_steps", tokens: { input: 10, output: 5 }, cost: 0 } }), ].join("\n"); + setMockJsonl(STEP_LIMIT_JSONL); const coreApi = makeCoreApi(0); vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); @@ -1507,10 +1527,10 @@ describe("execute — SIGTERM handler body (FAR-86 coverage)", () => { prompt: "p", opencodeArgs: [], promptMetrics: null, - podLogPath: `/paperclip/instances/default/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, + podLogPath: `/paperclip/instances/default/data/run-logs/co-1/agent-id-test/run-test-123.pod.ndjson`, }), buildPodLogPath: vi.fn((companyId: string, agentId: string, runId: string) => - `/paperclip/instances/default/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson` + `/paperclip/instances/default/data/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson` ), LARGE_PROMPT_THRESHOLD_BYTES: 256 * 1024, })); diff --git a/src/server/execute.ts b/src/server/execute.ts index f6f4a5c..2006624 100644 --- a/src/server/execute.ts +++ b/src/server/execute.ts @@ -269,40 +269,45 @@ export async function tailPodLogFile( let idleCount = 0; const accumulator: string[] = []; + const drain = async (): Promise => { + let size: number; + try { + const stat = await fh.stat(); + size = stat.size; + } catch { + return false; + } + if (size <= offset) return false; + const buf = Buffer.alloc(size - offset); + const { bytesRead } = await fh.read(buf, 0, buf.length, offset); + offset += bytesRead; + const chunk = buf.slice(0, bytesRead).toString("utf-8"); + const lineParts = (pending + chunk).split("\n"); + pending = lineParts.pop() ?? ""; + for (const line of lineParts) { + await onLog("stdout", line + "\n"); + accumulator.push(line + "\n"); + } + return bytesRead > 0; + }; + try { while (!stopSignal.stopped) { - const pollMs = idleCount >= IDLE_THRESHOLD ? POLL_IDLE_MS : POLL_ACTIVE_MS; - await new Promise((r) => setTimeout(r, pollMs)); - if (stopSignal.stopped) break; - - let size: number; - try { - const stat = await fh.stat(); - size = stat.size; - } catch { - break; - } - - if (size > offset) { - const buf = Buffer.alloc(size - offset); - const { bytesRead } = await fh.read(buf, 0, buf.length, offset); - offset += bytesRead; + const grew = await drain(); + if (grew) { idleCount = 0; - - const chunk = buf.slice(0, bytesRead).toString("utf-8"); - const lineParts = (pending + chunk).split("\n"); - pending = lineParts.pop() ?? ""; - - for (const line of lineParts) { - await onLog("stdout", line + "\n"); - accumulator.push(line + "\n"); - } } else { idleCount++; } + if (stopSignal.stopped) break; + const pollMs = idleCount >= IDLE_THRESHOLD ? POLL_IDLE_MS : POLL_ACTIVE_MS; + await new Promise((r) => setTimeout(r, pollMs)); } - // Final drain on stop + // Final drain after stopSignal — pick up any bytes written between the + // last read and the job reaching terminal state. + while (await drain()) { /* read until no more growth */ } + if (pending) { await onLog("stdout", pending + "\n"); accumulator.push(pending + "\n"); @@ -462,13 +467,22 @@ async function streamAndAwaitJob( return onLog(stream, chunk); }; - const tailResult = await tailPodLogFile(podLogPath, { onLog: wrappedOnLog, stopSignal }); - stdout = tailResult; - - // Wait for job completion (may already be done by the time we read the file) - const completionPromise = waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath); + // Run the file tail and the job-completion poll in parallel so that the + // tail loop has a way to stop: when waitForJobCompletion resolves it sets + // stopSignal.stopped, which lets tailPodLogFile drain and return. + const completionPromise = waitForJobCompletion(namespace, jobName, completionTimeoutMs, kubeconfigPath) + .then((r) => { stopSignal.stopped = true; return r; }); const completionGraced = completionWithGrace(completionPromise, LOG_EXIT_COMPLETION_GRACE_MS); - const completion = await completionGraced; + const [tailSettled, completionSettled] = await Promise.allSettled([ + tailPodLogFile(podLogPath, { onLog: wrappedOnLog, stopSignal }), + completionGraced, + ]); + stdout = tailSettled.status === "fulfilled" ? tailSettled.value : ""; + if (completionSettled.status === "rejected") { + stopSignal.stopped = true; + throw completionSettled.reason; + } + const completion = completionSettled.value; if (keepaliveTimer) { clearInterval(keepaliveTimer); diff --git a/src/server/job-manifest.test.ts b/src/server/job-manifest.test.ts index 8c94738..367f1a1 100644 --- a/src/server/job-manifest.test.ts +++ b/src/server/job-manifest.test.ts @@ -359,8 +359,9 @@ describe("init container is unchanged by agentDbClaimName", () => { it("does not add extra env vars to init container for dedicated PVC mode", () => { const result = buildJobManifest({ ctx: mockCtx, selfPod: mockSelfPod, agentDbClaimName: "opencode-db-agent-abc" }); const initCmd = result.job.spec?.template?.spec?.initContainers?.[0].command; - // mkdir is added for log directory but OPENCODE_DB_PATH env var is NOT added - expect(initCmd?.[2]).toContain("mkdir"); + // init container only writes the prompt; no mkdir (log dir exists on PVC) and no OPENCODE_DB_PATH env var + expect(initCmd?.[2]).not.toContain("mkdir"); + expect(initCmd?.[2]).toContain("/tmp/prompt/prompt.txt"); const initEnv = result.job.spec?.template?.spec?.initContainers?.[0].env ?? []; expect(initEnv.some((e) => e.name === "OPENCODE_DB_PATH")).toBe(false); }); diff --git a/src/server/job-manifest.ts b/src/server/job-manifest.ts index ff87f4e..0d940e0 100644 --- a/src/server/job-manifest.ts +++ b/src/server/job-manifest.ts @@ -25,7 +25,7 @@ function assertSafePathComponent(field: string, value: string): void { } export function buildPodLogPath(companyId: string, agentId: string, runId: string): string { - return `/paperclip/instances/default/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson`; + return `/paperclip/instances/default/data/run-logs/${companyId}/${agentId}/${runId}.pod.ndjson`; } export interface JobBuildInput { @@ -461,14 +461,14 @@ export function buildJobManifest(input: JobBuildInput): JobBuildResult { imagePullPolicy: "IfNotPresent", ...(input.promptSecretName ? { - command: ["sh", "-c", `mkdir -p /paperclip/instances/default/run-logs/${companyId}/${agentId} && cp /tmp/prompt-secret/prompt /tmp/prompt/prompt.txt`], + command: ["sh", "-c", `cp /tmp/prompt-secret/prompt /tmp/prompt/prompt.txt`], volumeMounts: [ { name: "prompt", mountPath: "/tmp/prompt" }, { name: "prompt-secret", mountPath: "/tmp/prompt-secret", readOnly: true }, ], } : { - command: ["sh", "-c", `mkdir -p /paperclip/instances/default/run-logs/${companyId}/${agentId} && printf '%s' \"$PROMPT_CONTENT\" > /tmp/prompt/prompt.txt`], + command: ["sh", "-c", `printf '%s' \"$PROMPT_CONTENT\" > /tmp/prompt/prompt.txt`], env: [{ name: "PROMPT_CONTENT", value: prompt }], volumeMounts: [{ name: "prompt", mountPath: "/tmp/prompt" }], }), diff --git a/src/server/test.test.ts b/src/server/test.test.ts new file mode 100644 index 0000000..8f7d032 --- /dev/null +++ b/src/server/test.test.ts @@ -0,0 +1,195 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { AdapterEnvironmentTestContext } from "@paperclipai/adapter-utils"; +import { testEnvironment } from "./test.js"; +import { getSelfPodInfo, getCoreApi, getAuthzApi } from "./k8s-client.js"; + +vi.mock("./k8s-client.js", () => ({ + getSelfPodInfo: vi.fn(), + getCoreApi: vi.fn(), + getAuthzApi: vi.fn(), +})); + +const SELF_POD = { + namespace: "ns-self", + image: "img:1", + imagePullSecrets: [], + pvcClaimName: "paperclip-pvc", + inheritedEnv: {}, + inheritedEnvValueFrom: [], + inheritedEnvFrom: [], + dnsConfig: undefined, + secretVolumes: [], +} as unknown as Awaited>; + +function makeCtx(config: Record = {}): AdapterEnvironmentTestContext { + return { adapterType: "opencode_k8s", config } as unknown as AdapterEnvironmentTestContext; +} + +function makeAuthz(allowedFor: (resource: string, verb: string) => boolean) { + return { + createSelfSubjectAccessReview: vi.fn().mockImplementation(async ({ body }: { body: { spec: { resourceAttributes: { resource: string; verb: string } } } }) => { + const { resource, verb } = body.spec.resourceAttributes; + return { status: { allowed: allowedFor(resource, verb) } }; + }), + }; +} + +function makeCore(overrides: Partial<{ readNamespace: ReturnType; readNamespacedSecret: ReturnType; readNamespacedPersistentVolumeClaim: ReturnType }> = {}) { + return { + readNamespace: overrides.readNamespace ?? vi.fn().mockResolvedValue({ metadata: { name: "ns" } }), + readNamespacedSecret: overrides.readNamespacedSecret ?? vi.fn().mockResolvedValue({ metadata: { name: "paperclip-secrets" } }), + readNamespacedPersistentVolumeClaim: overrides.readNamespacedPersistentVolumeClaim ?? vi.fn().mockResolvedValue({ spec: { accessModes: ["ReadWriteMany"] } }), + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getSelfPodInfo).mockResolvedValue(SELF_POD); + vi.mocked(getCoreApi).mockReturnValue(makeCore() as unknown as ReturnType); + vi.mocked(getAuthzApi).mockReturnValue(makeAuthz(() => true) as unknown as ReturnType); +}); + +describe("testEnvironment — happy path", () => { + it("returns pass when API, namespace, RBAC, secret, and RWX PVC all check out", async () => { + const result = await testEnvironment(makeCtx()); + + expect(result.adapterType).toBe("opencode_k8s"); + expect(result.status).toBe("pass"); + expect(result.checks.find((c) => c.code === "k8s_api_reachable")).toBeDefined(); + expect(result.checks.find((c) => c.code === "k8s_pvc_rwx")).toBeDefined(); + expect(result.checks.find((c) => c.code === "k8s_secret_exists")).toBeDefined(); + expect(typeof result.testedAt).toBe("string"); + }); + + it("skips namespace lookup and emits k8s_namespace_exists when target == self pod namespace", async () => { + const coreApi = makeCore(); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx()); + + expect(coreApi.readNamespace).not.toHaveBeenCalled(); + expect(result.checks.find((c) => c.code === "k8s_namespace_exists")?.message).toContain("pod namespace"); + }); + + it("calls readNamespace when target namespace differs from self pod namespace", async () => { + const coreApi = makeCore(); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx({ namespace: "ns-other" })); + + expect(coreApi.readNamespace).toHaveBeenCalledWith({ name: "ns-other" }); + expect(result.checks.find((c) => c.code === "k8s_namespace_exists")).toBeDefined(); + }); +}); + +describe("testEnvironment — early-return paths", () => { + it("returns fail and short-circuits when K8s API is unreachable", async () => { + vi.mocked(getSelfPodInfo).mockRejectedValueOnce(new Error("ECONNREFUSED")); + + const result = await testEnvironment(makeCtx()); + + expect(result.status).toBe("fail"); + expect(result.checks.find((c) => c.code === "k8s_api_unreachable")).toBeDefined(); + // RBAC, secret, and PVC checks should be skipped when API is unreachable + expect(result.checks.some((c) => c.code.startsWith("k8s_rbac_"))).toBe(false); + }); +}); + +describe("testEnvironment — namespace warning", () => { + it("emits warn (but proceeds) when readNamespace fails for a different namespace", async () => { + const coreApi = makeCore({ + readNamespace: vi.fn().mockRejectedValue(new Error("forbidden")), + }); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx({ namespace: "ns-other" })); + + expect(result.checks.find((c) => c.code === "k8s_namespace_check_failed")).toBeDefined(); + // Should still proceed with downstream checks + expect(result.checks.some((c) => c.code.startsWith("k8s_rbac_"))).toBe(true); + }); +}); + +describe("testEnvironment — RBAC", () => { + it("emits error checks for denied verbs and degrades status to fail", async () => { + vi.mocked(getAuthzApi).mockReturnValue( + makeAuthz((resource, verb) => !(resource === "jobs" && verb === "create")) as unknown as ReturnType, + ); + + const result = await testEnvironment(makeCtx()); + + const denied = result.checks.find((c) => c.code === "k8s_rbac_job_create"); + expect(denied?.level).toBe("error"); + expect(result.status).toBe("fail"); + }); + + it("emits warn when SelfSubjectAccessReview itself throws", async () => { + vi.mocked(getAuthzApi).mockReturnValue({ + createSelfSubjectAccessReview: vi.fn().mockRejectedValue(new Error("SSAR not available")), + } as unknown as ReturnType); + + const result = await testEnvironment(makeCtx()); + + const rbacWarns = result.checks.filter((c) => c.code.startsWith("k8s_rbac_") && c.level === "warn"); + expect(rbacWarns.length).toBeGreaterThan(0); + }); +}); + +describe("testEnvironment — secrets", () => { + it("emits warn when the secret is not found", async () => { + const coreApi = makeCore({ + readNamespacedSecret: vi.fn().mockRejectedValue(new Error("not found")), + }); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx()); + + expect(result.checks.find((c) => c.code === "k8s_secret_missing")).toBeDefined(); + expect(result.status).toBe("warn"); + }); + + it("uses configured secretRef when provided", async () => { + const coreApi = makeCore(); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + await testEnvironment(makeCtx({ secretRef: "custom-secret" })); + + expect(coreApi.readNamespacedSecret).toHaveBeenCalledWith({ name: "custom-secret", namespace: "ns-self" }); + }); +}); + +describe("testEnvironment — PVC", () => { + it("emits warn when no PVC is mounted on /paperclip", async () => { + vi.mocked(getSelfPodInfo).mockResolvedValue({ ...SELF_POD, pvcClaimName: null }); + + const result = await testEnvironment(makeCtx()); + + expect(result.checks.find((c) => c.code === "k8s_pvc_not_detected")).toBeDefined(); + expect(result.status).toBe("warn"); + }); + + it("emits warn when PVC access mode is not ReadWriteMany", async () => { + const coreApi = makeCore({ + readNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({ spec: { accessModes: ["ReadWriteOnce"] } }), + }); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx()); + + const pvcCheck = result.checks.find((c) => c.code === "k8s_pvc_not_rwx"); + expect(pvcCheck).toBeDefined(); + expect(pvcCheck?.message).toContain("ReadWriteOnce"); + expect(result.status).toBe("warn"); + }); + + it("emits warn when reading the PVC fails", async () => { + const coreApi = makeCore({ + readNamespacedPersistentVolumeClaim: vi.fn().mockRejectedValue(new Error("api error")), + }); + vi.mocked(getCoreApi).mockReturnValue(coreApi as unknown as ReturnType); + + const result = await testEnvironment(makeCtx()); + + expect(result.checks.find((c) => c.code === "k8s_pvc_check_failed")).toBeDefined(); + }); +});