feat(plugin): add kubernetes sandbox provider

2026-05-12 12:13:07 -05:00
parent 55d6c5bfa4
commit 163e3ca1a5
42 changed files with 4168 additions and 0 deletions
@@ -0,0 +1,22 @@
+import { execSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const KIND_CONTEXT = "kind-paperclip";
+
+export function readKindKubeconfig(): string {
+  return readFileSync(join(homedir(), ".kube", "config"), "utf-8");
+}
+
+export function kubectl(args: string): string {
+  return execSync(`kubectl --context ${KIND_CONTEXT} ${args}`, { encoding: "utf-8" });
+}
+
+export function deleteNamespaceIfExists(namespace: string): void {
+  try {
+    kubectl(`delete namespace ${namespace} --wait=true --timeout=60s --ignore-not-found`);
+  } catch {
+    // ignore
+  }
+}
@@ -0,0 +1,205 @@
+/**
+ * End-to-end integration test against a local kind cluster.
+ *
+ * PREREQUISITES (operator must perform before running this test):
+ *   1. Create the kind cluster:
+ *        kind create cluster --name paperclip
+ *   2. Pre-load the alpine image so the Job can start without network access:
+ *        docker pull alpine:3.20
+ *        docker tag alpine:3.20 localhost/paperclip-agent:latest
+ *        kind load docker-image localhost/paperclip-agent:latest --name paperclip
+ *   3. For the sandbox-cr backend test, the agent-sandbox controller must be installed:
+ *        kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/install.yaml
+ *      And a tini-bearing image pre-loaded (e.g. the same localhost/paperclip-agent:latest
+ *      if it includes /usr/bin/tini and /bin/sh).
+ *   4. Set the env var and run:
+ *        RUN_K8S_INTEGRATION_TESTS=1 pnpm test
+ *
+ * The namespace is derived from companySlug ("spike-e2e") + namespacePrefix
+ * ("paperclip-"), resolving to "paperclip-spike-e2e".
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import plugin from "../../src/plugin.js";
+import { createKubeConfig } from "../../src/kube-client.js";
+import { execInPod } from "../../src/pod-exec.js";
+import { sandboxCrOrchestrator } from "../../src/sandbox-cr-orchestrator.js";
+import { deleteNamespaceIfExists, kubectl, readKindKubeconfig } from "./_kind-harness.js";
+
+const NAMESPACE = "paperclip-spike-e2e";
+
+describe("plugin-kubernetes end-to-end", () => {
+  beforeAll(() => {
+    if (process.env.RUN_K8S_INTEGRATION_TESTS !== "1") return;
+    deleteNamespaceIfExists(NAMESPACE);
+  });
+
+  afterAll(() => {
+    if (process.env.RUN_K8S_INTEGRATION_TESTS !== "1") return;
+    deleteNamespaceIfExists(NAMESPACE);
+  });
+
+  // ── Job backend (stable fallback) ─────────────────────────────────────────
+
+  it.runIf(process.env.RUN_K8S_INTEGRATION_TESTS === "1")(
+    "[job backend] acquireLease creates tenant + Job + supporting resources; releaseLease cascade-deletes them",
+    async () => {
+      const kubeconfig = readKindKubeconfig();
+      const config = {
+        inCluster: false,
+        kubeconfig,
+        companySlug: "spike-e2e",
+        adapterType: "claude_local",
+        backend: "job",
+        imageAllowList: [] as string[],
+        podActivityDeadlineSec: 60,
+        jobTtlSecondsAfterFinished: 60,
+      };
+
+      const lease = await plugin.definition.onEnvironmentAcquireLease!({
+        driverKey: "kubernetes",
+        config,
+        runId: "r-test-e2e-job",
+        companyId: "11111111-1111-1111-1111-111111111111",
+        environmentId: "env-test",
+      });
+
+      expect(lease.providerLeaseId).toMatch(/^pc-/);
+
+      // Verify the Job exists in the tenant namespace
+      const jobs = kubectl(`get jobs -n ${NAMESPACE} -o name`);
+      expect(jobs).toContain(`job.batch/${lease.providerLeaseId}`);
+
+      // Verify the tenant namespace has the expected supporting resources
+      const all = kubectl(
+        `get sa,role,rolebinding,resourcequota,limitrange,networkpolicy -n ${NAMESPACE} -o name`,
+      );
+      expect(all).toContain("serviceaccount/paperclip-tenant-sa");
+      expect(all).toContain("role.rbac.authorization.k8s.io/paperclip-tenant-role");
+      expect(all).toContain("rolebinding.rbac.authorization.k8s.io/paperclip-tenant-rb");
+      expect(all).toContain("resourcequota/paperclip-quota");
+      expect(all).toContain("limitrange/paperclip-limits");
+      expect(all).toContain("networkpolicy.networking.k8s.io/paperclip-deny-all");
+      expect(all).toContain("networkpolicy.networking.k8s.io/paperclip-egress-allow");
+
+      // Verify the namespace has PSS-restricted labels
+      const ns = kubectl(`get namespace ${NAMESPACE} -o jsonpath='{.metadata.labels}'`);
+      expect(ns).toContain("pod-security.kubernetes.io/enforce");
+      expect(ns).toContain("restricted");
+
+      // Verify the per-run Secret exists (owned by the Job for cascade deletion)
+      const secrets = kubectl(`get secrets -n ${NAMESPACE} -o name`);
+      expect(secrets).toContain(`secret/${lease.providerLeaseId}-env`);
+
+      // Release — deletes the Job with Foreground propagation, which cascade-deletes
+      // the owned Secret via owner references set at acquireLease time.
+      await plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        config,
+        providerLeaseId: lease.providerLeaseId,
+        leaseMetadata: lease.metadata,
+        companyId: "11111111-1111-1111-1111-111111111111",
+        environmentId: "env-test",
+      });
+
+      // Allow a brief grace window for Foreground propagation to finish.
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+
+      const jobsAfter = kubectl(`get jobs -n ${NAMESPACE} -o name 2>&1 || true`);
+      expect(jobsAfter).not.toContain(`job.batch/${lease.providerLeaseId}`);
+    },
+    180_000,
+  );
+
+  // ── Sandbox-CR backend (alpha, requires agent-sandbox controller) ──────────
+
+  it.runIf(process.env.RUN_K8S_INTEGRATION_TESTS === "1")(
+    "[sandbox-cr backend] acquireLease creates Sandbox CR + supporting resources; pod becomes Ready; execInPod runs echo hello; releaseLease deletes CR",
+    async () => {
+      const kubeconfig = readKindKubeconfig();
+      const config = {
+        inCluster: false,
+        kubeconfig,
+        companySlug: "spike-e2e",
+        adapterType: "claude_local",
+        backend: "sandbox-cr",
+        imageAllowList: [] as string[],
+        podActivityDeadlineSec: 120,
+        jobTtlSecondsAfterFinished: 60,
+      };
+
+      const lease = await plugin.definition.onEnvironmentAcquireLease!({
+        driverKey: "kubernetes",
+        config,
+        runId: "r-test-e2e-sandbox-cr",
+        companyId: "22222222-2222-2222-2222-222222222222",
+        environmentId: "env-test-cr",
+      });
+
+      expect(lease.providerLeaseId).toMatch(/^pc-/);
+
+      // Verify the Sandbox CR exists in the tenant namespace
+      const sandboxes = kubectl(
+        `get sandboxes.agents.x-k8s.io -n ${NAMESPACE} -o name 2>&1`,
+      );
+      expect(sandboxes).toContain(`sandbox.agents.x-k8s.io/${lease.providerLeaseId}`);
+
+      // Verify the per-run Secret exists (owned by the Sandbox CR)
+      const secrets = kubectl(`get secrets -n ${NAMESPACE} -o name`);
+      expect(secrets).toContain(`secret/${lease.providerLeaseId}-env`);
+
+      // Wait for the Sandbox pod to become Ready
+      const kc = createKubeConfig({ inCluster: false, kubeconfig });
+      const { makeKubeClients } = await import("../../src/kube-client.js");
+      const clients = makeKubeClients(kc);
+
+      await sandboxCrOrchestrator.waitForCompletion(
+        clients,
+        NAMESPACE,
+        lease.providerLeaseId,
+        { timeoutMs: 90_000, pollMs: 3000 },
+      );
+
+      // Resolve the pod name
+      const podName = await sandboxCrOrchestrator.findPod(
+        clients,
+        NAMESPACE,
+        lease.providerLeaseId,
+      );
+      expect(podName).toBeTruthy();
+
+      // Exec a simple echo command into the running pod
+      const execResult = await execInPod(
+        kc,
+        NAMESPACE,
+        podName!,
+        "agent",
+        ["echo", "hello"],
+      );
+
+      expect(execResult.exitCode).toBe(0);
+      expect(execResult.stdout.trim()).toBe("hello");
+
+      // Release — deletes the Sandbox CR with Foreground propagation.
+      await plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        config,
+        providerLeaseId: lease.providerLeaseId,
+        leaseMetadata: lease.metadata,
+        companyId: "22222222-2222-2222-2222-222222222222",
+        environmentId: "env-test-cr",
+      });
+
+      // Allow a brief grace window for Foreground propagation.
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+
+      const sandboxesAfter = kubectl(
+        `get sandboxes.agents.x-k8s.io -n ${NAMESPACE} -o name 2>&1 || true`,
+      );
+      expect(sandboxesAfter).not.toContain(
+        `sandbox.agents.x-k8s.io/${lease.providerLeaseId}`,
+      );
+    },
+    300_000,
+  );
+});