feat: add K8s API server, orchestrator abstraction, and CI pipeline

- Add apps/api/ — Hono REST API server for managing pentest scans via K8s Jobs - POST/GET /api/scans, GET /api/scans/:id, cancel, report endpoints - Bearer token auth, Temporal client integration, K8s Job builder - Dockerfile, Kustomize manifests (Deployment, Service, RBAC) - Add CLI orchestrator abstraction (docker.ts → Orchestrator interface) - DockerOrchestrator and K8sOrchestrator implementations - Backend detection via SHANNON_BACKEND env var or --backend flag - Add CI workflow: type-check + lint on PR, build+push both images on main - Switch all workflows to self-hosted runners (runners-farhoodliquor) - Add shannon-api image build to release and release-beta workflows - Add root infra/kustomization.yaml as Flux entry point - Export PipelineProgress from @shannon/worker/pipeline Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-19 13:08:51 -04:00
parent 54c92e8142
commit 1bbdd7acba
36 changed files with 2635 additions and 414 deletions
@@ -0,0 +1,141 @@
+/**
+ * K8s Job spec builder for worker scan Jobs.
+ * Constructs a Job that runs the Shannon worker image with the correct
+ * volumes, env, and security context. Optionally includes a git clone init container.
+ */
+
+import type * as k8s from '@kubernetes/client-node';
+
+export interface JobParams {
+  readonly jobName: string;
+  readonly namespace: string;
+  readonly workerImage: string;
+  readonly targetUrl: string;
+  readonly taskQueue: string;
+  readonly workspace: string;
+  readonly credentialsSecretName: string;
+  readonly gitUrl?: string;
+  readonly gitRef?: string;
+  readonly repoPath?: string;
+  readonly configYaml?: string;
+  readonly pipelineTesting?: boolean;
+}
+
+const WORKER_LABEL = 'shannon-worker';
+const REPO_MOUNT_PATH = '/repo';
+
+export function buildJobSpec(params: JobParams): k8s.V1Job {
+  const repoPath = params.repoPath ?? REPO_MOUNT_PATH;
+
+  // 1. Build worker command
+  const command = ['node', 'apps/worker/dist/temporal/worker.js', params.targetUrl, repoPath];
+  const args: string[] = ['--task-queue', params.taskQueue, '--workspace', params.workspace];
+  if (params.pipelineTesting) {
+    args.push('--pipeline-testing');
+  }
+
+  // 2. Build volumes and mounts
+  const volumes: k8s.V1Volume[] = [
+    { name: 'workspaces', persistentVolumeClaim: { claimName: 'shannon-workspaces' } },
+    { name: 'shm', emptyDir: { medium: 'Memory', sizeLimit: '2Gi' } },
+  ];
+
+  const volumeMounts: k8s.V1VolumeMount[] = [
+    { name: 'workspaces', mountPath: '/app/workspaces' },
+    { name: 'shm', mountPath: '/dev/shm' },
+  ];
+
+  // Overlay dirs (writable areas over the read-only repo)
+  for (const overlay of ['deliverables', 'scratchpad', 'playwright-cli']) {
+    const volName = `overlay-${overlay}`;
+    volumes.push({ name: volName, emptyDir: {} });
+    volumeMounts.push({
+      name: volName,
+      mountPath: `${repoPath}/.shannon/${overlay === 'playwright-cli' ? '.playwright-cli' : overlay}`,
+    });
+  }
+
+  // 3. Repo volume — emptyDir for git clone, or PVC sub-path for pre-staged repos
+  const initContainers: k8s.V1Container[] = [];
+
+  if (params.gitUrl) {
+    // Git clone into an emptyDir
+    volumes.push({ name: 'repo', emptyDir: {} });
+    volumeMounts.push({ name: 'repo', mountPath: REPO_MOUNT_PATH, readOnly: true });
+
+    const cloneArgs = ['clone', '--depth', '1'];
+    if (params.gitRef) {
+      cloneArgs.push('--branch', params.gitRef);
+    }
+    cloneArgs.push(params.gitUrl, REPO_MOUNT_PATH);
+
+    initContainers.push({
+      name: 'git-clone',
+      image: 'bitnami/git:2',
+      command: ['git'],
+      args: cloneArgs,
+      volumeMounts: [{ name: 'repo', mountPath: REPO_MOUNT_PATH }],
+    });
+  } else if (params.repoPath) {
+    // Repo already on a PVC — mount the workspaces PVC (assumes repo is staged there)
+    volumeMounts.push({
+      name: 'workspaces',
+      mountPath: repoPath,
+      readOnly: true,
+      subPath: `repos/${params.workspace}`,
+    });
+  }
+
+  // 4. Env vars
+  const env: k8s.V1EnvVar[] = [{ name: 'TEMPORAL_ADDRESS', value: 'shannon-temporal:7233' }];
+
+  // 5. Construct the Job
+  return {
+    apiVersion: 'batch/v1',
+    kind: 'Job',
+    metadata: {
+      name: params.jobName,
+      namespace: params.namespace,
+      labels: {
+        app: WORKER_LABEL,
+        'shannon.io/workspace': params.workspace,
+        'shannon.io/scan-id': params.jobName,
+      },
+    },
+    spec: {
+      backoffLimit: 0,
+      ttlSecondsAfterFinished: 3600,
+      template: {
+        metadata: {
+          labels: {
+            app: WORKER_LABEL,
+            'shannon.io/workspace': params.workspace,
+          },
+        },
+        spec: {
+          restartPolicy: 'Never',
+          serviceAccountName: 'default',
+          securityContext: {
+            seccompProfile: { type: 'Unconfined' },
+          },
+          ...(initContainers.length > 0 && { initContainers }),
+          containers: [
+            {
+              name: 'worker',
+              image: params.workerImage,
+              command,
+              args,
+              env,
+              envFrom: [{ secretRef: { name: params.credentialsSecretName } }],
+              volumeMounts,
+              resources: {
+                requests: { memory: '2Gi' },
+              },
+            },
+          ],
+          volumes,
+        },
+      },
+    },
+  };
+}