feat: add K8s API server, orchestrator abstraction, and CI pipeline
- Add apps/api/ — Hono REST API server for managing pentest scans via K8s Jobs - POST/GET /api/scans, GET /api/scans/:id, cancel, report endpoints - Bearer token auth, Temporal client integration, K8s Job builder - Dockerfile, Kustomize manifests (Deployment, Service, RBAC) - Add CLI orchestrator abstraction (docker.ts → Orchestrator interface) - DockerOrchestrator and K8sOrchestrator implementations - Backend detection via SHANNON_BACKEND env var or --backend flag - Add CI workflow: type-check + lint on PR, build+push both images on main - Switch all workflows to self-hosted runners (runners-farhoodliquor) - Add shannon-api image build to release and release-beta workflows - Add root infra/kustomization.yaml as Flux entry point - Export PipelineProgress from @shannon/worker/pipeline Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* K8s Job spec builder for worker scan Jobs.
|
||||
* Constructs a Job that runs the Shannon worker image with the correct
|
||||
* volumes, env, and security context. Optionally includes a git clone init container.
|
||||
*/
|
||||
|
||||
import type * as k8s from '@kubernetes/client-node';
|
||||
|
||||
export interface JobParams {
|
||||
readonly jobName: string;
|
||||
readonly namespace: string;
|
||||
readonly workerImage: string;
|
||||
readonly targetUrl: string;
|
||||
readonly taskQueue: string;
|
||||
readonly workspace: string;
|
||||
readonly credentialsSecretName: string;
|
||||
readonly gitUrl?: string;
|
||||
readonly gitRef?: string;
|
||||
readonly repoPath?: string;
|
||||
readonly configYaml?: string;
|
||||
readonly pipelineTesting?: boolean;
|
||||
}
|
||||
|
||||
const WORKER_LABEL = 'shannon-worker';
|
||||
const REPO_MOUNT_PATH = '/repo';
|
||||
|
||||
export function buildJobSpec(params: JobParams): k8s.V1Job {
|
||||
const repoPath = params.repoPath ?? REPO_MOUNT_PATH;
|
||||
|
||||
// 1. Build worker command
|
||||
const command = ['node', 'apps/worker/dist/temporal/worker.js', params.targetUrl, repoPath];
|
||||
const args: string[] = ['--task-queue', params.taskQueue, '--workspace', params.workspace];
|
||||
if (params.pipelineTesting) {
|
||||
args.push('--pipeline-testing');
|
||||
}
|
||||
|
||||
// 2. Build volumes and mounts
|
||||
const volumes: k8s.V1Volume[] = [
|
||||
{ name: 'workspaces', persistentVolumeClaim: { claimName: 'shannon-workspaces' } },
|
||||
{ name: 'shm', emptyDir: { medium: 'Memory', sizeLimit: '2Gi' } },
|
||||
];
|
||||
|
||||
const volumeMounts: k8s.V1VolumeMount[] = [
|
||||
{ name: 'workspaces', mountPath: '/app/workspaces' },
|
||||
{ name: 'shm', mountPath: '/dev/shm' },
|
||||
];
|
||||
|
||||
// Overlay dirs (writable areas over the read-only repo)
|
||||
for (const overlay of ['deliverables', 'scratchpad', 'playwright-cli']) {
|
||||
const volName = `overlay-${overlay}`;
|
||||
volumes.push({ name: volName, emptyDir: {} });
|
||||
volumeMounts.push({
|
||||
name: volName,
|
||||
mountPath: `${repoPath}/.shannon/${overlay === 'playwright-cli' ? '.playwright-cli' : overlay}`,
|
||||
});
|
||||
}
|
||||
|
||||
// 3. Repo volume — emptyDir for git clone, or PVC sub-path for pre-staged repos
|
||||
const initContainers: k8s.V1Container[] = [];
|
||||
|
||||
if (params.gitUrl) {
|
||||
// Git clone into an emptyDir
|
||||
volumes.push({ name: 'repo', emptyDir: {} });
|
||||
volumeMounts.push({ name: 'repo', mountPath: REPO_MOUNT_PATH, readOnly: true });
|
||||
|
||||
const cloneArgs = ['clone', '--depth', '1'];
|
||||
if (params.gitRef) {
|
||||
cloneArgs.push('--branch', params.gitRef);
|
||||
}
|
||||
cloneArgs.push(params.gitUrl, REPO_MOUNT_PATH);
|
||||
|
||||
initContainers.push({
|
||||
name: 'git-clone',
|
||||
image: 'bitnami/git:2',
|
||||
command: ['git'],
|
||||
args: cloneArgs,
|
||||
volumeMounts: [{ name: 'repo', mountPath: REPO_MOUNT_PATH }],
|
||||
});
|
||||
} else if (params.repoPath) {
|
||||
// Repo already on a PVC — mount the workspaces PVC (assumes repo is staged there)
|
||||
volumeMounts.push({
|
||||
name: 'workspaces',
|
||||
mountPath: repoPath,
|
||||
readOnly: true,
|
||||
subPath: `repos/${params.workspace}`,
|
||||
});
|
||||
}
|
||||
|
||||
// 4. Env vars
|
||||
const env: k8s.V1EnvVar[] = [{ name: 'TEMPORAL_ADDRESS', value: 'shannon-temporal:7233' }];
|
||||
|
||||
// 5. Construct the Job
|
||||
return {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: {
|
||||
name: params.jobName,
|
||||
namespace: params.namespace,
|
||||
labels: {
|
||||
app: WORKER_LABEL,
|
||||
'shannon.io/workspace': params.workspace,
|
||||
'shannon.io/scan-id': params.jobName,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
ttlSecondsAfterFinished: 3600,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
app: WORKER_LABEL,
|
||||
'shannon.io/workspace': params.workspace,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
serviceAccountName: 'default',
|
||||
securityContext: {
|
||||
seccompProfile: { type: 'Unconfined' },
|
||||
},
|
||||
...(initContainers.length > 0 && { initContainers }),
|
||||
containers: [
|
||||
{
|
||||
name: 'worker',
|
||||
image: params.workerImage,
|
||||
command,
|
||||
args,
|
||||
env,
|
||||
envFrom: [{ secretRef: { name: params.credentialsSecretName } }],
|
||||
volumeMounts,
|
||||
resources: {
|
||||
requests: { memory: '2Gi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
volumes,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* K8s Job lifecycle management — create, delete, list worker Jobs.
|
||||
*/
|
||||
|
||||
import type * as k8s from '@kubernetes/client-node';
|
||||
|
||||
const WORKER_LABEL = 'shannon-worker';
|
||||
|
||||
export async function createJob(batchApi: k8s.BatchV1Api, namespace: string, job: k8s.V1Job): Promise<void> {
|
||||
await batchApi.createNamespacedJob({ namespace, body: job });
|
||||
}
|
||||
|
||||
export async function deleteJob(batchApi: k8s.BatchV1Api, namespace: string, name: string): Promise<void> {
|
||||
await batchApi.deleteNamespacedJob({
|
||||
name,
|
||||
namespace,
|
||||
propagationPolicy: 'Background',
|
||||
});
|
||||
}
|
||||
|
||||
export async function getJob(batchApi: k8s.BatchV1Api, namespace: string, name: string): Promise<k8s.V1Job | null> {
|
||||
try {
|
||||
return await batchApi.readNamespacedJob({ name, namespace });
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listWorkerJobs(batchApi: k8s.BatchV1Api, namespace: string): Promise<k8s.V1Job[]> {
|
||||
const response = await batchApi.listNamespacedJob({
|
||||
namespace,
|
||||
labelSelector: `app=${WORKER_LABEL}`,
|
||||
});
|
||||
return response.items;
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Scan lifecycle orchestration — combines Temporal queries with K8s Job management.
|
||||
* This is the main service that route handlers delegate to.
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import type * as k8s from '@kubernetes/client-node';
|
||||
import type { Client } from '@temporalio/client';
|
||||
import type { Config } from '../config.js';
|
||||
import type { CreateScanInput, ScanResponse } from '../types/api.js';
|
||||
import { buildJobSpec } from './job-builder.js';
|
||||
import { createJob, deleteJob, listWorkerJobs } from './job-manager.js';
|
||||
import { cancelWorkflow, queryProgress } from './temporal-client.js';
|
||||
import { listWorkspaces, readReport, readSessionJson } from './workspace-reader.js';
|
||||
|
||||
function randomSuffix(): string {
|
||||
return crypto.randomBytes(4).toString('hex');
|
||||
}
|
||||
|
||||
// === Start Scan ===
|
||||
|
||||
export async function startScan(
|
||||
config: Config,
|
||||
batchApi: k8s.BatchV1Api,
|
||||
input: CreateScanInput,
|
||||
): Promise<ScanResponse> {
|
||||
const suffix = randomSuffix();
|
||||
const taskQueue = `api-${suffix}`;
|
||||
const jobName = `shannon-worker-${suffix}`;
|
||||
|
||||
const workspace =
|
||||
input.workspace ?? `${new URL(input.targetUrl).hostname.replace(/[^a-zA-Z0-9-]/g, '-')}_shannon-${Date.now()}`;
|
||||
|
||||
const job = buildJobSpec({
|
||||
jobName,
|
||||
namespace: config.k8sNamespace,
|
||||
workerImage: config.workerImage,
|
||||
targetUrl: input.targetUrl,
|
||||
taskQueue,
|
||||
workspace,
|
||||
credentialsSecretName: config.credentialsSecretName,
|
||||
...(input.gitUrl && { gitUrl: input.gitUrl }),
|
||||
...(input.gitRef && { gitRef: input.gitRef }),
|
||||
...(input.repoPath && { repoPath: input.repoPath }),
|
||||
...(input.configYaml && { configYaml: input.configYaml }),
|
||||
...(input.pipelineTesting && { pipelineTesting: true }),
|
||||
});
|
||||
|
||||
await createJob(batchApi, config.k8sNamespace, job);
|
||||
|
||||
return {
|
||||
id: jobName,
|
||||
workspace,
|
||||
targetUrl: input.targetUrl,
|
||||
status: 'running',
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// === Get Scan ===
|
||||
|
||||
export async function getScan(config: Config, temporalClient: Client, scanId: string): Promise<ScanResponse | null> {
|
||||
// 1. Try Temporal query for live progress
|
||||
try {
|
||||
const progress = await queryProgress(temporalClient, scanId);
|
||||
return {
|
||||
id: scanId,
|
||||
workspace: scanId,
|
||||
targetUrl: '',
|
||||
status: progress.status,
|
||||
createdAt: new Date(progress.startTime).toISOString(),
|
||||
completedAgents: progress.completedAgents,
|
||||
agentMetrics: progress.agentMetrics,
|
||||
...(progress.currentPhase && { currentPhase: progress.currentPhase }),
|
||||
...(progress.currentAgent && { currentAgent: progress.currentAgent }),
|
||||
...(progress.summary && { summary: progress.summary }),
|
||||
...(progress.error && { error: progress.error }),
|
||||
};
|
||||
} catch {
|
||||
// Workflow not found in Temporal — try workspace session.json
|
||||
}
|
||||
|
||||
// 2. Fall back to workspace session.json (completed/historical scans)
|
||||
const session = readSessionJson(config.workspacesDir, scanId);
|
||||
if (!session) return null;
|
||||
|
||||
return {
|
||||
id: session.originalWorkflowId ?? scanId,
|
||||
workspace: session.workspace,
|
||||
targetUrl: session.webUrl ?? '',
|
||||
status: 'completed',
|
||||
createdAt: session.startTime ? new Date(session.startTime).toISOString() : '',
|
||||
};
|
||||
}
|
||||
|
||||
// === List Scans ===
|
||||
|
||||
export async function listScans(
|
||||
config: Config,
|
||||
_temporalClient: Client,
|
||||
batchApi: k8s.BatchV1Api,
|
||||
): Promise<ScanResponse[]> {
|
||||
const results: ScanResponse[] = [];
|
||||
|
||||
// 1. Running scans from K8s Jobs
|
||||
const jobs = await listWorkerJobs(batchApi, config.k8sNamespace);
|
||||
for (const job of jobs) {
|
||||
const jobName = job.metadata?.name ?? '';
|
||||
const workspace = job.metadata?.labels?.['shannon.io/workspace'] ?? jobName;
|
||||
const startTime = job.status?.startTime;
|
||||
|
||||
results.push({
|
||||
id: jobName,
|
||||
workspace,
|
||||
targetUrl: '',
|
||||
status: job.status?.succeeded ? 'completed' : job.status?.failed ? 'failed' : 'running',
|
||||
createdAt: startTime ? new Date(startTime).toISOString() : '',
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Historical scans from workspace session.json files
|
||||
const workspaces = listWorkspaces(config.workspacesDir);
|
||||
const jobNames = new Set(results.map((r) => r.workspace));
|
||||
|
||||
for (const ws of workspaces) {
|
||||
if (jobNames.has(ws.workspace)) continue;
|
||||
results.push({
|
||||
id: ws.originalWorkflowId ?? ws.workspace,
|
||||
workspace: ws.workspace,
|
||||
targetUrl: ws.webUrl ?? '',
|
||||
status: 'completed',
|
||||
createdAt: ws.startTime ? new Date(ws.startTime).toISOString() : '',
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// === Cancel Scan ===
|
||||
|
||||
export async function cancelScan(
|
||||
config: Config,
|
||||
temporalClient: Client,
|
||||
batchApi: k8s.BatchV1Api,
|
||||
scanId: string,
|
||||
): Promise<void> {
|
||||
// Cancel Temporal workflow (best-effort)
|
||||
try {
|
||||
await cancelWorkflow(temporalClient, scanId);
|
||||
} catch {
|
||||
// Workflow may have already completed
|
||||
}
|
||||
|
||||
// Delete K8s Job
|
||||
try {
|
||||
await deleteJob(batchApi, config.k8sNamespace, scanId);
|
||||
} catch {
|
||||
// Job may have already been cleaned up
|
||||
}
|
||||
}
|
||||
|
||||
// === Get Report ===
|
||||
|
||||
export async function getReport(config: Config, scanId: string): Promise<string | null> {
|
||||
return readReport(config.workspacesDir, scanId);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Temporal client management — connection lifecycle and workflow operations.
|
||||
* Uses @temporalio/client (not worker) since the API server only submits and queries workflows.
|
||||
*/
|
||||
|
||||
import type { PipelineProgress } from '@shannon/worker/pipeline';
|
||||
import { Client, Connection } from '@temporalio/client';
|
||||
|
||||
export interface TemporalClients {
|
||||
readonly client: Client;
|
||||
readonly connection: Connection;
|
||||
}
|
||||
|
||||
export async function connectTemporal(address: string): Promise<TemporalClients> {
|
||||
console.log(`Connecting to Temporal at ${address}...`);
|
||||
const connection = await Connection.connect({ address });
|
||||
const client = new Client({ connection });
|
||||
console.log('Temporal connected.');
|
||||
return { client, connection };
|
||||
}
|
||||
|
||||
export async function disconnectTemporal(clients: TemporalClients): Promise<void> {
|
||||
await clients.connection.close();
|
||||
}
|
||||
|
||||
/** Query a workflow's progress via the getProgress query. */
|
||||
export async function queryProgress(client: Client, workflowId: string): Promise<PipelineProgress> {
|
||||
const handle = client.workflow.getHandle(workflowId);
|
||||
return handle.query<PipelineProgress>('getProgress');
|
||||
}
|
||||
|
||||
/** Cancel a running workflow. */
|
||||
export async function cancelWorkflow(client: Client, workflowId: string): Promise<void> {
|
||||
const handle = client.workflow.getHandle(workflowId);
|
||||
await handle.cancel();
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Workspace reader — reads session.json and deliverables from the shared workspaces PVC.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export interface SessionInfo {
|
||||
readonly workspace: string;
|
||||
readonly originalWorkflowId?: string;
|
||||
readonly webUrl?: string;
|
||||
readonly startTime?: number;
|
||||
readonly cost?: number;
|
||||
readonly resumeAttempts?: readonly { workflowId: string; timestamp: number }[];
|
||||
}
|
||||
|
||||
export function readSessionJson(workspacesDir: string, workspace: string): SessionInfo | null {
|
||||
const sessionPath = path.join(workspacesDir, workspace, 'session.json');
|
||||
try {
|
||||
const raw = fs.readFileSync(sessionPath, 'utf-8');
|
||||
const data = JSON.parse(raw) as Record<string, unknown>;
|
||||
const session = data.session as Record<string, unknown> | undefined;
|
||||
const originalWorkflowId = session?.originalWorkflowId as string | undefined;
|
||||
const webUrl = session?.webUrl as string | undefined;
|
||||
const startTime = session?.startTime as number | undefined;
|
||||
const cost = session?.totalCostUsd as number | undefined;
|
||||
const resumeAttempts = session?.resumeAttempts as SessionInfo['resumeAttempts'];
|
||||
|
||||
return {
|
||||
workspace,
|
||||
...(originalWorkflowId && { originalWorkflowId }),
|
||||
...(webUrl && { webUrl }),
|
||||
...(startTime && { startTime }),
|
||||
...(cost && { cost }),
|
||||
...(resumeAttempts && { resumeAttempts }),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function readReport(workspacesDir: string, workspace: string): string | null {
|
||||
const delivDir = path.join(workspacesDir, workspace, 'deliverables');
|
||||
try {
|
||||
const files = fs.readdirSync(delivDir);
|
||||
const reportFile = files.find((f) => f.includes('report') && f.endsWith('.md'));
|
||||
if (!reportFile) return null;
|
||||
return fs.readFileSync(path.join(delivDir, reportFile), 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function listWorkspaces(workspacesDir: string): SessionInfo[] {
|
||||
try {
|
||||
const entries = fs.readdirSync(workspacesDir, { withFileTypes: true });
|
||||
const results: SessionInfo[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const session = readSessionJson(workspacesDir, entry.name);
|
||||
if (session) {
|
||||
results.push(session);
|
||||
}
|
||||
}
|
||||
|
||||
return results.sort((a, b) => (b.startTime ?? 0) - (a.startTime ?? 0));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user