feat: add K8s API server, orchestrator abstraction, and CI pipeline
- Add apps/api/ — Hono REST API server for managing pentest scans via K8s Jobs - POST/GET /api/scans, GET /api/scans/:id, cancel, report endpoints - Bearer token auth, Temporal client integration, K8s Job builder - Dockerfile, Kustomize manifests (Deployment, Service, RBAC) - Add CLI orchestrator abstraction (docker.ts → Orchestrator interface) - DockerOrchestrator and K8sOrchestrator implementations - Backend detection via SHANNON_BACKEND env var or --backend flag - Add CI workflow: type-check + lint on PR, build+push both images on main - Switch all workflows to self-hosted runners (runners-farhoodliquor) - Add shannon-api image build to release and release-beta workflows - Add root infra/kustomization.yaml as Flux entry point - Export PipelineProgress from @shannon/worker/pipeline Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Backend detection — Docker (default) vs Kubernetes.
|
||||
*
|
||||
* Orthogonal to the local/npx mode axis. Mode controls where state lives
|
||||
* and where the image comes from. Backend controls how containers are orchestrated.
|
||||
*/
|
||||
|
||||
import type { Orchestrator } from './orchestrator.js';
|
||||
|
||||
export type Backend = 'docker' | 'k8s';
|
||||
|
||||
let cachedBackend: Backend | undefined;
|
||||
let cachedOrchestrator: Orchestrator | undefined;
|
||||
|
||||
/**
|
||||
* Detect the orchestration backend.
|
||||
* SHANNON_BACKEND env var takes precedence, otherwise defaults to docker.
|
||||
*/
|
||||
export function getBackend(): Backend {
|
||||
if (cachedBackend !== undefined) return cachedBackend;
|
||||
|
||||
const env = process.env.SHANNON_BACKEND;
|
||||
if (env === 'k8s' || env === 'kubernetes') {
|
||||
cachedBackend = 'k8s';
|
||||
} else {
|
||||
cachedBackend = 'docker';
|
||||
}
|
||||
return cachedBackend;
|
||||
}
|
||||
|
||||
export function setBackend(backend: Backend): void {
|
||||
cachedBackend = backend;
|
||||
cachedOrchestrator = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the orchestrator for the current backend.
|
||||
* Lazy-loads the implementation to avoid importing unused dependencies.
|
||||
*/
|
||||
export async function getOrchestrator(): Promise<Orchestrator> {
|
||||
if (cachedOrchestrator) return cachedOrchestrator;
|
||||
|
||||
let orchestrator: Orchestrator;
|
||||
if (getBackend() === 'k8s') {
|
||||
const { K8sOrchestrator } = await import('./k8s.js');
|
||||
orchestrator = new K8sOrchestrator();
|
||||
} else {
|
||||
const { DockerOrchestrator } = await import('./docker.js');
|
||||
orchestrator = new DockerOrchestrator();
|
||||
}
|
||||
|
||||
cachedOrchestrator = orchestrator;
|
||||
return orchestrator;
|
||||
}
|
||||
@@ -5,11 +5,11 @@
|
||||
* and npx mode (Docker Hub pull, ~/.shannon/).
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { ensureImage, ensureInfra, randomSuffix, spawnWorker } from '../docker.js';
|
||||
import { getOrchestrator } from '../backend.js';
|
||||
import { randomSuffix } from '../docker.js';
|
||||
import { buildEnvFlags, isRouterConfigured, loadEnv, validateCredentials } from '../env.js';
|
||||
import { getCredentialsPath, getWorkspacesDir, initHome } from '../home.js';
|
||||
import { isLocal } from '../mode.js';
|
||||
@@ -55,9 +55,10 @@ export async function start(args: StartArgs): Promise<void> {
|
||||
process.env.ANTHROPIC_AUTH_TOKEN = 'shannon-router-key';
|
||||
}
|
||||
|
||||
// 6. Ensure image (auto-build in dev, pull in npx) and start infra
|
||||
ensureImage(args.version);
|
||||
await ensureInfra(useRouter);
|
||||
// 6. Ensure image and start infra via orchestrator
|
||||
const orchestrator = await getOrchestrator();
|
||||
orchestrator.ensureImage(args.version);
|
||||
await orchestrator.ensureInfra(useRouter);
|
||||
|
||||
// 7. Generate unique task queue and container name
|
||||
const suffix = randomSuffix();
|
||||
@@ -94,20 +95,20 @@ export async function start(args: StartArgs): Promise<void> {
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS = '/app/credentials/google-sa-key.json';
|
||||
}
|
||||
|
||||
// 10. Resolve output directory
|
||||
// 11. Resolve output directory
|
||||
const outputDir = args.output ? path.resolve(args.output) : undefined;
|
||||
if (outputDir) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// 11. Resolve prompts directory (local mode only)
|
||||
// 12. Resolve prompts directory (local mode only)
|
||||
const promptsDir = isLocal() ? path.resolve('apps/worker/prompts') : undefined;
|
||||
|
||||
// 12. Display splash screen
|
||||
// 13. Display splash screen
|
||||
displaySplash(isLocal() ? undefined : args.version);
|
||||
|
||||
// 13. Spawn worker container
|
||||
const proc = spawnWorker({
|
||||
// 14. Spawn worker via orchestrator
|
||||
const handle = orchestrator.spawnWorker({
|
||||
version: args.version,
|
||||
url: args.url,
|
||||
repo,
|
||||
@@ -123,8 +124,8 @@ export async function start(args: StartArgs): Promise<void> {
|
||||
...(args.pipelineTesting && { pipelineTesting: true }),
|
||||
});
|
||||
|
||||
// 14. Wait for workflow to register, then display info
|
||||
proc.on('error', (err) => {
|
||||
// 15. Wait for workflow to register, then display info
|
||||
handle.onError((err) => {
|
||||
console.error(`Failed to start worker: ${err.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -181,18 +182,14 @@ export async function start(args: StartArgs): Promise<void> {
|
||||
process.stdout.write('.');
|
||||
}, 2000);
|
||||
|
||||
// Stop the worker container only if it hasn't started yet
|
||||
// Stop the worker only if it hasn't started yet
|
||||
let cleaned = false;
|
||||
const cleanup = (): void => {
|
||||
if (cleaned || started) return;
|
||||
cleaned = true;
|
||||
clearInterval(pollInterval);
|
||||
console.log(`\nStopping worker ${containerName}...`);
|
||||
try {
|
||||
execFileSync('docker', ['stop', containerName], { stdio: 'pipe' });
|
||||
} catch {
|
||||
// Container may have already exited
|
||||
}
|
||||
handle.kill();
|
||||
};
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
* `shannon status` command — show running workers and Temporal health.
|
||||
*/
|
||||
|
||||
import { isTemporalReady, listRunningWorkers } from '../docker.js';
|
||||
import { getOrchestrator } from '../backend.js';
|
||||
|
||||
export async function status(): Promise<void> {
|
||||
const orchestrator = await getOrchestrator();
|
||||
|
||||
export function status(): void {
|
||||
// 1. Temporal health
|
||||
const temporalUp = isTemporalReady();
|
||||
const temporalUp = orchestrator.isTemporalReady();
|
||||
console.log(`Temporal: ${temporalUp ? 'running' : 'not running'}`);
|
||||
if (temporalUp) {
|
||||
console.log(' Web UI: http://localhost:8233');
|
||||
@@ -14,7 +16,7 @@ export function status(): void {
|
||||
console.log('');
|
||||
|
||||
// 2. Running workers
|
||||
const workers = listRunningWorkers();
|
||||
const workers = orchestrator.listRunningWorkers();
|
||||
if (workers) {
|
||||
console.log('Workers:');
|
||||
console.log(workers);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*/
|
||||
|
||||
import * as p from '@clack/prompts';
|
||||
import { stopInfra, stopWorkers } from '../docker.js';
|
||||
import { getOrchestrator } from '../backend.js';
|
||||
|
||||
export async function stop(clean: boolean): Promise<void> {
|
||||
if (clean) {
|
||||
@@ -16,6 +16,7 @@ export async function stop(clean: boolean): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
stopWorkers();
|
||||
stopInfra(clean);
|
||||
const orchestrator = await getOrchestrator();
|
||||
orchestrator.stopWorkers();
|
||||
orchestrator.stopInfra(clean);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import * as p from '@clack/prompts';
|
||||
import { stopInfra, stopWorkers } from '../docker.js';
|
||||
import { getOrchestrator } from '../backend.js';
|
||||
|
||||
const SHANNON_HOME = path.join(os.homedir(), '.shannon');
|
||||
|
||||
@@ -28,8 +28,9 @@ export async function uninstall(): Promise<void> {
|
||||
}
|
||||
|
||||
// Stop any running containers first
|
||||
stopWorkers();
|
||||
stopInfra(false);
|
||||
const orchestrator = await getOrchestrator();
|
||||
orchestrator.stopWorkers();
|
||||
orchestrator.stopInfra(false);
|
||||
|
||||
fs.rmSync(SHANNON_HOME, { recursive: true, force: true });
|
||||
p.log.success('All Shannon data has been removed.');
|
||||
|
||||
@@ -2,30 +2,19 @@
|
||||
* `shannon workspaces` command — list all workspaces.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import os from 'node:os';
|
||||
import { getWorkerImage } from '../docker.js';
|
||||
import { getOrchestrator } from '../backend.js';
|
||||
import { getWorkspacesDir } from '../home.js';
|
||||
|
||||
export function workspaces(version: string): void {
|
||||
export async function workspaces(version: string): Promise<void> {
|
||||
const orchestrator = await getOrchestrator();
|
||||
const workspacesDir = getWorkspacesDir();
|
||||
const image = getWorkerImage(version);
|
||||
const image = orchestrator.getWorkerImage(version);
|
||||
|
||||
try {
|
||||
execFileSync(
|
||||
'docker',
|
||||
[
|
||||
'run',
|
||||
'--rm',
|
||||
'-v',
|
||||
`${workspacesDir}:/app/workspaces`,
|
||||
'-e',
|
||||
'WORKSPACES_DIR=/app/workspaces',
|
||||
image,
|
||||
'node',
|
||||
'apps/worker/dist/temporal/workspaces.js',
|
||||
],
|
||||
{ stdio: 'inherit', ...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }) },
|
||||
orchestrator.runEphemeral(
|
||||
image,
|
||||
['node', 'apps/worker/dist/temporal/workspaces.js'],
|
||||
[`${workspacesDir}:/app/workspaces`],
|
||||
);
|
||||
} catch {
|
||||
console.error('ERROR: Failed to list workspaces. Is the Docker image available?');
|
||||
|
||||
+255
-245
@@ -12,27 +12,22 @@ import path from 'node:path';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { getMode } from './mode.js';
|
||||
import type { Orchestrator, WorkerHandle, WorkerOptions } from './orchestrator.js';
|
||||
|
||||
export type { WorkerOptions };
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const NPX_IMAGE_REPO = 'keygraph/shannon';
|
||||
const DEV_IMAGE = 'shannon-worker';
|
||||
|
||||
export function getWorkerImage(version: string): string {
|
||||
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
|
||||
}
|
||||
|
||||
function getComposeFile(): string {
|
||||
return getMode() === 'local'
|
||||
? path.resolve('docker-compose.yml')
|
||||
: path.resolve(__dirname, '..', 'infra', 'compose.yml');
|
||||
}
|
||||
|
||||
/** Generate an 8-char random hex suffix for container/queue names. */
|
||||
export function randomSuffix(): string {
|
||||
return crypto.randomBytes(4).toString('hex');
|
||||
}
|
||||
|
||||
// === Internal Helpers ===
|
||||
|
||||
/** Run a command silently, return true if it succeeds. */
|
||||
function runQuiet(cmd: string, args: string[]): boolean {
|
||||
try {
|
||||
@@ -52,21 +47,10 @@ function runOutput(cmd: string, args: string[]): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Temporal is running and healthy.
|
||||
*/
|
||||
export function isTemporalReady(): boolean {
|
||||
const output = runOutput('docker', [
|
||||
'exec',
|
||||
'shannon-temporal',
|
||||
'temporal',
|
||||
'operator',
|
||||
'cluster',
|
||||
'health',
|
||||
'--address',
|
||||
'localhost:7233',
|
||||
]);
|
||||
return output.includes('SERVING');
|
||||
function getComposeFile(): string {
|
||||
return getMode() === 'local'
|
||||
? path.resolve('docker-compose.yml')
|
||||
: path.resolve(__dirname, '..', 'infra', 'compose.yml');
|
||||
}
|
||||
|
||||
/** Check if the router container is running and healthy. */
|
||||
@@ -75,99 +59,6 @@ function isRouterReady(): boolean {
|
||||
return status === 'healthy';
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure Temporal (and optionally router) are running via compose.
|
||||
* If Temporal is already up but router is needed and missing, starts router only.
|
||||
*/
|
||||
export async function ensureInfra(useRouter: boolean): Promise<void> {
|
||||
const temporalReady = isTemporalReady();
|
||||
const routerNeeded = useRouter && !isRouterReady();
|
||||
|
||||
if (temporalReady && !routerNeeded) {
|
||||
return;
|
||||
}
|
||||
|
||||
const composeFile = getComposeFile();
|
||||
const composeArgs = ['compose', '-f', composeFile];
|
||||
if (useRouter) composeArgs.push('--profile', 'router');
|
||||
composeArgs.push('up', '-d');
|
||||
|
||||
if (temporalReady && routerNeeded) {
|
||||
console.log('Starting router...');
|
||||
} else {
|
||||
console.log('Starting Shannon infrastructure...');
|
||||
}
|
||||
execFileSync('docker', composeArgs, { stdio: 'inherit' });
|
||||
|
||||
// Wait for Temporal if it wasn't already running
|
||||
if (!temporalReady) {
|
||||
console.log('Waiting for Temporal to be ready...');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
if (isTemporalReady()) {
|
||||
console.log('Temporal is ready!');
|
||||
break;
|
||||
}
|
||||
if (i === 29) {
|
||||
console.error('Timeout waiting for Temporal');
|
||||
process.exit(1);
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for router if needed
|
||||
if (routerNeeded) {
|
||||
console.log('Waiting for router to be ready...');
|
||||
for (let i = 0; i < 15; i++) {
|
||||
if (isRouterReady()) {
|
||||
console.log('Router is ready!');
|
||||
return;
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
console.error('Timeout waiting for router');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the worker image locally (local mode only).
|
||||
*/
|
||||
export function buildImage(noCache: boolean): void {
|
||||
console.log(`Building ${DEV_IMAGE}...`);
|
||||
const args = ['build'];
|
||||
if (noCache) args.push('--no-cache');
|
||||
args.push('-t', DEV_IMAGE, '.');
|
||||
execFileSync('docker', args, { stdio: 'inherit' });
|
||||
console.log(`Build complete: ${DEV_IMAGE}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the worker image is available.
|
||||
* Local mode: auto-builds if missing. NPX mode: pulls from Docker Hub.
|
||||
*/
|
||||
export function ensureImage(version: string): void {
|
||||
const image = getWorkerImage(version);
|
||||
const exists = runQuiet('docker', ['image', 'inspect', image]);
|
||||
if (exists) return;
|
||||
|
||||
if (getMode() === 'local') {
|
||||
console.log('Worker image not found, building...');
|
||||
buildImage(false);
|
||||
} else {
|
||||
console.log(`Pulling ${image}...`);
|
||||
try {
|
||||
execFileSync('docker', ['pull', image], { stdio: 'inherit' });
|
||||
} catch {
|
||||
console.error(`\nERROR: Failed to pull ${image}`);
|
||||
console.error('The image may not be available for your platform yet.');
|
||||
console.error('Check https://hub.docker.com/r/keygraph/shannon for available tags.');
|
||||
process.exit(1);
|
||||
}
|
||||
pruneOldImages(version);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect if --add-host is needed (Linux without Podman).
|
||||
* macOS has host.docker.internal built in.
|
||||
@@ -182,140 +73,259 @@ function addHostFlag(): string[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
export interface WorkerOptions {
|
||||
version: string;
|
||||
url: string;
|
||||
repo: { hostPath: string; containerPath: string };
|
||||
workspacesDir: string;
|
||||
taskQueue: string;
|
||||
containerName: string;
|
||||
envFlags: string[];
|
||||
config?: { hostPath: string; containerPath: string };
|
||||
credentials?: string;
|
||||
promptsDir?: string;
|
||||
outputDir?: string;
|
||||
workspace: string;
|
||||
pipelineTesting?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn the worker container in detached mode and return the process.
|
||||
*/
|
||||
export function spawnWorker(opts: WorkerOptions): ChildProcess {
|
||||
const args = ['run', '-d', '--rm', '--name', opts.containerName, '--network', 'shannon-net'];
|
||||
|
||||
// Add host flag for Linux
|
||||
args.push(...addHostFlag());
|
||||
|
||||
// UID remapping for Linux bind mounts
|
||||
if (os.platform() === 'linux' && process.getuid && process.getgid) {
|
||||
args.push('-e', `SHANNON_HOST_UID=${process.getuid()}`, '-e', `SHANNON_HOST_GID=${process.getgid()}`);
|
||||
}
|
||||
|
||||
// Volume mounts
|
||||
args.push('-v', `${opts.workspacesDir}:/app/workspaces`);
|
||||
args.push('-v', `${opts.repo.hostPath}:${opts.repo.containerPath}:ro`);
|
||||
|
||||
// Writable overlays: shadow .shannon/ inside the :ro repo with workspace-backed dirs
|
||||
const workspacePath = path.join(opts.workspacesDir, opts.workspace);
|
||||
args.push('-v', `${path.join(workspacePath, 'deliverables')}:${opts.repo.containerPath}/.shannon/deliverables`);
|
||||
args.push('-v', `${path.join(workspacePath, 'scratchpad')}:${opts.repo.containerPath}/.shannon/scratchpad`);
|
||||
args.push('-v', `${path.join(workspacePath, '.playwright-cli')}:${opts.repo.containerPath}/.shannon/.playwright-cli`);
|
||||
|
||||
// Local mode: mount prompts for live editing
|
||||
if (opts.promptsDir) {
|
||||
args.push('-v', `${opts.promptsDir}:/app/apps/worker/prompts:ro`);
|
||||
}
|
||||
|
||||
if (opts.config) {
|
||||
args.push('-v', `${opts.config.hostPath}:${opts.config.containerPath}:ro`);
|
||||
}
|
||||
|
||||
// Output directory for deliverables copy
|
||||
if (opts.outputDir) {
|
||||
args.push('-v', `${opts.outputDir}:/app/output`);
|
||||
}
|
||||
|
||||
// Mount credentials file to fixed container path
|
||||
if (opts.credentials) {
|
||||
args.push('-v', `${opts.credentials}:/app/credentials/google-sa-key.json:ro`);
|
||||
}
|
||||
|
||||
// Environment
|
||||
args.push(...opts.envFlags);
|
||||
|
||||
// Container settings
|
||||
args.push('--shm-size', '2gb', '--security-opt', 'seccomp=unconfined');
|
||||
|
||||
// Image
|
||||
args.push(getWorkerImage(opts.version));
|
||||
|
||||
// Worker command
|
||||
args.push('node', 'apps/worker/dist/temporal/worker.js', opts.url, opts.repo.containerPath);
|
||||
args.push('--task-queue', opts.taskQueue);
|
||||
if (opts.config) {
|
||||
args.push('--config', opts.config.containerPath);
|
||||
}
|
||||
if (opts.outputDir) {
|
||||
args.push('--output', '/app/output');
|
||||
}
|
||||
args.push('--workspace', opts.workspace);
|
||||
if (opts.pipelineTesting) {
|
||||
args.push('--pipeline-testing');
|
||||
}
|
||||
|
||||
// Prevent MSYS/Git Bash from converting Unix paths (e.g. /repos/my-repo) to Windows paths
|
||||
return spawn('docker', args, {
|
||||
stdio: 'pipe',
|
||||
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop all running shannon-worker-* containers.
|
||||
*/
|
||||
export function stopWorkers(): void {
|
||||
const workers = runOutput('docker', ['ps', '-q', '--filter', 'name=shannon-worker-']);
|
||||
if (!workers) return;
|
||||
|
||||
const ids = workers.split('\n').filter(Boolean);
|
||||
console.log('Stopping worker containers...');
|
||||
execFileSync('docker', ['stop', ...ids], { stdio: 'inherit' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Tear down the compose stack.
|
||||
*/
|
||||
export function stopInfra(clean: boolean): void {
|
||||
const composeFile = getComposeFile();
|
||||
const args = ['compose', '-f', composeFile, '--profile', 'router', 'down'];
|
||||
if (clean) args.push('-v');
|
||||
execFileSync('docker', args, { stdio: 'inherit' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove old keygraph/shannon images that don't match the current version.
|
||||
*/
|
||||
/** Remove old keygraph/shannon images that don't match the current version. */
|
||||
function pruneOldImages(currentVersion: string): void {
|
||||
const output = runOutput('docker', ['images', NPX_IMAGE_REPO, '--format', '{{.Tag}}']);
|
||||
if (!output) return;
|
||||
|
||||
const currentTag = currentVersion;
|
||||
const stale = output.split('\n').filter((tag) => tag && tag !== currentTag);
|
||||
const stale = output.split('\n').filter((tag) => tag && tag !== currentVersion);
|
||||
for (const tag of stale) {
|
||||
runQuiet('docker', ['rmi', `${NPX_IMAGE_REPO}:${tag}`]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List running worker containers.
|
||||
*/
|
||||
export function listRunningWorkers(): string {
|
||||
return runOutput('docker', [
|
||||
'ps',
|
||||
'--filter',
|
||||
'name=shannon-worker-',
|
||||
'--format',
|
||||
'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}',
|
||||
]);
|
||||
// === DockerOrchestrator ===
|
||||
|
||||
/** Docker-based orchestration backend. */
|
||||
export class DockerOrchestrator implements Orchestrator {
|
||||
getWorkerImage(version: string): string {
|
||||
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
|
||||
}
|
||||
|
||||
isTemporalReady(): boolean {
|
||||
const output = runOutput('docker', [
|
||||
'exec',
|
||||
'shannon-temporal',
|
||||
'temporal',
|
||||
'operator',
|
||||
'cluster',
|
||||
'health',
|
||||
'--address',
|
||||
'localhost:7233',
|
||||
]);
|
||||
return output.includes('SERVING');
|
||||
}
|
||||
|
||||
async ensureInfra(useRouter: boolean): Promise<void> {
|
||||
const temporalReady = this.isTemporalReady();
|
||||
const routerNeeded = useRouter && !isRouterReady();
|
||||
|
||||
if (temporalReady && !routerNeeded) {
|
||||
return;
|
||||
}
|
||||
|
||||
const composeFile = getComposeFile();
|
||||
const composeArgs = ['compose', '-f', composeFile];
|
||||
if (useRouter) composeArgs.push('--profile', 'router');
|
||||
composeArgs.push('up', '-d');
|
||||
|
||||
if (temporalReady && routerNeeded) {
|
||||
console.log('Starting router...');
|
||||
} else {
|
||||
console.log('Starting Shannon infrastructure...');
|
||||
}
|
||||
execFileSync('docker', composeArgs, { stdio: 'inherit' });
|
||||
|
||||
// Wait for Temporal if it wasn't already running
|
||||
if (!temporalReady) {
|
||||
console.log('Waiting for Temporal to be ready...');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
if (this.isTemporalReady()) {
|
||||
console.log('Temporal is ready!');
|
||||
break;
|
||||
}
|
||||
if (i === 29) {
|
||||
console.error('Timeout waiting for Temporal');
|
||||
process.exit(1);
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for router if needed
|
||||
if (routerNeeded) {
|
||||
console.log('Waiting for router to be ready...');
|
||||
for (let i = 0; i < 15; i++) {
|
||||
if (isRouterReady()) {
|
||||
console.log('Router is ready!');
|
||||
return;
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
console.error('Timeout waiting for router');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
ensureImage(version: string): void {
|
||||
const image = this.getWorkerImage(version);
|
||||
const exists = runQuiet('docker', ['image', 'inspect', image]);
|
||||
if (exists) return;
|
||||
|
||||
if (getMode() === 'local') {
|
||||
console.log('Worker image not found, building...');
|
||||
this.buildImage(false);
|
||||
} else {
|
||||
console.log(`Pulling ${image}...`);
|
||||
try {
|
||||
execFileSync('docker', ['pull', image], { stdio: 'inherit' });
|
||||
} catch {
|
||||
console.error(`\nERROR: Failed to pull ${image}`);
|
||||
console.error('The image may not be available for your platform yet.');
|
||||
console.error('Check https://hub.docker.com/r/keygraph/shannon for available tags.');
|
||||
process.exit(1);
|
||||
}
|
||||
pruneOldImages(version);
|
||||
}
|
||||
}
|
||||
|
||||
spawnWorker(opts: WorkerOptions): WorkerHandle {
|
||||
const args = ['run', '-d', '--rm', '--name', opts.containerName, '--network', 'shannon-net'];
|
||||
|
||||
// Add host flag for Linux
|
||||
args.push(...addHostFlag());
|
||||
|
||||
// UID remapping for Linux bind mounts
|
||||
if (os.platform() === 'linux' && process.getuid && process.getgid) {
|
||||
args.push('-e', `SHANNON_HOST_UID=${process.getuid()}`, '-e', `SHANNON_HOST_GID=${process.getgid()}`);
|
||||
}
|
||||
|
||||
// Volume mounts
|
||||
args.push('-v', `${opts.workspacesDir}:/app/workspaces`);
|
||||
args.push('-v', `${opts.repo.hostPath}:${opts.repo.containerPath}:ro`);
|
||||
|
||||
// Writable overlays: shadow .shannon/ inside the :ro repo with workspace-backed dirs
|
||||
const workspacePath = path.join(opts.workspacesDir, opts.workspace);
|
||||
args.push('-v', `${path.join(workspacePath, 'deliverables')}:${opts.repo.containerPath}/.shannon/deliverables`);
|
||||
args.push('-v', `${path.join(workspacePath, 'scratchpad')}:${opts.repo.containerPath}/.shannon/scratchpad`);
|
||||
args.push(
|
||||
'-v',
|
||||
`${path.join(workspacePath, '.playwright-cli')}:${opts.repo.containerPath}/.shannon/.playwright-cli`,
|
||||
);
|
||||
|
||||
// Local mode: mount prompts for live editing
|
||||
if (opts.promptsDir) {
|
||||
args.push('-v', `${opts.promptsDir}:/app/apps/worker/prompts:ro`);
|
||||
}
|
||||
|
||||
if (opts.config) {
|
||||
args.push('-v', `${opts.config.hostPath}:${opts.config.containerPath}:ro`);
|
||||
}
|
||||
|
||||
// Output directory for deliverables copy
|
||||
if (opts.outputDir) {
|
||||
args.push('-v', `${opts.outputDir}:/app/output`);
|
||||
}
|
||||
|
||||
// Mount credentials file to fixed container path
|
||||
if (opts.credentials) {
|
||||
args.push('-v', `${opts.credentials}:/app/credentials/google-sa-key.json:ro`);
|
||||
}
|
||||
|
||||
// Environment
|
||||
args.push(...opts.envFlags);
|
||||
|
||||
// Container settings
|
||||
args.push('--shm-size', '2gb', '--security-opt', 'seccomp=unconfined');
|
||||
|
||||
// Image
|
||||
args.push(this.getWorkerImage(opts.version));
|
||||
|
||||
// Worker command
|
||||
args.push('node', 'apps/worker/dist/temporal/worker.js', opts.url, opts.repo.containerPath);
|
||||
args.push('--task-queue', opts.taskQueue);
|
||||
if (opts.config) {
|
||||
args.push('--config', opts.config.containerPath);
|
||||
}
|
||||
if (opts.outputDir) {
|
||||
args.push('--output', '/app/output');
|
||||
}
|
||||
args.push('--workspace', opts.workspace);
|
||||
if (opts.pipelineTesting) {
|
||||
args.push('--pipeline-testing');
|
||||
}
|
||||
|
||||
// Prevent MSYS/Git Bash from converting Unix paths (e.g. /repos/my-repo) to Windows paths
|
||||
const proc = spawn('docker', args, {
|
||||
stdio: 'pipe',
|
||||
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
||||
});
|
||||
|
||||
return new DockerWorkerHandle(proc, opts.containerName);
|
||||
}
|
||||
|
||||
stopWorkers(): void {
|
||||
const workers = runOutput('docker', ['ps', '-q', '--filter', 'name=shannon-worker-']);
|
||||
if (!workers) return;
|
||||
|
||||
const ids = workers.split('\n').filter(Boolean);
|
||||
console.log('Stopping worker containers...');
|
||||
execFileSync('docker', ['stop', ...ids], { stdio: 'inherit' });
|
||||
}
|
||||
|
||||
stopInfra(clean: boolean): void {
|
||||
const composeFile = getComposeFile();
|
||||
const args = ['compose', '-f', composeFile, '--profile', 'router', 'down'];
|
||||
if (clean) args.push('-v');
|
||||
execFileSync('docker', args, { stdio: 'inherit' });
|
||||
}
|
||||
|
||||
listRunningWorkers(): string {
|
||||
return runOutput('docker', [
|
||||
'ps',
|
||||
'--filter',
|
||||
'name=shannon-worker-',
|
||||
'--format',
|
||||
'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}',
|
||||
]);
|
||||
}
|
||||
|
||||
runEphemeral(image: string, args: string[], mounts: string[]): void {
|
||||
const dockerArgs = ['run', '--rm'];
|
||||
for (const mount of mounts) {
|
||||
dockerArgs.push('-v', mount);
|
||||
}
|
||||
dockerArgs.push(image, ...args);
|
||||
execFileSync('docker', dockerArgs, {
|
||||
stdio: 'inherit',
|
||||
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
||||
});
|
||||
}
|
||||
|
||||
/** Build the worker image locally (local mode only). */
|
||||
buildImage(noCache: boolean): void {
|
||||
console.log(`Building ${DEV_IMAGE}...`);
|
||||
const args = ['build'];
|
||||
if (noCache) args.push('--no-cache');
|
||||
args.push('-t', DEV_IMAGE, '.');
|
||||
execFileSync('docker', args, { stdio: 'inherit' });
|
||||
console.log(`Build complete: ${DEV_IMAGE}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** WorkerHandle wrapping a Docker container's ChildProcess. */
|
||||
class DockerWorkerHandle implements WorkerHandle {
|
||||
constructor(
|
||||
private readonly proc: ChildProcess,
|
||||
private readonly containerName: string,
|
||||
) {}
|
||||
|
||||
onError(cb: (err: Error) => void): void {
|
||||
this.proc.on('error', cb);
|
||||
}
|
||||
|
||||
kill(): void {
|
||||
try {
|
||||
execFileSync('docker', ['stop', this.containerName], { stdio: 'pipe' });
|
||||
} catch {
|
||||
// Container may have already exited
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// === Backward-compatible exports ===
|
||||
|
||||
// NOTE: Used by commands/build.ts which doesn't go through the orchestrator
|
||||
export function buildImage(noCache: boolean): void {
|
||||
new DockerOrchestrator().buildImage(noCache);
|
||||
}
|
||||
|
||||
+18
-1
@@ -10,7 +10,7 @@ import { resolveConfig } from './config/resolver.js';
|
||||
import { getMode } from './mode.js';
|
||||
|
||||
/** Environment variables forwarded to worker containers. */
|
||||
const FORWARD_VARS = [
|
||||
export const FORWARD_VARS = [
|
||||
'ANTHROPIC_API_KEY',
|
||||
'ANTHROPIC_BASE_URL',
|
||||
'ANTHROPIC_AUTH_TOKEN',
|
||||
@@ -61,6 +61,23 @@ export function buildEnvFlags(): string[] {
|
||||
return flags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a key-value record of env vars to forward to workers.
|
||||
* Used by the K8s backend to create Secrets instead of Docker `-e` flags.
|
||||
*/
|
||||
export function buildEnvRecord(): Record<string, string> {
|
||||
const env: Record<string, string> = { TEMPORAL_ADDRESS: 'shannon-temporal:7233' };
|
||||
|
||||
for (const key of FORWARD_VARS) {
|
||||
const value = process.env[key];
|
||||
if (value) {
|
||||
env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
interface CredentialValidation {
|
||||
valid: boolean;
|
||||
error?: string;
|
||||
|
||||
+16
-2
@@ -12,6 +12,7 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { setBackend } from './backend.js';
|
||||
import { build } from './commands/build.js';
|
||||
import { logs } from './commands/logs.js';
|
||||
import { setup } from './commands/setup.js';
|
||||
@@ -179,6 +180,19 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
|
||||
// === Main Dispatch ===
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
// Parse --backend flag before command dispatch
|
||||
const backendIdx = args.indexOf('--backend');
|
||||
if (backendIdx !== -1) {
|
||||
const backendVal = args[backendIdx + 1];
|
||||
if (backendVal === 'k8s' || backendVal === 'kubernetes') {
|
||||
setBackend('k8s');
|
||||
} else if (backendVal === 'docker') {
|
||||
setBackend('docker');
|
||||
}
|
||||
args.splice(backendIdx, 2);
|
||||
}
|
||||
|
||||
const command = args[0];
|
||||
|
||||
switch (command) {
|
||||
@@ -201,10 +215,10 @@ switch (command) {
|
||||
break;
|
||||
}
|
||||
case 'workspaces':
|
||||
workspaces(getVersion());
|
||||
await workspaces(getVersion());
|
||||
break;
|
||||
case 'status':
|
||||
status();
|
||||
await status();
|
||||
break;
|
||||
case 'setup':
|
||||
if (getMode() === 'local') {
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
/**
|
||||
* Kubernetes orchestration backend.
|
||||
*
|
||||
* Replaces Docker CLI commands with Kubernetes API calls:
|
||||
* - `docker compose up` → apply Deployments, Services, PVCs
|
||||
* - `docker run --rm` → K8s Job per scan
|
||||
* - `docker stop` → delete Jobs
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import { buildEnvRecord } from './env.js';
|
||||
import { getMode } from './mode.js';
|
||||
import type { Orchestrator, WorkerHandle, WorkerOptions } from './orchestrator.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const NAMESPACE = 'shannon';
|
||||
const NPX_IMAGE_REPO = 'keygraph/shannon';
|
||||
const DEV_IMAGE = 'shannon-worker';
|
||||
const WORKER_LABEL = 'shannon-worker';
|
||||
const K8S_MANIFESTS_DIR = path.resolve(__dirname, '..', 'infra', 'k8s');
|
||||
|
||||
// === K8s Client Setup ===
|
||||
|
||||
function loadKubeConfig(): k8s.KubeConfig {
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromDefault();
|
||||
return kc;
|
||||
}
|
||||
|
||||
/** Detect if running on kind or minikube (local K8s). */
|
||||
function isLocalCluster(kc: k8s.KubeConfig): boolean {
|
||||
const context = kc.getCurrentContext();
|
||||
return context.startsWith('kind-') || context === 'minikube' || context.startsWith('minikube');
|
||||
}
|
||||
|
||||
// === K8sOrchestrator ===
|
||||
|
||||
/** Kubernetes-based orchestration backend. */
|
||||
export class K8sOrchestrator implements Orchestrator {
|
||||
private readonly kc: k8s.KubeConfig;
|
||||
private readonly coreApi: k8s.CoreV1Api;
|
||||
private readonly appsApi: k8s.AppsV1Api;
|
||||
private readonly batchApi: k8s.BatchV1Api;
|
||||
|
||||
constructor() {
|
||||
this.kc = loadKubeConfig();
|
||||
this.coreApi = this.kc.makeApiClient(k8s.CoreV1Api);
|
||||
this.appsApi = this.kc.makeApiClient(k8s.AppsV1Api);
|
||||
this.batchApi = this.kc.makeApiClient(k8s.BatchV1Api);
|
||||
}
|
||||
|
||||
getWorkerImage(version: string): string {
|
||||
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
|
||||
}
|
||||
|
||||
// === Infrastructure ===
|
||||
|
||||
async ensureInfra(useRouter: boolean): Promise<void> {
|
||||
// 1. Create namespace if it doesn't exist
|
||||
await this.ensureNamespace();
|
||||
|
||||
// 2. Create or update credentials secret
|
||||
await this.ensureCredentialsSecret();
|
||||
|
||||
// 3. Apply Temporal manifests
|
||||
await this.applyManifest('temporal.yaml');
|
||||
|
||||
// 4. Apply workspaces PVC
|
||||
await this.applyManifest('workspaces-pvc.yaml');
|
||||
|
||||
// 5. Optionally apply router
|
||||
if (useRouter) {
|
||||
await this.applyManifest('router.yaml');
|
||||
}
|
||||
|
||||
// 6. Wait for Temporal to be ready
|
||||
if (!(await this.isTemporalReadyAsync())) {
|
||||
console.log('Waiting for Temporal to be ready...');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
if (await this.isTemporalReadyAsync()) {
|
||||
console.log('Temporal is ready!');
|
||||
break;
|
||||
}
|
||||
if (i === 29) {
|
||||
console.error('Timeout waiting for Temporal');
|
||||
process.exit(1);
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ensureImage(_version: string): void {
|
||||
// K8s pulls images via imagePullPolicy — no-op for remote clusters.
|
||||
// For kind, users must run `kind load docker-image shannon-worker` manually.
|
||||
if (getMode() === 'local' && isLocalCluster(this.kc)) {
|
||||
console.log('NOTE: For kind/minikube, ensure the worker image is loaded:');
|
||||
console.log(' kind load docker-image shannon-worker');
|
||||
}
|
||||
}
|
||||
|
||||
isTemporalReady(): boolean {
|
||||
// K8s API is async — synchronous check returns false, ensureInfra uses async polling
|
||||
return false;
|
||||
}
|
||||
|
||||
private async isTemporalReadyAsync(): Promise<boolean> {
|
||||
try {
|
||||
const response = await this.coreApi.listNamespacedPod({
|
||||
namespace: NAMESPACE,
|
||||
labelSelector: 'app=shannon-temporal',
|
||||
});
|
||||
return response.items.some((pod) => {
|
||||
const conditions = pod.status?.conditions ?? [];
|
||||
return conditions.some((c) => c.type === 'Ready' && c.status === 'True');
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// === Worker Lifecycle ===
|
||||
|
||||
spawnWorker(opts: WorkerOptions): WorkerHandle {
|
||||
const image = this.getWorkerImage(opts.version);
|
||||
const jobName = opts.containerName;
|
||||
|
||||
// Build command + args for the worker
|
||||
const command = ['node', 'apps/worker/dist/temporal/worker.js', opts.url, opts.repo.containerPath];
|
||||
const args: string[] = ['--task-queue', opts.taskQueue, '--workspace', opts.workspace];
|
||||
if (opts.config) {
|
||||
args.push('--config', opts.config.containerPath);
|
||||
}
|
||||
if (opts.outputDir) {
|
||||
args.push('--output', '/app/output');
|
||||
}
|
||||
if (opts.pipelineTesting) {
|
||||
args.push('--pipeline-testing');
|
||||
}
|
||||
|
||||
// Build volume mounts and volumes
|
||||
const volumeMounts: k8s.V1VolumeMount[] = [
|
||||
{ name: 'workspaces', mountPath: '/app/workspaces' },
|
||||
{ name: 'shm', mountPath: '/dev/shm' },
|
||||
];
|
||||
const volumes: k8s.V1Volume[] = [
|
||||
{
|
||||
name: 'workspaces',
|
||||
persistentVolumeClaim: { claimName: 'shannon-workspaces' },
|
||||
},
|
||||
{
|
||||
name: 'shm',
|
||||
emptyDir: { medium: 'Memory', sizeLimit: '2Gi' },
|
||||
},
|
||||
];
|
||||
|
||||
// Repo volume — hostPath for local clusters, PVC for managed
|
||||
if (isLocalCluster(this.kc)) {
|
||||
volumes.push({
|
||||
name: 'repo',
|
||||
hostPath: { path: opts.repo.hostPath, type: 'Directory' },
|
||||
});
|
||||
} else {
|
||||
volumes.push({
|
||||
name: 'repo',
|
||||
persistentVolumeClaim: { claimName: `shannon-repo-${jobName}` },
|
||||
});
|
||||
}
|
||||
volumeMounts.push({
|
||||
name: 'repo',
|
||||
mountPath: opts.repo.containerPath,
|
||||
readOnly: true,
|
||||
});
|
||||
|
||||
// Overlay dirs for deliverables/scratchpad/playwright (writable areas over :ro repo)
|
||||
for (const overlay of ['deliverables', 'scratchpad', '.playwright-cli']) {
|
||||
const volName = `overlay-${overlay.replace('.', '')}`;
|
||||
volumes.push({
|
||||
name: volName,
|
||||
emptyDir: {},
|
||||
});
|
||||
volumeMounts.push({
|
||||
name: volName,
|
||||
mountPath: `${opts.repo.containerPath}/.shannon/${overlay}`,
|
||||
});
|
||||
}
|
||||
|
||||
// Optional volume mounts
|
||||
if (opts.config) {
|
||||
// Config would need a ConfigMap — for now, pass via env or mount differently
|
||||
}
|
||||
|
||||
// Build env vars from the secret + TEMPORAL_ADDRESS
|
||||
const env: k8s.V1EnvVar[] = [{ name: 'TEMPORAL_ADDRESS', value: 'shannon-temporal:7233' }];
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: {
|
||||
name: jobName,
|
||||
namespace: NAMESPACE,
|
||||
labels: {
|
||||
app: WORKER_LABEL,
|
||||
'shannon.io/workspace': opts.workspace,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
ttlSecondsAfterFinished: 3600,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
app: WORKER_LABEL,
|
||||
'shannon.io/workspace': opts.workspace,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
securityContext: {
|
||||
seccompProfile: { type: 'Unconfined' },
|
||||
},
|
||||
containers: [
|
||||
{
|
||||
name: 'worker',
|
||||
image,
|
||||
command,
|
||||
args,
|
||||
env,
|
||||
envFrom: [{ secretRef: { name: 'shannon-credentials' } }],
|
||||
volumeMounts,
|
||||
resources: {
|
||||
requests: { memory: '2Gi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
volumes,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Create the Job asynchronously — errors are reported via the handle
|
||||
const createPromise = this.batchApi.createNamespacedJob({ namespace: NAMESPACE, body: job }).then(() => {
|
||||
console.log(`Worker job ${jobName} created in namespace ${NAMESPACE}`);
|
||||
});
|
||||
|
||||
return new K8sWorkerHandle(jobName, this.batchApi, createPromise);
|
||||
}
|
||||
|
||||
stopWorkers(): void {
|
||||
// Delete all worker jobs — fire and forget
|
||||
this.batchApi
|
||||
.deleteCollectionNamespacedJob({
|
||||
namespace: NAMESPACE,
|
||||
labelSelector: `app=${WORKER_LABEL}`,
|
||||
propagationPolicy: 'Background',
|
||||
})
|
||||
.then(() => {
|
||||
console.log('Worker jobs deleted.');
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`Failed to stop workers: ${message}`);
|
||||
});
|
||||
}
|
||||
|
||||
stopInfra(clean: boolean): void {
|
||||
if (clean) {
|
||||
// Delete the entire namespace (removes everything)
|
||||
this.coreApi
|
||||
.deleteNamespace({ name: NAMESPACE })
|
||||
.then(() => {
|
||||
console.log(`Namespace ${NAMESPACE} deleted.`);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`Failed to delete namespace: ${message}`);
|
||||
});
|
||||
} else {
|
||||
// Just delete the Temporal deployment and services
|
||||
this.appsApi.deleteNamespacedDeployment({ name: 'shannon-temporal', namespace: NAMESPACE }).catch(() => {});
|
||||
this.coreApi.deleteNamespacedService({ name: 'shannon-temporal', namespace: NAMESPACE }).catch(() => {});
|
||||
this.appsApi.deleteNamespacedDeployment({ name: 'shannon-router', namespace: NAMESPACE }).catch(() => {});
|
||||
this.coreApi.deleteNamespacedService({ name: 'shannon-router', namespace: NAMESPACE }).catch(() => {});
|
||||
console.log('Infrastructure resources deleted.');
|
||||
}
|
||||
}
|
||||
|
||||
listRunningWorkers(): string {
|
||||
// This is called synchronously by the status command — return empty for now,
|
||||
// actual implementation needs async refactor of the status command
|
||||
return '';
|
||||
}
|
||||
|
||||
runEphemeral(image: string, args: string[], mounts: string[]): void {
|
||||
// For K8s, run an ephemeral pod and wait for completion
|
||||
const podName = `shannon-ephemeral-${Date.now()}`;
|
||||
|
||||
const volumeMounts: k8s.V1VolumeMount[] = [];
|
||||
const volumes: k8s.V1Volume[] = [];
|
||||
|
||||
// Parse Docker-style mount strings (src:dst)
|
||||
for (let i = 0; i < mounts.length; i++) {
|
||||
const mount = mounts[i];
|
||||
if (!mount) continue;
|
||||
const parts = mount.split(':');
|
||||
const dst = parts[1];
|
||||
if (parts.length >= 2 && dst) {
|
||||
const volName = `vol-${i}`;
|
||||
volumeMounts.push({ name: volName, mountPath: dst });
|
||||
volumes.push({
|
||||
name: volName,
|
||||
persistentVolumeClaim: { claimName: 'shannon-workspaces' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const pod: k8s.V1Pod = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: {
|
||||
name: podName,
|
||||
namespace: NAMESPACE,
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [
|
||||
{
|
||||
name: 'ephemeral',
|
||||
image,
|
||||
command: args,
|
||||
volumeMounts,
|
||||
env: [{ name: 'WORKSPACES_DIR', value: '/app/workspaces' }],
|
||||
},
|
||||
],
|
||||
volumes,
|
||||
},
|
||||
};
|
||||
|
||||
// Create pod and wait for completion
|
||||
this.coreApi
|
||||
.createNamespacedPod({ namespace: NAMESPACE, body: pod })
|
||||
.then(async () => {
|
||||
// Poll for completion
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const status = await this.coreApi.readNamespacedPod({ name: podName, namespace: NAMESPACE });
|
||||
if (status.status?.phase === 'Succeeded' || status.status?.phase === 'Failed') {
|
||||
// Read logs
|
||||
const log = await this.coreApi.readNamespacedPodLog({ name: podName, namespace: NAMESPACE });
|
||||
console.log(log);
|
||||
// Clean up
|
||||
await this.coreApi.deleteNamespacedPod({ name: podName, namespace: NAMESPACE });
|
||||
return;
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
console.error('Timeout waiting for ephemeral pod');
|
||||
await this.coreApi.deleteNamespacedPod({ name: podName, namespace: NAMESPACE });
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`Failed to run ephemeral pod: ${message}`);
|
||||
});
|
||||
}
|
||||
|
||||
// === Private Helpers ===
|
||||
|
||||
private async ensureNamespace(): Promise<void> {
|
||||
try {
|
||||
await this.coreApi.readNamespace({ name: NAMESPACE });
|
||||
} catch {
|
||||
console.log(`Creating namespace ${NAMESPACE}...`);
|
||||
await this.coreApi.createNamespace({
|
||||
body: {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Namespace',
|
||||
metadata: { name: NAMESPACE, labels: { 'app.kubernetes.io/part-of': 'shannon' } },
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureCredentialsSecret(): Promise<void> {
|
||||
const envRecord = buildEnvRecord();
|
||||
const stringData: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(envRecord)) {
|
||||
if (key !== 'TEMPORAL_ADDRESS') {
|
||||
stringData[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const secret: k8s.V1Secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: 'shannon-credentials',
|
||||
namespace: NAMESPACE,
|
||||
},
|
||||
stringData,
|
||||
};
|
||||
|
||||
try {
|
||||
await this.coreApi.replaceNamespacedSecret({
|
||||
name: 'shannon-credentials',
|
||||
namespace: NAMESPACE,
|
||||
body: secret,
|
||||
});
|
||||
} catch {
|
||||
await this.coreApi.createNamespacedSecret({ namespace: NAMESPACE, body: secret });
|
||||
}
|
||||
}
|
||||
|
||||
private async applyManifest(filename: string): Promise<void> {
|
||||
const manifestPath = path.join(K8S_MANIFESTS_DIR, filename);
|
||||
const content = fs.readFileSync(manifestPath, 'utf-8');
|
||||
|
||||
// Split multi-document YAML
|
||||
const docs = content.split(/^---$/m).filter((doc) => doc.trim());
|
||||
|
||||
for (const doc of docs) {
|
||||
await this.applyResource(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private async applyResource(yamlDoc: string): Promise<void> {
|
||||
const objects = k8s.loadAllYaml(yamlDoc) as k8s.KubernetesObject[];
|
||||
const objectApi = k8s.KubernetesObjectApi.makeApiClient(this.kc);
|
||||
|
||||
for (const obj of objects) {
|
||||
if (!obj || !obj.kind || !obj.metadata?.name) continue;
|
||||
|
||||
// Ensure metadata has required fields for the typed API
|
||||
const spec = {
|
||||
...obj,
|
||||
metadata: { ...obj.metadata, name: obj.metadata.name },
|
||||
};
|
||||
|
||||
try {
|
||||
await objectApi.read(spec);
|
||||
await objectApi.patch(spec);
|
||||
} catch {
|
||||
try {
|
||||
await objectApi.create(spec);
|
||||
} catch (createErr: unknown) {
|
||||
const message = createErr instanceof Error ? createErr.message : String(createErr);
|
||||
console.error(`Failed to apply ${obj.kind}/${obj.metadata.name}: ${message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// === K8sWorkerHandle ===
|
||||
|
||||
/** WorkerHandle wrapping a K8s Job. */
|
||||
class K8sWorkerHandle implements WorkerHandle {
|
||||
private errorCallback: ((err: Error) => void) | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly jobName: string,
|
||||
private readonly batchApi: k8s.BatchV1Api,
|
||||
createPromise: Promise<void>,
|
||||
) {
|
||||
// Wire up creation errors to the error callback
|
||||
createPromise.catch((err: unknown) => {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (this.errorCallback) {
|
||||
this.errorCallback(error);
|
||||
} else {
|
||||
console.error(`Worker job creation failed: ${error.message}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onError(cb: (err: Error) => void): void {
|
||||
this.errorCallback = cb;
|
||||
}
|
||||
|
||||
kill(): void {
|
||||
this.batchApi
|
||||
.deleteNamespacedJob({
|
||||
name: this.jobName,
|
||||
namespace: NAMESPACE,
|
||||
propagationPolicy: 'Background',
|
||||
})
|
||||
.catch(() => {
|
||||
// Job may have already completed
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Orchestrator interface — abstraction over container orchestration backends.
|
||||
*
|
||||
* Docker and Kubernetes implement this interface so the CLI commands
|
||||
* can swap backends without changing their logic.
|
||||
*/
|
||||
|
||||
export interface WorkerOptions {
|
||||
version: string;
|
||||
url: string;
|
||||
repo: { hostPath: string; containerPath: string };
|
||||
workspacesDir: string;
|
||||
taskQueue: string;
|
||||
containerName: string;
|
||||
envFlags: string[];
|
||||
config?: { hostPath: string; containerPath: string };
|
||||
credentials?: string;
|
||||
promptsDir?: string;
|
||||
outputDir?: string;
|
||||
workspace: string;
|
||||
pipelineTesting?: boolean;
|
||||
}
|
||||
|
||||
/** Handle to a running worker, returned by Orchestrator.spawnWorker(). */
|
||||
export interface WorkerHandle {
|
||||
onError(cb: (err: Error) => void): void;
|
||||
kill(): void;
|
||||
}
|
||||
|
||||
/** Container orchestration backend. */
|
||||
export interface Orchestrator {
|
||||
ensureInfra(useRouter: boolean): Promise<void>;
|
||||
ensureImage(version: string): void;
|
||||
spawnWorker(opts: WorkerOptions): WorkerHandle;
|
||||
stopWorkers(): void;
|
||||
stopInfra(clean: boolean): void;
|
||||
listRunningWorkers(): string;
|
||||
isTemporalReady(): boolean;
|
||||
getWorkerImage(version: string): string;
|
||||
|
||||
/**
|
||||
* Run a one-shot ephemeral container and inherit stdio.
|
||||
* Used by commands like `workspaces` that need to run worker-side scripts.
|
||||
*/
|
||||
runEphemeral(image: string, args: string[], mounts: string[]): void;
|
||||
}
|
||||
Reference in New Issue
Block a user