backport: provider extensions and drop claude-code-router mode
Cherry-pick of KeygraphHQ/shannon#295 (581c208). Upstream changes: removes router mode from CLI/worker, adds provider extensions, new report-output-provider and checkpoint-provider interfaces, refactored workflow orchestration. Conflicts resolved: kept our README.md, CLAUDE.md, and deleted compose files. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+3
-23
@@ -5,7 +5,7 @@
|
|||||||
CLAUDE_CODE_MAX_OUTPUT_TOKENS=64000
|
CLAUDE_CODE_MAX_OUTPUT_TOKENS=64000
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# OPTION 1: Direct Anthropic (default, no router)
|
# OPTION 1: Direct Anthropic
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
ANTHROPIC_API_KEY=your-api-key-here
|
ANTHROPIC_API_KEY=your-api-key-here
|
||||||
|
|
||||||
@@ -19,20 +19,6 @@ ANTHROPIC_API_KEY=your-api-key-here
|
|||||||
# ANTHROPIC_BASE_URL=https://your-proxy.example.com
|
# ANTHROPIC_BASE_URL=https://your-proxy.example.com
|
||||||
# ANTHROPIC_AUTH_TOKEN=your-auth-token # Auth token for the custom endpoint
|
# ANTHROPIC_AUTH_TOKEN=your-auth-token # Auth token for the custom endpoint
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# OPTION 3: Router Mode (use alternative providers)
|
|
||||||
# =============================================================================
|
|
||||||
# Enable router mode by running: ./shannon start ... ROUTER=true
|
|
||||||
# Then configure ONE of the providers below:
|
|
||||||
|
|
||||||
# --- OpenAI ---
|
|
||||||
# OPENAI_API_KEY=sk-your-openai-key
|
|
||||||
# ROUTER_DEFAULT=openai,gpt-5.2
|
|
||||||
|
|
||||||
# --- OpenRouter (access Gemini 3 models via single API) ---
|
|
||||||
# OPENROUTER_API_KEY=sk-or-your-openrouter-key
|
|
||||||
# ROUTER_DEFAULT=openrouter,google/gemini-3-flash-preview
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Model Tier Overrides (Anthropic API / OAuth / Custom Base URL / Bedrock)
|
# Model Tier Overrides (Anthropic API / OAuth / Custom Base URL / Bedrock)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -43,7 +29,7 @@ ANTHROPIC_API_KEY=your-api-key-here
|
|||||||
# ANTHROPIC_LARGE_MODEL=... # Large tier (default: claude-opus-4-6)
|
# ANTHROPIC_LARGE_MODEL=... # Large tier (default: claude-opus-4-6)
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# OPTION 4: AWS Bedrock
|
# OPTION 3: AWS Bedrock
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# https://aws.amazon.com/blogs/machine-learning/accelerate-ai-development-with-amazon-bedrock-api-keys/
|
# https://aws.amazon.com/blogs/machine-learning/accelerate-ai-development-with-amazon-bedrock-api-keys/
|
||||||
# Requires the model tier overrides above to be set with Bedrock-specific model IDs.
|
# Requires the model tier overrides above to be set with Bedrock-specific model IDs.
|
||||||
@@ -57,7 +43,7 @@ ANTHROPIC_API_KEY=your-api-key-here
|
|||||||
# AWS_BEARER_TOKEN_BEDROCK=your-bearer-token
|
# AWS_BEARER_TOKEN_BEDROCK=your-bearer-token
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# OPTION 5: Google Vertex AI
|
# OPTION 4: Google Vertex AI
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-partner-models
|
# https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-partner-models
|
||||||
# Requires a GCP service account with roles/aiplatform.user.
|
# Requires a GCP service account with roles/aiplatform.user.
|
||||||
@@ -72,9 +58,3 @@ ANTHROPIC_API_KEY=your-api-key-here
|
|||||||
# CLOUD_ML_REGION=us-east5
|
# CLOUD_ML_REGION=us-east5
|
||||||
# ANTHROPIC_VERTEX_PROJECT_ID=your-gcp-project-id
|
# ANTHROPIC_VERTEX_PROJECT_ID=your-gcp-project-id
|
||||||
# GOOGLE_APPLICATION_CREDENTIALS=./credentials/google-sa-key.json
|
# GOOGLE_APPLICATION_CREDENTIALS=./credentials/google-sa-key.json
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# Available Models
|
|
||||||
# =============================================================================
|
|
||||||
# OpenAI: gpt-5.2, gpt-5-mini
|
|
||||||
# OpenRouter: google/gemini-3-flash-preview
|
|
||||||
|
|||||||
@@ -120,8 +120,6 @@ body:
|
|||||||
- "Custom base URL (proxy/gateway)"
|
- "Custom base URL (proxy/gateway)"
|
||||||
- "AWS Bedrock"
|
- "AWS Bedrock"
|
||||||
- "Google Vertex AI"
|
- "Google Vertex AI"
|
||||||
- "Router - OpenAI"
|
|
||||||
- "Router - OpenRouter"
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
{
|
|
||||||
"HOST": "0.0.0.0",
|
|
||||||
"APIKEY": "shannon-router-key",
|
|
||||||
"LOG": true,
|
|
||||||
"LOG_LEVEL": "info",
|
|
||||||
"NON_INTERACTIVE_MODE": true,
|
|
||||||
"API_TIMEOUT_MS": 600000,
|
|
||||||
"Providers": [
|
|
||||||
{
|
|
||||||
"name": "openai",
|
|
||||||
"api_base_url": "https://api.openai.com/v1/chat/completions",
|
|
||||||
"api_key": "$OPENAI_API_KEY",
|
|
||||||
"models": ["gpt-5.2", "gpt-5-mini"],
|
|
||||||
"transformer": {
|
|
||||||
"use": [["maxcompletiontokens", { "max_completion_tokens": 16384 }]]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "openrouter",
|
|
||||||
"api_base_url": "https://openrouter.ai/api/v1/chat/completions",
|
|
||||||
"api_key": "$OPENROUTER_API_KEY",
|
|
||||||
"models": ["google/gemini-3-flash-preview"],
|
|
||||||
"transformer": {
|
|
||||||
"use": ["openrouter"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"Router": {
|
|
||||||
"default": "$ROUTER_DEFAULT"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -13,7 +13,7 @@ import { type ShannonConfig, saveConfig } from '../config/writer.js';
|
|||||||
|
|
||||||
const SHANNON_HOME = path.join(os.homedir(), '.shannon');
|
const SHANNON_HOME = path.join(os.homedir(), '.shannon');
|
||||||
|
|
||||||
type Provider = 'anthropic' | 'custom_base_url' | 'bedrock' | 'vertex' | 'router';
|
type Provider = 'anthropic' | 'custom_base_url' | 'bedrock' | 'vertex';
|
||||||
|
|
||||||
export async function setup(): Promise<void> {
|
export async function setup(): Promise<void> {
|
||||||
p.intro('Shannon Setup');
|
p.intro('Shannon Setup');
|
||||||
@@ -26,7 +26,6 @@ export async function setup(): Promise<void> {
|
|||||||
{ value: 'custom_base_url' as const, label: 'Custom Base URL', hint: 'proxies, gateways' },
|
{ value: 'custom_base_url' as const, label: 'Custom Base URL', hint: 'proxies, gateways' },
|
||||||
{ value: 'bedrock' as const, label: 'Claude via AWS Bedrock' },
|
{ value: 'bedrock' as const, label: 'Claude via AWS Bedrock' },
|
||||||
{ value: 'vertex' as const, label: 'Claude via Google Vertex AI' },
|
{ value: 'vertex' as const, label: 'Claude via Google Vertex AI' },
|
||||||
{ value: 'router' as const, label: 'Router', hint: 'experimental' },
|
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
if (p.isCancel(provider)) return cancelAndExit();
|
if (p.isCancel(provider)) return cancelAndExit();
|
||||||
@@ -51,8 +50,6 @@ async function setupProvider(provider: Provider): Promise<ShannonConfig> {
|
|||||||
return setupBedrock();
|
return setupBedrock();
|
||||||
case 'vertex':
|
case 'vertex':
|
||||||
return setupVertex();
|
return setupVertex();
|
||||||
case 'router':
|
|
||||||
return setupRouter();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -282,50 +279,6 @@ async function setupVertex(): Promise<ShannonConfig> {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function setupRouter(): Promise<ShannonConfig> {
|
|
||||||
const routerProvider = await p.select({
|
|
||||||
message: 'Router provider',
|
|
||||||
options: [
|
|
||||||
{ value: 'openai' as const, label: 'OpenAI' },
|
|
||||||
{ value: 'openrouter' as const, label: 'OpenRouter' },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
if (p.isCancel(routerProvider)) return cancelAndExit();
|
|
||||||
|
|
||||||
const apiKey = await promptSecret(
|
|
||||||
routerProvider === 'openai' ? 'Enter your OpenAI API key' : 'Enter your OpenRouter API key',
|
|
||||||
);
|
|
||||||
|
|
||||||
let defaultModel: string;
|
|
||||||
if (routerProvider === 'openai') {
|
|
||||||
const model = await p.select({
|
|
||||||
message: 'Default model',
|
|
||||||
options: [
|
|
||||||
{ value: 'gpt-5.2' as const, label: 'GPT-5.2' },
|
|
||||||
{ value: 'gpt-5-mini' as const, label: 'GPT-5 Mini' },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
if (p.isCancel(model)) return cancelAndExit();
|
|
||||||
defaultModel = `openai,${model}`;
|
|
||||||
} else {
|
|
||||||
const model = await p.select({
|
|
||||||
message: 'Default model',
|
|
||||||
options: [{ value: 'google/gemini-3-flash-preview' as const, label: 'Google Gemini 3 Flash Preview' }],
|
|
||||||
});
|
|
||||||
if (p.isCancel(model)) return cancelAndExit();
|
|
||||||
defaultModel = `openrouter,${model}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const router: ShannonConfig['router'] = { default: defaultModel };
|
|
||||||
if (routerProvider === 'openai') {
|
|
||||||
router.openai_key = apiKey;
|
|
||||||
} else {
|
|
||||||
router.openrouter_key = apiKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { router };
|
|
||||||
}
|
|
||||||
|
|
||||||
// === Helpers ===
|
// === Helpers ===
|
||||||
|
|
||||||
async function promptSecret(message: string): Promise<string> {
|
async function promptSecret(message: string): Promise<string> {
|
||||||
|
|||||||
@@ -5,12 +5,11 @@
|
|||||||
* and npx mode (Docker Hub pull, ~/.shannon/).
|
* and npx mode (Docker Hub pull, ~/.shannon/).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import os from 'node:os';
|
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { getOrchestrator } from '../backend.js';
|
import { ensureImage, ensureInfra, randomSuffix, spawnWorker } from '../docker.js';
|
||||||
import { randomSuffix } from '../docker.js';
|
import { buildEnvFlags, loadEnv, validateCredentials } from '../env.js';
|
||||||
import { buildEnvFlags, isRouterConfigured, loadEnv, validateCredentials } from '../env.js';
|
|
||||||
import { getCredentialsPath, getWorkspacesDir, initHome } from '../home.js';
|
import { getCredentialsPath, getWorkspacesDir, initHome } from '../home.js';
|
||||||
import { isLocal } from '../mode.js';
|
import { isLocal } from '../mode.js';
|
||||||
import { resolveConfig, resolveRepo } from '../paths.js';
|
import { resolveConfig, resolveRepo } from '../paths.js';
|
||||||
@@ -23,7 +22,6 @@ export interface StartArgs {
|
|||||||
workspace?: string;
|
workspace?: string;
|
||||||
output?: string;
|
output?: string;
|
||||||
pipelineTesting: boolean;
|
pipelineTesting: boolean;
|
||||||
router: boolean;
|
|
||||||
version: string;
|
version: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,13 +30,12 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
initHome();
|
initHome();
|
||||||
loadEnv();
|
loadEnv();
|
||||||
|
|
||||||
// 2. Validate credentials and auto-detect router mode
|
// 2. Validate credentials
|
||||||
const creds = validateCredentials();
|
const creds = validateCredentials();
|
||||||
if (!creds.valid) {
|
if (!creds.valid) {
|
||||||
console.error(`ERROR: ${creds.error}`);
|
console.error(`ERROR: ${creds.error}`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
const useRouter = args.router || isRouterConfigured();
|
|
||||||
|
|
||||||
// 3. Resolve paths
|
// 3. Resolve paths
|
||||||
const repo = resolveRepo(args.repo);
|
const repo = resolveRepo(args.repo);
|
||||||
@@ -49,27 +46,20 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
fs.mkdirSync(workspacesDir, { recursive: true });
|
fs.mkdirSync(workspacesDir, { recursive: true });
|
||||||
fs.chmodSync(workspacesDir, 0o777);
|
fs.chmodSync(workspacesDir, 0o777);
|
||||||
|
|
||||||
// 5. Handle router env
|
// 5. Ensure image (auto-build in dev, pull in npx) and start infra
|
||||||
if (useRouter) {
|
ensureImage(args.version);
|
||||||
process.env.ANTHROPIC_BASE_URL = 'http://shannon-router:3456';
|
await ensureInfra();
|
||||||
process.env.ANTHROPIC_AUTH_TOKEN = 'shannon-router-key';
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. Ensure image and start infra via orchestrator
|
// 6. Generate unique task queue and container name
|
||||||
const orchestrator = await getOrchestrator();
|
|
||||||
orchestrator.ensureImage(args.version);
|
|
||||||
await orchestrator.ensureInfra(useRouter);
|
|
||||||
|
|
||||||
// 7. Generate unique task queue and container name
|
|
||||||
const suffix = randomSuffix();
|
const suffix = randomSuffix();
|
||||||
const taskQueue = `shannon-${suffix}`;
|
const taskQueue = `shannon-${suffix}`;
|
||||||
const containerName = `shannon-worker-${suffix}`;
|
const containerName = `shannon-worker-${suffix}`;
|
||||||
|
|
||||||
// 8. Generate workspace name if not provided
|
// 7. Generate workspace name if not provided
|
||||||
const workspace =
|
const workspace =
|
||||||
args.workspace ?? `${new URL(args.url).hostname.replace(/[^a-zA-Z0-9-]/g, '-')}_shannon-${Date.now()}`;
|
args.workspace ?? `${new URL(args.url).hostname.replace(/[^a-zA-Z0-9-]/g, '-')}_shannon-${Date.now()}`;
|
||||||
|
|
||||||
// 9. Create writable overlay directories (mounted over :ro repo paths inside container)
|
// 8. Create writable overlay directories (mounted over :ro repo paths inside container)
|
||||||
// Workspace dir must be 0o777 so the container user (UID 1001) can create audit subdirs
|
// Workspace dir must be 0o777 so the container user (UID 1001) can create audit subdirs
|
||||||
const workspacePath = path.join(workspacesDir, workspace);
|
const workspacePath = path.join(workspacesDir, workspace);
|
||||||
fs.mkdirSync(workspacePath, { recursive: true });
|
fs.mkdirSync(workspacePath, { recursive: true });
|
||||||
@@ -80,12 +70,10 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
fs.chmodSync(dirPath, 0o777);
|
fs.chmodSync(dirPath, 0o777);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 10. Pre-create overlay mount points (Linux :ro mounts can't auto-create them)
|
// 9. Pre-create overlay mount points (:ro mounts can't auto-create them)
|
||||||
if (os.platform() === 'linux') {
|
const shannonDir = path.join(repo.hostPath, '.shannon');
|
||||||
const shannonDir = path.join(repo.hostPath, '.shannon');
|
for (const dir of ['deliverables', 'scratchpad', '.playwright-cli']) {
|
||||||
for (const dir of ['deliverables', 'scratchpad', '.playwright-cli']) {
|
fs.mkdirSync(path.join(shannonDir, dir), { recursive: true });
|
||||||
fs.mkdirSync(path.join(shannonDir, dir), { recursive: true });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const credentialsPath = getCredentialsPath();
|
const credentialsPath = getCredentialsPath();
|
||||||
@@ -95,20 +83,20 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
process.env.GOOGLE_APPLICATION_CREDENTIALS = '/app/credentials/google-sa-key.json';
|
process.env.GOOGLE_APPLICATION_CREDENTIALS = '/app/credentials/google-sa-key.json';
|
||||||
}
|
}
|
||||||
|
|
||||||
// 11. Resolve output directory
|
// 10. Resolve output directory
|
||||||
const outputDir = args.output ? path.resolve(args.output) : undefined;
|
const outputDir = args.output ? path.resolve(args.output) : undefined;
|
||||||
if (outputDir) {
|
if (outputDir) {
|
||||||
fs.mkdirSync(outputDir, { recursive: true });
|
fs.mkdirSync(outputDir, { recursive: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
// 12. Resolve prompts directory (local mode only)
|
// 11. Resolve prompts directory (local mode only)
|
||||||
const promptsDir = isLocal() ? path.resolve('apps/worker/prompts') : undefined;
|
const promptsDir = isLocal() ? path.resolve('apps/worker/prompts') : undefined;
|
||||||
|
|
||||||
// 13. Display splash screen
|
// 12. Display splash screen
|
||||||
displaySplash(isLocal() ? undefined : args.version);
|
displaySplash(isLocal() ? undefined : args.version);
|
||||||
|
|
||||||
// 14. Spawn worker via orchestrator
|
// 13. Spawn worker container
|
||||||
const handle = orchestrator.spawnWorker({
|
const proc = spawnWorker({
|
||||||
version: args.version,
|
version: args.version,
|
||||||
url: args.url,
|
url: args.url,
|
||||||
repo,
|
repo,
|
||||||
@@ -124,8 +112,8 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
...(args.pipelineTesting && { pipelineTesting: true }),
|
...(args.pipelineTesting && { pipelineTesting: true }),
|
||||||
});
|
});
|
||||||
|
|
||||||
// 15. Wait for workflow to register, then display info
|
// 14. Wait for workflow to register, then display info
|
||||||
handle.onError((err) => {
|
proc.on('error', (err) => {
|
||||||
console.error(`Failed to start worker: ${err.message}`);
|
console.error(`Failed to start worker: ${err.message}`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
});
|
});
|
||||||
@@ -173,7 +161,7 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
|
|
||||||
// Clear waiting line and show info
|
// Clear waiting line and show info
|
||||||
process.stdout.write('\r\x1b[K');
|
process.stdout.write('\r\x1b[K');
|
||||||
printInfo(args, useRouter, workspace, workflowId, repo.hostPath, workspacesDir);
|
printInfo(args, workspace, workflowId, repo.hostPath, workspacesDir);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
@@ -182,14 +170,18 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
process.stdout.write('.');
|
process.stdout.write('.');
|
||||||
}, 2000);
|
}, 2000);
|
||||||
|
|
||||||
// Stop the worker only if it hasn't started yet
|
// Stop the worker container only if it hasn't started yet
|
||||||
let cleaned = false;
|
let cleaned = false;
|
||||||
const cleanup = (): void => {
|
const cleanup = (): void => {
|
||||||
if (cleaned || started) return;
|
if (cleaned || started) return;
|
||||||
cleaned = true;
|
cleaned = true;
|
||||||
clearInterval(pollInterval);
|
clearInterval(pollInterval);
|
||||||
console.log(`\nStopping worker ${containerName}...`);
|
console.log(`\nStopping worker ${containerName}...`);
|
||||||
handle.kill();
|
try {
|
||||||
|
execFileSync('docker', ['stop', containerName], { stdio: 'pipe' });
|
||||||
|
} catch {
|
||||||
|
// Container may have already exited
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
process.on('SIGINT', () => {
|
process.on('SIGINT', () => {
|
||||||
@@ -205,7 +197,6 @@ export async function start(args: StartArgs): Promise<void> {
|
|||||||
|
|
||||||
function printInfo(
|
function printInfo(
|
||||||
args: StartArgs,
|
args: StartArgs,
|
||||||
routerActive: boolean,
|
|
||||||
workspace: string,
|
workspace: string,
|
||||||
workflowId: string,
|
workflowId: string,
|
||||||
repoPath: string,
|
repoPath: string,
|
||||||
@@ -223,9 +214,6 @@ function printInfo(
|
|||||||
if (args.pipelineTesting) {
|
if (args.pipelineTesting) {
|
||||||
console.log(' Mode: Pipeline Testing');
|
console.log(' Mode: Pipeline Testing');
|
||||||
}
|
}
|
||||||
if (routerActive) {
|
|
||||||
console.log(' Router: Enabled');
|
|
||||||
}
|
|
||||||
console.log('');
|
console.log('');
|
||||||
console.log(' Monitor:');
|
console.log(' Monitor:');
|
||||||
if (workflowId) {
|
if (workflowId) {
|
||||||
|
|||||||
@@ -44,11 +44,6 @@ const CONFIG_MAP: readonly ConfigMapping[] = [
|
|||||||
{ env: 'ANTHROPIC_BASE_URL', toml: 'custom_base_url.base_url', type: 'string' },
|
{ env: 'ANTHROPIC_BASE_URL', toml: 'custom_base_url.base_url', type: 'string' },
|
||||||
{ env: 'ANTHROPIC_AUTH_TOKEN', toml: 'custom_base_url.auth_token', type: 'string' },
|
{ env: 'ANTHROPIC_AUTH_TOKEN', toml: 'custom_base_url.auth_token', type: 'string' },
|
||||||
|
|
||||||
// Router
|
|
||||||
{ env: 'ROUTER_DEFAULT', toml: 'router.default', type: 'string' },
|
|
||||||
{ env: 'OPENAI_API_KEY', toml: 'router.openai_key', type: 'string' },
|
|
||||||
{ env: 'OPENROUTER_API_KEY', toml: 'router.openrouter_key', type: 'string' },
|
|
||||||
|
|
||||||
// Model tiers
|
// Model tiers
|
||||||
{ env: 'ANTHROPIC_SMALL_MODEL', toml: 'models.small', type: 'string' },
|
{ env: 'ANTHROPIC_SMALL_MODEL', toml: 'models.small', type: 'string' },
|
||||||
{ env: 'ANTHROPIC_MEDIUM_MODEL', toml: 'models.medium', type: 'string' },
|
{ env: 'ANTHROPIC_MEDIUM_MODEL', toml: 'models.medium', type: 'string' },
|
||||||
@@ -165,20 +160,6 @@ function validateProviderFields(config: TOMLConfig, provider: string, errors: st
|
|||||||
validateModelTiers(config, 'vertex', errors);
|
validateModelTiers(config, 'vertex', errors);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'router': {
|
|
||||||
if (!keys.includes('default')) {
|
|
||||||
errors.push('[router] missing required key: default');
|
|
||||||
}
|
|
||||||
if (!keys.includes('openai_key') && !keys.includes('openrouter_key')) {
|
|
||||||
errors.push('[router] requires either openai_key or openrouter_key');
|
|
||||||
}
|
|
||||||
const models = config.models as Record<string, unknown> | undefined;
|
|
||||||
if (models && typeof models === 'object' && Object.keys(models).length > 0) {
|
|
||||||
errors.push('[models] is not supported with [router]');
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,7 +223,7 @@ function validateConfig(config: TOMLConfig): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 4. Only one provider section allowed (ignore empty sections)
|
// 4. Only one provider section allowed (ignore empty sections)
|
||||||
const PROVIDER_SECTIONS = ['anthropic', 'custom_base_url', 'bedrock', 'vertex', 'router'] as const;
|
const PROVIDER_SECTIONS = ['anthropic', 'custom_base_url', 'bedrock', 'vertex'] as const;
|
||||||
const present = PROVIDER_SECTIONS.filter((s) => {
|
const present = PROVIDER_SECTIONS.filter((s) => {
|
||||||
const section = config[s];
|
const section = config[s];
|
||||||
return section && typeof section === 'object' && Object.keys(section).length > 0;
|
return section && typeof section === 'object' && Object.keys(section).length > 0;
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ export interface ShannonConfig {
|
|||||||
custom_base_url?: { base_url?: string; auth_token?: string };
|
custom_base_url?: { base_url?: string; auth_token?: string };
|
||||||
bedrock?: { use?: boolean; region?: string; token?: string };
|
bedrock?: { use?: boolean; region?: string; token?: string };
|
||||||
vertex?: { use?: boolean; region?: string; project_id?: string; key_path?: string };
|
vertex?: { use?: boolean; region?: string; project_id?: string; key_path?: string };
|
||||||
router?: { default?: string; openai_key?: string; openrouter_key?: string };
|
|
||||||
models?: { small?: string; medium?: string; large?: string };
|
models?: { small?: string; medium?: string; large?: string };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+212
-259
@@ -12,22 +12,27 @@ import path from 'node:path';
|
|||||||
import { setTimeout as sleep } from 'node:timers/promises';
|
import { setTimeout as sleep } from 'node:timers/promises';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { getMode } from './mode.js';
|
import { getMode } from './mode.js';
|
||||||
import type { Orchestrator, WorkerHandle, WorkerOptions } from './orchestrator.js';
|
|
||||||
|
|
||||||
export type { WorkerOptions };
|
|
||||||
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
const NPX_IMAGE_REPO = 'keygraph/shannon';
|
const NPX_IMAGE_REPO = 'keygraph/shannon';
|
||||||
const DEV_IMAGE = 'shannon-worker';
|
const DEV_IMAGE = 'shannon-worker';
|
||||||
|
|
||||||
|
export function getWorkerImage(version: string): string {
|
||||||
|
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getComposeFile(): string {
|
||||||
|
return getMode() === 'local'
|
||||||
|
? path.resolve('docker-compose.yml')
|
||||||
|
: path.resolve(__dirname, '..', 'infra', 'compose.yml');
|
||||||
|
}
|
||||||
|
|
||||||
/** Generate an 8-char random hex suffix for container/queue names. */
|
/** Generate an 8-char random hex suffix for container/queue names. */
|
||||||
export function randomSuffix(): string {
|
export function randomSuffix(): string {
|
||||||
return crypto.randomBytes(4).toString('hex');
|
return crypto.randomBytes(4).toString('hex');
|
||||||
}
|
}
|
||||||
|
|
||||||
// === Internal Helpers ===
|
|
||||||
|
|
||||||
/** Run a command silently, return true if it succeeds. */
|
/** Run a command silently, return true if it succeeds. */
|
||||||
function runQuiet(cmd: string, args: string[]): boolean {
|
function runQuiet(cmd: string, args: string[]): boolean {
|
||||||
try {
|
try {
|
||||||
@@ -47,16 +52,83 @@ function runOutput(cmd: string, args: string[]): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function getComposeFile(): string {
|
/**
|
||||||
return getMode() === 'local'
|
* Check if Temporal is running and healthy.
|
||||||
? path.resolve('docker-compose.yml')
|
*/
|
||||||
: path.resolve(__dirname, '..', 'infra', 'compose.yml');
|
export function isTemporalReady(): boolean {
|
||||||
|
const output = runOutput('docker', [
|
||||||
|
'exec',
|
||||||
|
'shannon-temporal',
|
||||||
|
'temporal',
|
||||||
|
'operator',
|
||||||
|
'cluster',
|
||||||
|
'health',
|
||||||
|
'--address',
|
||||||
|
'localhost:7233',
|
||||||
|
]);
|
||||||
|
return output.includes('SERVING');
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Check if the router container is running and healthy. */
|
/**
|
||||||
function isRouterReady(): boolean {
|
* Ensure Temporal is running via compose.
|
||||||
const status = runOutput('docker', ['inspect', '--format', '{{.State.Health.Status}}', 'shannon-router']);
|
*/
|
||||||
return status === 'healthy';
|
export async function ensureInfra(): Promise<void> {
|
||||||
|
if (isTemporalReady()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const composeFile = getComposeFile();
|
||||||
|
console.log('Starting Shannon infrastructure...');
|
||||||
|
execFileSync('docker', ['compose', '-f', composeFile, 'up', '-d'], { stdio: 'inherit' });
|
||||||
|
|
||||||
|
console.log('Waiting for Temporal to be ready...');
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
|
if (isTemporalReady()) {
|
||||||
|
console.log('Temporal is ready!');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await sleep(2000);
|
||||||
|
}
|
||||||
|
console.error('Timeout waiting for Temporal');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the worker image locally (local mode only).
|
||||||
|
*/
|
||||||
|
export function buildImage(noCache: boolean): void {
|
||||||
|
console.log(`Building ${DEV_IMAGE}...`);
|
||||||
|
const args = ['build'];
|
||||||
|
if (noCache) args.push('--no-cache');
|
||||||
|
args.push('-t', DEV_IMAGE, '.');
|
||||||
|
execFileSync('docker', args, { stdio: 'inherit' });
|
||||||
|
console.log(`Build complete: ${DEV_IMAGE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure the worker image is available.
|
||||||
|
* Local mode: auto-builds if missing. NPX mode: pulls from Docker Hub.
|
||||||
|
*/
|
||||||
|
export function ensureImage(version: string): void {
|
||||||
|
const image = getWorkerImage(version);
|
||||||
|
const exists = runQuiet('docker', ['image', 'inspect', image]);
|
||||||
|
if (exists) return;
|
||||||
|
|
||||||
|
if (getMode() === 'local') {
|
||||||
|
console.log('Worker image not found, building...');
|
||||||
|
buildImage(false);
|
||||||
|
} else {
|
||||||
|
console.log(`Pulling ${image}...`);
|
||||||
|
try {
|
||||||
|
execFileSync('docker', ['pull', image], { stdio: 'inherit' });
|
||||||
|
} catch {
|
||||||
|
console.error(`\nERROR: Failed to pull ${image}`);
|
||||||
|
console.error('The image may not be available for your platform yet.');
|
||||||
|
console.error('Check https://hub.docker.com/r/keygraph/shannon for available tags.');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
pruneOldImages(version);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -73,259 +145,140 @@ function addHostFlag(): string[] {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove old keygraph/shannon images that don't match the current version. */
|
export interface WorkerOptions {
|
||||||
|
version: string;
|
||||||
|
url: string;
|
||||||
|
repo: { hostPath: string; containerPath: string };
|
||||||
|
workspacesDir: string;
|
||||||
|
taskQueue: string;
|
||||||
|
containerName: string;
|
||||||
|
envFlags: string[];
|
||||||
|
config?: { hostPath: string; containerPath: string };
|
||||||
|
credentials?: string;
|
||||||
|
promptsDir?: string;
|
||||||
|
outputDir?: string;
|
||||||
|
workspace: string;
|
||||||
|
pipelineTesting?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spawn the worker container in detached mode and return the process.
|
||||||
|
*/
|
||||||
|
export function spawnWorker(opts: WorkerOptions): ChildProcess {
|
||||||
|
const args = ['run', '-d', '--rm', '--name', opts.containerName, '--network', 'shannon-net'];
|
||||||
|
|
||||||
|
// Add host flag for Linux
|
||||||
|
args.push(...addHostFlag());
|
||||||
|
|
||||||
|
// UID remapping for Linux bind mounts
|
||||||
|
if (os.platform() === 'linux' && process.getuid && process.getgid) {
|
||||||
|
args.push('-e', `SHANNON_HOST_UID=${process.getuid()}`, '-e', `SHANNON_HOST_GID=${process.getgid()}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Volume mounts
|
||||||
|
args.push('-v', `${opts.workspacesDir}:/app/workspaces`);
|
||||||
|
args.push('-v', `${opts.repo.hostPath}:${opts.repo.containerPath}:ro`);
|
||||||
|
|
||||||
|
// Writable overlays: shadow .shannon/ inside the :ro repo with workspace-backed dirs
|
||||||
|
const workspacePath = path.join(opts.workspacesDir, opts.workspace);
|
||||||
|
args.push('-v', `${path.join(workspacePath, 'deliverables')}:${opts.repo.containerPath}/.shannon/deliverables`);
|
||||||
|
args.push('-v', `${path.join(workspacePath, 'scratchpad')}:${opts.repo.containerPath}/.shannon/scratchpad`);
|
||||||
|
args.push('-v', `${path.join(workspacePath, '.playwright-cli')}:${opts.repo.containerPath}/.shannon/.playwright-cli`);
|
||||||
|
|
||||||
|
// Local mode: mount prompts for live editing
|
||||||
|
if (opts.promptsDir) {
|
||||||
|
args.push('-v', `${opts.promptsDir}:/app/apps/worker/prompts:ro`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.config) {
|
||||||
|
args.push('-v', `${opts.config.hostPath}:${opts.config.containerPath}:ro`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output directory for deliverables copy
|
||||||
|
if (opts.outputDir) {
|
||||||
|
args.push('-v', `${opts.outputDir}:/app/output`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mount credentials file to fixed container path
|
||||||
|
if (opts.credentials) {
|
||||||
|
args.push('-v', `${opts.credentials}:/app/credentials/google-sa-key.json:ro`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Environment
|
||||||
|
args.push(...opts.envFlags);
|
||||||
|
|
||||||
|
// Container settings
|
||||||
|
args.push('--shm-size', '2gb', '--security-opt', 'seccomp=unconfined');
|
||||||
|
|
||||||
|
// Image
|
||||||
|
args.push(getWorkerImage(opts.version));
|
||||||
|
|
||||||
|
// Worker command
|
||||||
|
args.push('node', 'apps/worker/dist/temporal/worker.js', opts.url, opts.repo.containerPath);
|
||||||
|
args.push('--task-queue', opts.taskQueue);
|
||||||
|
if (opts.config) {
|
||||||
|
args.push('--config', opts.config.containerPath);
|
||||||
|
}
|
||||||
|
if (opts.outputDir) {
|
||||||
|
args.push('--output', '/app/output');
|
||||||
|
}
|
||||||
|
args.push('--workspace', opts.workspace);
|
||||||
|
if (opts.pipelineTesting) {
|
||||||
|
args.push('--pipeline-testing');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prevent MSYS/Git Bash from converting Unix paths (e.g. /repos/my-repo) to Windows paths
|
||||||
|
return spawn('docker', args, {
|
||||||
|
stdio: 'pipe',
|
||||||
|
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop all running shannon-worker-* containers.
|
||||||
|
*/
|
||||||
|
export function stopWorkers(): void {
|
||||||
|
const workers = runOutput('docker', ['ps', '-q', '--filter', 'name=shannon-worker-']);
|
||||||
|
if (!workers) return;
|
||||||
|
|
||||||
|
const ids = workers.split('\n').filter(Boolean);
|
||||||
|
console.log('Stopping worker containers...');
|
||||||
|
execFileSync('docker', ['stop', ...ids], { stdio: 'inherit' });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tear down the compose stack.
|
||||||
|
*/
|
||||||
|
export function stopInfra(clean: boolean): void {
|
||||||
|
const composeFile = getComposeFile();
|
||||||
|
const args = ['compose', '-f', composeFile, 'down'];
|
||||||
|
if (clean) args.push('-v');
|
||||||
|
execFileSync('docker', args, { stdio: 'inherit' });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove old keygraph/shannon images that don't match the current version.
|
||||||
|
*/
|
||||||
function pruneOldImages(currentVersion: string): void {
|
function pruneOldImages(currentVersion: string): void {
|
||||||
const output = runOutput('docker', ['images', NPX_IMAGE_REPO, '--format', '{{.Tag}}']);
|
const output = runOutput('docker', ['images', NPX_IMAGE_REPO, '--format', '{{.Tag}}']);
|
||||||
if (!output) return;
|
if (!output) return;
|
||||||
|
|
||||||
const stale = output.split('\n').filter((tag) => tag && tag !== currentVersion);
|
const currentTag = currentVersion;
|
||||||
|
const stale = output.split('\n').filter((tag) => tag && tag !== currentTag);
|
||||||
for (const tag of stale) {
|
for (const tag of stale) {
|
||||||
runQuiet('docker', ['rmi', `${NPX_IMAGE_REPO}:${tag}`]);
|
runQuiet('docker', ['rmi', `${NPX_IMAGE_REPO}:${tag}`]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// === DockerOrchestrator ===
|
/**
|
||||||
|
* List running worker containers.
|
||||||
/** Docker-based orchestration backend. */
|
*/
|
||||||
export class DockerOrchestrator implements Orchestrator {
|
export function listRunningWorkers(): string {
|
||||||
getWorkerImage(version: string): string {
|
return runOutput('docker', [
|
||||||
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
|
'ps',
|
||||||
}
|
'--filter',
|
||||||
|
'name=shannon-worker-',
|
||||||
isTemporalReady(): boolean {
|
'--format',
|
||||||
const output = runOutput('docker', [
|
'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}',
|
||||||
'exec',
|
]);
|
||||||
'shannon-temporal',
|
|
||||||
'temporal',
|
|
||||||
'operator',
|
|
||||||
'cluster',
|
|
||||||
'health',
|
|
||||||
'--address',
|
|
||||||
'localhost:7233',
|
|
||||||
]);
|
|
||||||
return output.includes('SERVING');
|
|
||||||
}
|
|
||||||
|
|
||||||
async ensureInfra(useRouter: boolean): Promise<void> {
|
|
||||||
const temporalReady = this.isTemporalReady();
|
|
||||||
const routerNeeded = useRouter && !isRouterReady();
|
|
||||||
|
|
||||||
if (temporalReady && !routerNeeded) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const composeFile = getComposeFile();
|
|
||||||
const composeArgs = ['compose', '-f', composeFile];
|
|
||||||
if (useRouter) composeArgs.push('--profile', 'router');
|
|
||||||
composeArgs.push('up', '-d');
|
|
||||||
|
|
||||||
if (temporalReady && routerNeeded) {
|
|
||||||
console.log('Starting router...');
|
|
||||||
} else {
|
|
||||||
console.log('Starting Shannon infrastructure...');
|
|
||||||
}
|
|
||||||
execFileSync('docker', composeArgs, { stdio: 'inherit' });
|
|
||||||
|
|
||||||
// Wait for Temporal if it wasn't already running
|
|
||||||
if (!temporalReady) {
|
|
||||||
console.log('Waiting for Temporal to be ready...');
|
|
||||||
for (let i = 0; i < 30; i++) {
|
|
||||||
if (this.isTemporalReady()) {
|
|
||||||
console.log('Temporal is ready!');
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (i === 29) {
|
|
||||||
console.error('Timeout waiting for Temporal');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
await sleep(2000);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for router if needed
|
|
||||||
if (routerNeeded) {
|
|
||||||
console.log('Waiting for router to be ready...');
|
|
||||||
for (let i = 0; i < 15; i++) {
|
|
||||||
if (isRouterReady()) {
|
|
||||||
console.log('Router is ready!');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await sleep(2000);
|
|
||||||
}
|
|
||||||
console.error('Timeout waiting for router');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ensureImage(version: string): void {
|
|
||||||
const image = this.getWorkerImage(version);
|
|
||||||
const exists = runQuiet('docker', ['image', 'inspect', image]);
|
|
||||||
if (exists) return;
|
|
||||||
|
|
||||||
if (getMode() === 'local') {
|
|
||||||
console.log('Worker image not found, building...');
|
|
||||||
this.buildImage(false);
|
|
||||||
} else {
|
|
||||||
console.log(`Pulling ${image}...`);
|
|
||||||
try {
|
|
||||||
execFileSync('docker', ['pull', image], { stdio: 'inherit' });
|
|
||||||
} catch {
|
|
||||||
console.error(`\nERROR: Failed to pull ${image}`);
|
|
||||||
console.error('The image may not be available for your platform yet.');
|
|
||||||
console.error('Check https://hub.docker.com/r/keygraph/shannon for available tags.');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
pruneOldImages(version);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
spawnWorker(opts: WorkerOptions): WorkerHandle {
|
|
||||||
const args = ['run', '-d', '--rm', '--name', opts.containerName, '--network', 'shannon-net'];
|
|
||||||
|
|
||||||
// Add host flag for Linux
|
|
||||||
args.push(...addHostFlag());
|
|
||||||
|
|
||||||
// UID remapping for Linux bind mounts
|
|
||||||
if (os.platform() === 'linux' && process.getuid && process.getgid) {
|
|
||||||
args.push('-e', `SHANNON_HOST_UID=${process.getuid()}`, '-e', `SHANNON_HOST_GID=${process.getgid()}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Volume mounts
|
|
||||||
args.push('-v', `${opts.workspacesDir}:/app/workspaces`);
|
|
||||||
args.push('-v', `${opts.repo.hostPath}:${opts.repo.containerPath}:ro`);
|
|
||||||
|
|
||||||
// Writable overlays: shadow .shannon/ inside the :ro repo with workspace-backed dirs
|
|
||||||
const workspacePath = path.join(opts.workspacesDir, opts.workspace);
|
|
||||||
args.push('-v', `${path.join(workspacePath, 'deliverables')}:${opts.repo.containerPath}/.shannon/deliverables`);
|
|
||||||
args.push('-v', `${path.join(workspacePath, 'scratchpad')}:${opts.repo.containerPath}/.shannon/scratchpad`);
|
|
||||||
args.push(
|
|
||||||
'-v',
|
|
||||||
`${path.join(workspacePath, '.playwright-cli')}:${opts.repo.containerPath}/.shannon/.playwright-cli`,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Local mode: mount prompts for live editing
|
|
||||||
if (opts.promptsDir) {
|
|
||||||
args.push('-v', `${opts.promptsDir}:/app/apps/worker/prompts:ro`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (opts.config) {
|
|
||||||
args.push('-v', `${opts.config.hostPath}:${opts.config.containerPath}:ro`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output directory for deliverables copy
|
|
||||||
if (opts.outputDir) {
|
|
||||||
args.push('-v', `${opts.outputDir}:/app/output`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mount credentials file to fixed container path
|
|
||||||
if (opts.credentials) {
|
|
||||||
args.push('-v', `${opts.credentials}:/app/credentials/google-sa-key.json:ro`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Environment
|
|
||||||
args.push(...opts.envFlags);
|
|
||||||
|
|
||||||
// Container settings
|
|
||||||
args.push('--shm-size', '2gb', '--security-opt', 'seccomp=unconfined');
|
|
||||||
|
|
||||||
// Image
|
|
||||||
args.push(this.getWorkerImage(opts.version));
|
|
||||||
|
|
||||||
// Worker command
|
|
||||||
args.push('node', 'apps/worker/dist/temporal/worker.js', opts.url, opts.repo.containerPath);
|
|
||||||
args.push('--task-queue', opts.taskQueue);
|
|
||||||
if (opts.config) {
|
|
||||||
args.push('--config', opts.config.containerPath);
|
|
||||||
}
|
|
||||||
if (opts.outputDir) {
|
|
||||||
args.push('--output', '/app/output');
|
|
||||||
}
|
|
||||||
args.push('--workspace', opts.workspace);
|
|
||||||
if (opts.pipelineTesting) {
|
|
||||||
args.push('--pipeline-testing');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prevent MSYS/Git Bash from converting Unix paths (e.g. /repos/my-repo) to Windows paths
|
|
||||||
const proc = spawn('docker', args, {
|
|
||||||
stdio: 'pipe',
|
|
||||||
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
|
||||||
});
|
|
||||||
|
|
||||||
return new DockerWorkerHandle(proc, opts.containerName);
|
|
||||||
}
|
|
||||||
|
|
||||||
stopWorkers(): void {
|
|
||||||
const workers = runOutput('docker', ['ps', '-q', '--filter', 'name=shannon-worker-']);
|
|
||||||
if (!workers) return;
|
|
||||||
|
|
||||||
const ids = workers.split('\n').filter(Boolean);
|
|
||||||
console.log('Stopping worker containers...');
|
|
||||||
execFileSync('docker', ['stop', ...ids], { stdio: 'inherit' });
|
|
||||||
}
|
|
||||||
|
|
||||||
stopInfra(clean: boolean): void {
|
|
||||||
const composeFile = getComposeFile();
|
|
||||||
const args = ['compose', '-f', composeFile, '--profile', 'router', 'down'];
|
|
||||||
if (clean) args.push('-v');
|
|
||||||
execFileSync('docker', args, { stdio: 'inherit' });
|
|
||||||
}
|
|
||||||
|
|
||||||
listRunningWorkers(): string {
|
|
||||||
return runOutput('docker', [
|
|
||||||
'ps',
|
|
||||||
'--filter',
|
|
||||||
'name=shannon-worker-',
|
|
||||||
'--format',
|
|
||||||
'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}',
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
runEphemeral(image: string, args: string[], mounts: string[]): void {
|
|
||||||
const dockerArgs = ['run', '--rm'];
|
|
||||||
for (const mount of mounts) {
|
|
||||||
dockerArgs.push('-v', mount);
|
|
||||||
}
|
|
||||||
dockerArgs.push(image, ...args);
|
|
||||||
execFileSync('docker', dockerArgs, {
|
|
||||||
stdio: 'inherit',
|
|
||||||
...(os.platform() === 'win32' && { env: { ...process.env, MSYS_NO_PATHCONV: '1' } }),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Build the worker image locally (local mode only). */
|
|
||||||
buildImage(noCache: boolean): void {
|
|
||||||
console.log(`Building ${DEV_IMAGE}...`);
|
|
||||||
const args = ['build'];
|
|
||||||
if (noCache) args.push('--no-cache');
|
|
||||||
args.push('-t', DEV_IMAGE, '.');
|
|
||||||
execFileSync('docker', args, { stdio: 'inherit' });
|
|
||||||
console.log(`Build complete: ${DEV_IMAGE}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** WorkerHandle wrapping a Docker container's ChildProcess. */
|
|
||||||
class DockerWorkerHandle implements WorkerHandle {
|
|
||||||
constructor(
|
|
||||||
private readonly proc: ChildProcess,
|
|
||||||
private readonly containerName: string,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
onError(cb: (err: Error) => void): void {
|
|
||||||
this.proc.on('error', cb);
|
|
||||||
}
|
|
||||||
|
|
||||||
kill(): void {
|
|
||||||
try {
|
|
||||||
execFileSync('docker', ['stop', this.containerName], { stdio: 'pipe' });
|
|
||||||
} catch {
|
|
||||||
// Container may have already exited
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// === Backward-compatible exports ===
|
|
||||||
|
|
||||||
// NOTE: Used by commands/build.ts which doesn't go through the orchestrator
|
|
||||||
export function buildImage(noCache: boolean): void {
|
|
||||||
new DockerOrchestrator().buildImage(noCache);
|
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-13
@@ -14,7 +14,6 @@ export const FORWARD_VARS = [
|
|||||||
'ANTHROPIC_API_KEY',
|
'ANTHROPIC_API_KEY',
|
||||||
'ANTHROPIC_BASE_URL',
|
'ANTHROPIC_BASE_URL',
|
||||||
'ANTHROPIC_AUTH_TOKEN',
|
'ANTHROPIC_AUTH_TOKEN',
|
||||||
'ROUTER_DEFAULT',
|
|
||||||
'CLAUDE_CODE_OAUTH_TOKEN',
|
'CLAUDE_CODE_OAUTH_TOKEN',
|
||||||
'CLAUDE_CODE_USE_BEDROCK',
|
'CLAUDE_CODE_USE_BEDROCK',
|
||||||
'AWS_REGION',
|
'AWS_REGION',
|
||||||
@@ -27,8 +26,6 @@ export const FORWARD_VARS = [
|
|||||||
'ANTHROPIC_MEDIUM_MODEL',
|
'ANTHROPIC_MEDIUM_MODEL',
|
||||||
'ANTHROPIC_LARGE_MODEL',
|
'ANTHROPIC_LARGE_MODEL',
|
||||||
'CLAUDE_CODE_MAX_OUTPUT_TOKENS',
|
'CLAUDE_CODE_MAX_OUTPUT_TOKENS',
|
||||||
'OPENAI_API_KEY',
|
|
||||||
'OPENROUTER_API_KEY',
|
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -81,12 +78,7 @@ export function buildEnvRecord(): Record<string, string> {
|
|||||||
interface CredentialValidation {
|
interface CredentialValidation {
|
||||||
valid: boolean;
|
valid: boolean;
|
||||||
error?: string;
|
error?: string;
|
||||||
mode: 'api-key' | 'oauth' | 'custom-base-url' | 'bedrock' | 'vertex' | 'router';
|
mode: 'api-key' | 'oauth' | 'custom-base-url' | 'bedrock' | 'vertex';
|
||||||
}
|
|
||||||
|
|
||||||
/** Check if router credentials are present in the environment. */
|
|
||||||
export function isRouterConfigured(): boolean {
|
|
||||||
return !!(process.env.ROUTER_DEFAULT && (process.env.OPENAI_API_KEY || process.env.OPENROUTER_API_KEY));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Check if a custom Anthropic-compatible base URL is configured. */
|
/** Check if a custom Anthropic-compatible base URL is configured. */
|
||||||
@@ -102,7 +94,6 @@ function detectProviders(): string[] {
|
|||||||
if (isCustomBaseUrlConfigured()) providers.push('Custom Base URL');
|
if (isCustomBaseUrlConfigured()) providers.push('Custom Base URL');
|
||||||
if (process.env.CLAUDE_CODE_USE_BEDROCK === '1') providers.push('AWS Bedrock');
|
if (process.env.CLAUDE_CODE_USE_BEDROCK === '1') providers.push('AWS Bedrock');
|
||||||
if (process.env.CLAUDE_CODE_USE_VERTEX === '1') providers.push('Google Vertex');
|
if (process.env.CLAUDE_CODE_USE_VERTEX === '1') providers.push('Google Vertex');
|
||||||
if (isRouterConfigured()) providers.push('Router');
|
|
||||||
return providers;
|
return providers;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,9 +159,6 @@ export function validateCredentials(): CredentialValidation {
|
|||||||
}
|
}
|
||||||
return { valid: true, mode: 'vertex' };
|
return { valid: true, mode: 'vertex' };
|
||||||
}
|
}
|
||||||
if (isRouterConfigured()) {
|
|
||||||
return { valid: true, mode: 'router' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const hint =
|
const hint =
|
||||||
getMode() === 'local'
|
getMode() === 'local'
|
||||||
|
|||||||
@@ -70,7 +70,6 @@ Options for 'start':
|
|||||||
-o, --output <path> Copy deliverables to this directory after run
|
-o, --output <path> Copy deliverables to this directory after run
|
||||||
-w, --workspace <name> Named workspace (auto-resumes if exists)
|
-w, --workspace <name> Named workspace (auto-resumes if exists)
|
||||||
--pipeline-testing Use minimal prompts for fast testing
|
--pipeline-testing Use minimal prompts for fast testing
|
||||||
--router Route requests through claude-code-router
|
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
${prefix} start -u https://example.com -r ${mode === 'local' ? 'my-repo' : './my-repo'}
|
${prefix} start -u https://example.com -r ${mode === 'local' ? 'my-repo' : './my-repo'}
|
||||||
@@ -95,7 +94,6 @@ interface ParsedStartArgs {
|
|||||||
workspace?: string;
|
workspace?: string;
|
||||||
output?: string;
|
output?: string;
|
||||||
pipelineTesting: boolean;
|
pipelineTesting: boolean;
|
||||||
router: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseStartArgs(argv: string[]): ParsedStartArgs {
|
function parseStartArgs(argv: string[]): ParsedStartArgs {
|
||||||
@@ -105,7 +103,6 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
|
|||||||
let workspace: string | undefined;
|
let workspace: string | undefined;
|
||||||
let output: string | undefined;
|
let output: string | undefined;
|
||||||
let pipelineTesting = false;
|
let pipelineTesting = false;
|
||||||
let router = false;
|
|
||||||
|
|
||||||
for (let i = 0; i < argv.length; i++) {
|
for (let i = 0; i < argv.length; i++) {
|
||||||
const arg = argv[i];
|
const arg = argv[i];
|
||||||
@@ -150,9 +147,6 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
|
|||||||
case '--pipeline-testing':
|
case '--pipeline-testing':
|
||||||
pipelineTesting = true;
|
pipelineTesting = true;
|
||||||
break;
|
break;
|
||||||
case '--router':
|
|
||||||
router = true;
|
|
||||||
break;
|
|
||||||
default:
|
default:
|
||||||
console.error(`Unknown option: ${arg}`);
|
console.error(`Unknown option: ${arg}`);
|
||||||
console.error(`Run "${getMode() === 'local' ? './shannon' : 'npx @keygraph/shannon'} help" for usage`);
|
console.error(`Run "${getMode() === 'local' ? './shannon' : 'npx @keygraph/shannon'} help" for usage`);
|
||||||
@@ -170,7 +164,6 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
|
|||||||
url,
|
url,
|
||||||
repo,
|
repo,
|
||||||
pipelineTesting,
|
pipelineTesting,
|
||||||
router,
|
|
||||||
...(config && { config }),
|
...(config && { config }),
|
||||||
...(workspace && { workspace }),
|
...(workspace && { workspace }),
|
||||||
...(output && { output }),
|
...(output && { output }),
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ import { dispatchMessage } from './message-handlers.js';
|
|||||||
import { type ModelTier, resolveModel } from './models.js';
|
import { type ModelTier, resolveModel } from './models.js';
|
||||||
import { detectExecutionContext, formatCompletionMessage, formatErrorOutput } from './output-formatters.js';
|
import { detectExecutionContext, formatCompletionMessage, formatErrorOutput } from './output-formatters.js';
|
||||||
import { createProgressManager } from './progress-manager.js';
|
import { createProgressManager } from './progress-manager.js';
|
||||||
import { getActualModelName } from './router-utils.js';
|
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
var SHANNON_DISABLE_LOADER: boolean | undefined;
|
var SHANNON_DISABLE_LOADER: boolean | undefined;
|
||||||
@@ -184,7 +183,6 @@ export async function runClaudePrompt(
|
|||||||
case 'litellm_router':
|
case 'litellm_router':
|
||||||
if (providerConfig.baseUrl) sdkEnv.ANTHROPIC_BASE_URL = providerConfig.baseUrl;
|
if (providerConfig.baseUrl) sdkEnv.ANTHROPIC_BASE_URL = providerConfig.baseUrl;
|
||||||
if (providerConfig.authToken) sdkEnv.ANTHROPIC_AUTH_TOKEN = providerConfig.authToken;
|
if (providerConfig.authToken) sdkEnv.ANTHROPIC_AUTH_TOKEN = providerConfig.authToken;
|
||||||
if (providerConfig.routerDefault) sdkEnv.ROUTER_DEFAULT = providerConfig.routerDefault;
|
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
// 'anthropic_api' or unset — apiKey already handled above
|
// 'anthropic_api' or unset — apiKey already handled above
|
||||||
@@ -385,9 +383,8 @@ async function processMessageStream(
|
|||||||
if (dispatchResult.apiErrorDetected) {
|
if (dispatchResult.apiErrorDetected) {
|
||||||
apiErrorDetected = true;
|
apiErrorDetected = true;
|
||||||
}
|
}
|
||||||
// Capture model from SystemInitMessage, but override with router model if applicable
|
|
||||||
if (dispatchResult.model) {
|
if (dispatchResult.model) {
|
||||||
model = getActualModelName(dispatchResult.model);
|
model = dispatchResult.model;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ import {
|
|||||||
formatToolUseOutput,
|
formatToolUseOutput,
|
||||||
} from './output-formatters.js';
|
} from './output-formatters.js';
|
||||||
import type { ProgressManager } from './progress-manager.js';
|
import type { ProgressManager } from './progress-manager.js';
|
||||||
import { getActualModelName } from './router-utils.js';
|
|
||||||
import type {
|
import type {
|
||||||
ApiErrorDetection,
|
ApiErrorDetection,
|
||||||
AssistantMessage,
|
AssistantMessage,
|
||||||
@@ -309,12 +308,10 @@ export async function dispatchMessage(
|
|||||||
case 'system': {
|
case 'system': {
|
||||||
if (message.subtype === 'init') {
|
if (message.subtype === 'init') {
|
||||||
const initMsg = message as SystemInitMessage;
|
const initMsg = message as SystemInitMessage;
|
||||||
const actualModel = getActualModelName(initMsg.model);
|
|
||||||
if (!execContext.useCleanOutput) {
|
if (!execContext.useCleanOutput) {
|
||||||
logger.info(`Model: ${actualModel}, Permission: ${initMsg.permissionMode}`);
|
logger.info(`Model: ${initMsg.model}, Permission: ${initMsg.permissionMode}`);
|
||||||
}
|
}
|
||||||
// Return actual model for tracking in audit logs
|
return { type: 'continue', model: initMsg.model };
|
||||||
return { type: 'continue', model: actualModel };
|
|
||||||
}
|
}
|
||||||
return { type: 'continue' };
|
return { type: 'continue' };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
// Copyright (C) 2025 Keygraph, Inc.
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License version 3
|
|
||||||
// as published by the Free Software Foundation.
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the actual model name being used.
|
|
||||||
* When using claude-code-router, the SDK reports its configured model (claude-sonnet)
|
|
||||||
* but the actual model is determined by ROUTER_DEFAULT env var.
|
|
||||||
*/
|
|
||||||
export function getActualModelName(sdkReportedModel?: string): string | undefined {
|
|
||||||
const routerBaseUrl = process.env.ANTHROPIC_BASE_URL;
|
|
||||||
const routerDefault = process.env.ROUTER_DEFAULT;
|
|
||||||
|
|
||||||
// If router mode is active and ROUTER_DEFAULT is set, use that
|
|
||||||
if (routerBaseUrl && routerDefault) {
|
|
||||||
// ROUTER_DEFAULT format: "provider,model" (e.g., "gemini,gemini-2.5-pro")
|
|
||||||
const parts = routerDefault.split(',');
|
|
||||||
if (parts.length >= 2) {
|
|
||||||
return parts.slice(1).join(','); // Handle model names with commas
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall back to SDK-reported model
|
|
||||||
return sdkReportedModel;
|
|
||||||
}
|
|
||||||
@@ -1,21 +1,59 @@
|
|||||||
/**
|
/**
|
||||||
* CheckpointProvider — injectable interface for external state persistence.
|
* CheckpointProvider — injectable interface for external state persistence.
|
||||||
*
|
*
|
||||||
* Called after each agent completes to allow external progress tracking.
|
* Called before and after each agent to support skip-guard (resume) and
|
||||||
* During the concurrent vulnerability-exploitation phase, 5 pipelines run
|
* post-agent artifact persistence. During the concurrent vulnerability-exploitation
|
||||||
* in parallel — onAgentComplete fires per-agent for granular progress.
|
* phase, 5 pipelines run in parallel — methods fire per-agent for granular control.
|
||||||
*
|
*
|
||||||
* Default: no-op.
|
* Default: no-op (skip nothing, persist nothing).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { PipelineState } from '../temporal/shared.js';
|
import type { AgentMetrics, PipelineState } from '../temporal/shared.js';
|
||||||
|
|
||||||
|
/** Result of a pre-agent skip check. */
|
||||||
|
export interface SkipDecision {
|
||||||
|
readonly skip: boolean;
|
||||||
|
readonly metrics?: AgentMetrics; // Required when skip=true
|
||||||
|
}
|
||||||
|
|
||||||
|
/** File-system context passed after agent completion for artifact persistence. */
|
||||||
|
export interface CheckpointContext {
|
||||||
|
readonly repoPath: string;
|
||||||
|
readonly sessionId: string;
|
||||||
|
readonly deliverablesSubdir: string;
|
||||||
|
readonly outputPath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface CheckpointProvider {
|
export interface CheckpointProvider {
|
||||||
onAgentComplete(agentName: string, phase: string, state: PipelineState): Promise<void>;
|
/**
|
||||||
|
* Called before an agent activity executes.
|
||||||
|
* Return { skip: true, metrics } to skip the agent (e.g., output files already exist).
|
||||||
|
* Return { skip: false } to run normally.
|
||||||
|
*/
|
||||||
|
shouldSkipAgent(
|
||||||
|
agentName: string,
|
||||||
|
repoPath: string,
|
||||||
|
deliverablesSubdir: string,
|
||||||
|
): Promise<SkipDecision>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called after an agent activity succeeds.
|
||||||
|
* Receives pipeline state and optional file context for artifact persistence.
|
||||||
|
*/
|
||||||
|
onAgentComplete(
|
||||||
|
agentName: string,
|
||||||
|
phase: string,
|
||||||
|
state: PipelineState,
|
||||||
|
context?: CheckpointContext,
|
||||||
|
): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Default no-op implementation — no external checkpointing. */
|
/** Default no-op implementation — no external checkpointing. */
|
||||||
export class NoOpCheckpointProvider implements CheckpointProvider {
|
export class NoOpCheckpointProvider implements CheckpointProvider {
|
||||||
|
async shouldSkipAgent(): Promise<SkipDecision> {
|
||||||
|
return { skip: false };
|
||||||
|
}
|
||||||
|
|
||||||
async onAgentComplete(): Promise<void> {
|
async onAgentComplete(): Promise<void> {
|
||||||
// No-op
|
// No-op
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
/**
|
/**
|
||||||
* FindingsProvider — injectable interface for external findings integration.
|
* FindingsProvider — injectable interface for external findings integration.
|
||||||
*
|
*
|
||||||
* Allows external security data (SAST, SCA, secrets, etc.) to be merged
|
* Allows external security data from consumer-supplied sources to be merged
|
||||||
* into the exploitation pipeline between vulnerability analysis and exploitation.
|
* into the exploitation pipeline between vulnerability analysis and exploitation.
|
||||||
*
|
*
|
||||||
* Default: no-op returning { mergedCount: 0 }.
|
* Default: no-op returning { mergedCount: 0 }.
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
* Consumers can provide alternate implementations via the DI container.
|
* Consumers can provide alternate implementations via the DI container.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type { CheckpointProvider } from './checkpoint-provider.js';
|
export type { CheckpointProvider, CheckpointContext, SkipDecision } from './checkpoint-provider.js';
|
||||||
export { NoOpCheckpointProvider } from './checkpoint-provider.js';
|
export { NoOpCheckpointProvider } from './checkpoint-provider.js';
|
||||||
export type { FindingsProvider } from './findings-provider.js';
|
export type { FindingsProvider } from './findings-provider.js';
|
||||||
export { NoOpFindingsProvider } from './findings-provider.js';
|
export { NoOpFindingsProvider } from './findings-provider.js';
|
||||||
|
export type { ReportOutputProvider } from './report-output-provider.js';
|
||||||
|
export { NoOpReportOutputProvider } from './report-output-provider.js';
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
/**
|
||||||
|
* ReportOutputProvider — injectable interface for emitting an optional
|
||||||
|
* additional artifact alongside the assembled markdown report.
|
||||||
|
*
|
||||||
|
* Runs after the report agent has finalized
|
||||||
|
* `comprehensive_security_assessment_report.md`. Consumers can override to
|
||||||
|
* produce derived outputs; the default no-op produces nothing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ActivityInput } from '../temporal/activities.js';
|
||||||
|
import type { ActivityLogger } from '../types/activity-logger.js';
|
||||||
|
|
||||||
|
export interface ReportOutputProvider {
|
||||||
|
generate(input: ActivityInput, logger: ActivityLogger): Promise<{ outputPath?: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Default no-op implementation — no additional output produced. */
|
||||||
|
export class NoOpReportOutputProvider implements ReportOutputProvider {
|
||||||
|
async generate(): Promise<{ outputPath?: string }> {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ import type { CheckpointProvider } from '../interfaces/checkpoint-provider.js';
|
|||||||
import { NoOpCheckpointProvider } from '../interfaces/checkpoint-provider.js';
|
import { NoOpCheckpointProvider } from '../interfaces/checkpoint-provider.js';
|
||||||
import type { FindingsProvider } from '../interfaces/findings-provider.js';
|
import type { FindingsProvider } from '../interfaces/findings-provider.js';
|
||||||
import { NoOpFindingsProvider } from '../interfaces/findings-provider.js';
|
import { NoOpFindingsProvider } from '../interfaces/findings-provider.js';
|
||||||
|
import type { ReportOutputProvider } from '../interfaces/report-output-provider.js';
|
||||||
|
import { NoOpReportOutputProvider } from '../interfaces/report-output-provider.js';
|
||||||
import type { ContainerConfig } from '../types/config.js';
|
import type { ContainerConfig } from '../types/config.js';
|
||||||
import { AgentExecutionService } from './agent-execution.js';
|
import { AgentExecutionService } from './agent-execution.js';
|
||||||
import { ConfigLoaderService } from './config-loader.js';
|
import { ConfigLoaderService } from './config-loader.js';
|
||||||
@@ -40,6 +42,7 @@ export interface ContainerDependencies {
|
|||||||
readonly config: ContainerConfig;
|
readonly config: ContainerConfig;
|
||||||
readonly findingsProvider?: FindingsProvider;
|
readonly findingsProvider?: FindingsProvider;
|
||||||
readonly checkpointProvider?: CheckpointProvider;
|
readonly checkpointProvider?: CheckpointProvider;
|
||||||
|
readonly reportOutputProvider?: ReportOutputProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -59,6 +62,7 @@ export class Container {
|
|||||||
readonly exploitationChecker: ExploitationCheckerService;
|
readonly exploitationChecker: ExploitationCheckerService;
|
||||||
readonly findingsProvider: FindingsProvider;
|
readonly findingsProvider: FindingsProvider;
|
||||||
readonly checkpointProvider: CheckpointProvider;
|
readonly checkpointProvider: CheckpointProvider;
|
||||||
|
readonly reportOutputProvider: ReportOutputProvider;
|
||||||
|
|
||||||
constructor(deps: ContainerDependencies) {
|
constructor(deps: ContainerDependencies) {
|
||||||
this.sessionMetadata = deps.sessionMetadata;
|
this.sessionMetadata = deps.sessionMetadata;
|
||||||
@@ -72,6 +76,7 @@ export class Container {
|
|||||||
// Wire providers with default no-ops when not provided
|
// Wire providers with default no-ops when not provided
|
||||||
this.findingsProvider = deps.findingsProvider ?? new NoOpFindingsProvider();
|
this.findingsProvider = deps.findingsProvider ?? new NoOpFindingsProvider();
|
||||||
this.checkpointProvider = deps.checkpointProvider ?? new NoOpCheckpointProvider();
|
this.checkpointProvider = deps.checkpointProvider ?? new NoOpCheckpointProvider();
|
||||||
|
this.reportOutputProvider = deps.reportOutputProvider ?? new NoOpReportOutputProvider();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -87,6 +92,32 @@ const DEFAULT_CONFIG: ContainerConfig = {
|
|||||||
auditDir: './workspaces',
|
auditDir: './workspaces',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory function for creating containers.
|
||||||
|
*
|
||||||
|
* Default: creates a plain Container with NoOp providers. Consumers can call
|
||||||
|
* setContainerFactory() at worker startup to inject custom provider
|
||||||
|
* implementations into every container.
|
||||||
|
*/
|
||||||
|
type ContainerFactory = (
|
||||||
|
workflowId: string,
|
||||||
|
sessionMetadata: SessionMetadata,
|
||||||
|
config: ContainerConfig,
|
||||||
|
) => Container;
|
||||||
|
|
||||||
|
let containerFactory: ContainerFactory = (_workflowId, sessionMetadata, config) =>
|
||||||
|
new Container({ sessionMetadata, config });
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Override the default container factory.
|
||||||
|
*
|
||||||
|
* Call once at worker startup to inject providers into all containers
|
||||||
|
* created during the worker's lifetime.
|
||||||
|
*/
|
||||||
|
export function setContainerFactory(factory: ContainerFactory): void {
|
||||||
|
containerFactory = factory;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get or create a Container for a workflow.
|
* Get or create a Container for a workflow.
|
||||||
*
|
*
|
||||||
@@ -106,7 +137,7 @@ export function getOrCreateContainer(
|
|||||||
let container = containers.get(workflowId);
|
let container = containers.get(workflowId);
|
||||||
|
|
||||||
if (!container) {
|
if (!container) {
|
||||||
container = new Container({ sessionMetadata, config });
|
container = containerFactory(workflowId, sessionMetadata, config);
|
||||||
containers.set(workflowId, container);
|
containers.set(workflowId, container);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,9 @@ export { AgentExecutionService } from './agent-execution.js';
|
|||||||
|
|
||||||
export { ConfigLoaderService } from './config-loader.js';
|
export { ConfigLoaderService } from './config-loader.js';
|
||||||
export type { ContainerDependencies } from './container.js';
|
export type { ContainerDependencies } from './container.js';
|
||||||
export { Container, getContainer, getOrCreateContainer, removeContainer } from './container.js';
|
export { Container, getContainer, getOrCreateContainer, removeContainer, setContainerFactory } from './container.js';
|
||||||
export { ExploitationCheckerService } from './exploitation-checker.js';
|
export { ExploitationCheckerService } from './exploitation-checker.js';
|
||||||
export { loadPrompt } from './prompt-manager.js';
|
export { loadPrompt } from './prompt-manager.js';
|
||||||
export { assembleFinalReport, injectModelIntoReport } from './reporting.js';
|
export { assembleFinalReport, injectModelIntoReport } from './reporting.js';
|
||||||
|
export type { ClaudePromptResult } from '../ai/claude-executor.js';
|
||||||
|
export { runClaudePrompt } from '../ai/claude-executor.js';
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
* Checks run sequentially, cheapest first:
|
* Checks run sequentially, cheapest first:
|
||||||
* 1. Repository path exists and contains .git
|
* 1. Repository path exists and contains .git
|
||||||
* 2. Config file parses and validates (if provided)
|
* 2. Config file parses and validates (if provided)
|
||||||
* 3. Credentials validate via Claude Agent SDK query (API key, OAuth, Bedrock, Vertex AI, or router mode)
|
* 3. Credentials validate via Claude Agent SDK query (API key, OAuth, Bedrock, or Vertex AI)
|
||||||
* 4. Target URL is reachable from the container (DNS + HTTP)
|
* 4. Target URL is reachable from the container (DNS + HTTP)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@@ -473,7 +473,7 @@ async function validateTargetUrl(targetUrl: string, logger: ActivityLogger): Pro
|
|||||||
*
|
*
|
||||||
* 1. Repository path exists and contains .git
|
* 1. Repository path exists and contains .git
|
||||||
* 2. Config file parses and validates (if configPath provided)
|
* 2. Config file parses and validates (if configPath provided)
|
||||||
* 3. Credentials validate (API key, OAuth, or router mode)
|
* 3. Credentials validate (API key, OAuth, Bedrock, or Vertex AI)
|
||||||
* 4. Target URL is reachable from the container
|
* 4. Target URL is reachable from the container
|
||||||
*
|
*
|
||||||
* Returns on first failure.
|
* Returns on first failure.
|
||||||
|
|||||||
@@ -17,7 +17,11 @@ interface DeliverableFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Pure function: Assemble final report from specialist deliverables
|
// Pure function: Assemble final report from specialist deliverables
|
||||||
export async function assembleFinalReport(sourceDir: string, logger: ActivityLogger): Promise<string> {
|
export async function assembleFinalReport(
|
||||||
|
sourceDir: string,
|
||||||
|
deliverablesSubdir: string | undefined,
|
||||||
|
logger: ActivityLogger,
|
||||||
|
): Promise<string> {
|
||||||
const deliverableFiles: DeliverableFile[] = [
|
const deliverableFiles: DeliverableFile[] = [
|
||||||
{ name: 'Injection', path: 'injection_exploitation_evidence.md', required: false },
|
{ name: 'Injection', path: 'injection_exploitation_evidence.md', required: false },
|
||||||
{ name: 'XSS', path: 'xss_exploitation_evidence.md', required: false },
|
{ name: 'XSS', path: 'xss_exploitation_evidence.md', required: false },
|
||||||
@@ -29,7 +33,7 @@ export async function assembleFinalReport(sourceDir: string, logger: ActivityLog
|
|||||||
const sections: string[] = [];
|
const sections: string[] = [];
|
||||||
|
|
||||||
for (const file of deliverableFiles) {
|
for (const file of deliverableFiles) {
|
||||||
const filePath = path.join(deliverablesDir(sourceDir), file.path);
|
const filePath = path.join(deliverablesDir(sourceDir, deliverablesSubdir), file.path);
|
||||||
try {
|
try {
|
||||||
if (await fs.pathExists(filePath)) {
|
if (await fs.pathExists(filePath)) {
|
||||||
const content = await fs.readFile(filePath, 'utf8');
|
const content = await fs.readFile(filePath, 'utf8');
|
||||||
@@ -56,7 +60,7 @@ export async function assembleFinalReport(sourceDir: string, logger: ActivityLog
|
|||||||
}
|
}
|
||||||
|
|
||||||
const finalContent = sections.join('\n\n');
|
const finalContent = sections.join('\n\n');
|
||||||
const outputDir = deliverablesDir(sourceDir);
|
const outputDir = deliverablesDir(sourceDir, deliverablesSubdir);
|
||||||
const finalReportPath = path.join(outputDir, 'comprehensive_security_assessment_report.md');
|
const finalReportPath = path.join(outputDir, 'comprehensive_security_assessment_report.md');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -82,6 +86,7 @@ export async function assembleFinalReport(sourceDir: string, logger: ActivityLog
|
|||||||
*/
|
*/
|
||||||
export async function injectModelIntoReport(
|
export async function injectModelIntoReport(
|
||||||
repoPath: string,
|
repoPath: string,
|
||||||
|
deliverablesSubdir: string | undefined,
|
||||||
outputPath: string,
|
outputPath: string,
|
||||||
logger: ActivityLogger,
|
logger: ActivityLogger,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
@@ -118,7 +123,7 @@ export async function injectModelIntoReport(
|
|||||||
logger.info(`Injecting model info into report: ${modelStr}`);
|
logger.info(`Injecting model info into report: ${modelStr}`);
|
||||||
|
|
||||||
// 3. Read the final report
|
// 3. Read the final report
|
||||||
const reportPath = path.join(deliverablesDir(repoPath), 'comprehensive_security_assessment_report.md');
|
const reportPath = path.join(deliverablesDir(repoPath, deliverablesSubdir), 'comprehensive_security_assessment_report.md');
|
||||||
|
|
||||||
if (!(await fs.pathExists(reportPath))) {
|
if (!(await fs.pathExists(reportPath))) {
|
||||||
logger.warn('Final report not found, skipping model injection');
|
logger.warn('Final report not found, skipping model injection');
|
||||||
|
|||||||
@@ -103,7 +103,6 @@ export const AGENTS: Readonly<Record<AgentName, AgentDefinition>> = Object.freez
|
|||||||
prerequisites: ['injection-exploit', 'xss-exploit', 'auth-exploit', 'ssrf-exploit', 'authz-exploit'],
|
prerequisites: ['injection-exploit', 'xss-exploit', 'auth-exploit', 'ssrf-exploit', 'authz-exploit'],
|
||||||
promptTemplate: 'report-executive',
|
promptTemplate: 'report-executive',
|
||||||
deliverableFilename: 'comprehensive_security_assessment_report.md',
|
deliverableFilename: 'comprehensive_security_assessment_report.md',
|
||||||
modelTier: 'small',
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ import { AuditSession } from '../audit/index.js';
|
|||||||
import type { ResumeAttempt } from '../audit/metrics-tracker.js';
|
import type { ResumeAttempt } from '../audit/metrics-tracker.js';
|
||||||
import type { SessionMetadata } from '../audit/utils.js';
|
import type { SessionMetadata } from '../audit/utils.js';
|
||||||
import type { WorkflowSummary } from '../audit/workflow-logger.js';
|
import type { WorkflowSummary } from '../audit/workflow-logger.js';
|
||||||
import { DEFAULT_DELIVERABLES_SUBDIR, deliverablesDir } from '../paths.js';
|
import type { ContainerConfig, ProviderConfig } from '../types/config.js';
|
||||||
|
import type { CheckpointContext } from '../interfaces/checkpoint-provider.js';
|
||||||
import { getContainer, getOrCreateContainer, removeContainer } from '../services/container.js';
|
import { getContainer, getOrCreateContainer, removeContainer } from '../services/container.js';
|
||||||
import { classifyErrorForTemporal, PentestError } from '../services/error-handling.js';
|
import { classifyErrorForTemporal, PentestError } from '../services/error-handling.js';
|
||||||
import { ExploitationCheckerService } from '../services/exploitation-checker.js';
|
import { ExploitationCheckerService } from '../services/exploitation-checker.js';
|
||||||
@@ -33,9 +34,9 @@ import { assembleFinalReport, injectModelIntoReport } from '../services/reportin
|
|||||||
import { AGENTS } from '../session-manager.js';
|
import { AGENTS } from '../session-manager.js';
|
||||||
import type { AgentName } from '../types/agents.js';
|
import type { AgentName } from '../types/agents.js';
|
||||||
import { ALL_AGENTS } from '../types/agents.js';
|
import { ALL_AGENTS } from '../types/agents.js';
|
||||||
import type { ContainerConfig, ProviderConfig } from '../types/config.js';
|
|
||||||
import { ErrorCode } from '../types/errors.js';
|
import { ErrorCode } from '../types/errors.js';
|
||||||
import { isErr } from '../types/result.js';
|
import { isErr } from '../types/result.js';
|
||||||
|
import { DEFAULT_DELIVERABLES_SUBDIR, deliverablesDir } from '../paths.js';
|
||||||
import { fileExists, readJson } from '../utils/file-io.js';
|
import { fileExists, readJson } from '../utils/file-io.js';
|
||||||
import { createActivityLogger } from './activity-logger.js';
|
import { createActivityLogger } from './activity-logger.js';
|
||||||
import type { AgentMetrics, PipelineState, ResumeState } from './shared.js';
|
import type { AgentMetrics, PipelineState, ResumeState } from './shared.js';
|
||||||
@@ -131,6 +132,20 @@ function buildContainerConfig(input: ActivityInput): ContainerConfig {
|
|||||||
*/
|
*/
|
||||||
async function runAgentActivity(agentName: AgentName, input: ActivityInput): Promise<AgentMetrics> {
|
async function runAgentActivity(agentName: AgentName, input: ActivityInput): Promise<AgentMetrics> {
|
||||||
const { repoPath, configPath, pipelineTestingMode = false, workflowId, webUrl } = input;
|
const { repoPath, configPath, pipelineTestingMode = false, workflowId, webUrl } = input;
|
||||||
|
|
||||||
|
// Skip guard: the checkpoint provider decides whether to run the agent.
|
||||||
|
// The default NoOp provider always returns { skip: false }.
|
||||||
|
const skipContainer = getContainer(workflowId) ??
|
||||||
|
getOrCreateContainer(workflowId, buildSessionMetadata(input), buildContainerConfig(input));
|
||||||
|
const decision = await skipContainer.checkpointProvider.shouldSkipAgent(
|
||||||
|
agentName,
|
||||||
|
repoPath,
|
||||||
|
input.deliverablesSubdir ?? DEFAULT_DELIVERABLES_SUBDIR,
|
||||||
|
);
|
||||||
|
if (decision.skip && decision.metrics) {
|
||||||
|
return decision.metrics;
|
||||||
|
}
|
||||||
|
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const attemptNumber = Context.current().info.attempt;
|
const attemptNumber = Context.current().info.attempt;
|
||||||
|
|
||||||
@@ -288,7 +303,7 @@ export async function runReportAgent(input: ActivityInput): Promise<AgentMetrics
|
|||||||
* Runs cheap checks before any agent execution:
|
* Runs cheap checks before any agent execution:
|
||||||
* 1. Repository path exists with .git
|
* 1. Repository path exists with .git
|
||||||
* 2. Config file validates (if provided)
|
* 2. Config file validates (if provided)
|
||||||
* 3. Credential validation (API key, OAuth, or router mode)
|
* 3. Credential validation (API key, OAuth, Bedrock, or Vertex AI)
|
||||||
* 4. Target URL reachable from the container
|
* 4. Target URL reachable from the container
|
||||||
*
|
*
|
||||||
* NOT using runAgentActivity — preflight doesn't run an agent via the SDK.
|
* NOT using runAgentActivity — preflight doesn't run an agent via the SDK.
|
||||||
@@ -306,15 +321,7 @@ export async function runPreflightValidation(input: ActivityInput): Promise<void
|
|||||||
const logger = createActivityLogger();
|
const logger = createActivityLogger();
|
||||||
logger.info('Running preflight validation...', { attempt: attemptNumber });
|
logger.info('Running preflight validation...', { attempt: attemptNumber });
|
||||||
|
|
||||||
const result = await runPreflightChecks(
|
const result = await runPreflightChecks(input.webUrl, input.repoPath, input.configPath, logger, input.skipGitCheck, input.apiKey, input.providerConfig);
|
||||||
input.webUrl,
|
|
||||||
input.repoPath,
|
|
||||||
input.configPath,
|
|
||||||
logger,
|
|
||||||
input.skipGitCheck,
|
|
||||||
input.apiKey,
|
|
||||||
input.providerConfig,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (isErr(result)) {
|
if (isErr(result)) {
|
||||||
const classified = classifyErrorForTemporal(result.error);
|
const classified = classifyErrorForTemporal(result.error);
|
||||||
@@ -386,11 +393,11 @@ export async function initDeliverableGit(input: ActivityInput): Promise<void> {
|
|||||||
* Assemble the final report by concatenating exploitation evidence files.
|
* Assemble the final report by concatenating exploitation evidence files.
|
||||||
*/
|
*/
|
||||||
export async function assembleReportActivity(input: ActivityInput): Promise<void> {
|
export async function assembleReportActivity(input: ActivityInput): Promise<void> {
|
||||||
const { repoPath } = input;
|
const { repoPath, deliverablesSubdir } = input;
|
||||||
const logger = createActivityLogger();
|
const logger = createActivityLogger();
|
||||||
logger.info('Assembling deliverables from specialist agents...');
|
logger.info('Assembling deliverables from specialist agents...');
|
||||||
try {
|
try {
|
||||||
await assembleFinalReport(repoPath, logger);
|
await assembleFinalReport(repoPath, deliverablesSubdir, logger);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const err = error as Error;
|
const err = error as Error;
|
||||||
logger.warn(`Error assembling final report: ${err.message}`);
|
logger.warn(`Error assembling final report: ${err.message}`);
|
||||||
@@ -401,11 +408,11 @@ export async function assembleReportActivity(input: ActivityInput): Promise<void
|
|||||||
* Inject model metadata into the final report.
|
* Inject model metadata into the final report.
|
||||||
*/
|
*/
|
||||||
export async function injectReportMetadataActivity(input: ActivityInput): Promise<void> {
|
export async function injectReportMetadataActivity(input: ActivityInput): Promise<void> {
|
||||||
const { repoPath, sessionId, outputPath } = input;
|
const { repoPath, sessionId, outputPath, deliverablesSubdir } = input;
|
||||||
const logger = createActivityLogger();
|
const logger = createActivityLogger();
|
||||||
const effectiveOutputPath = outputPath ? path.join(outputPath, sessionId) : path.join('./workspaces', sessionId);
|
const effectiveOutputPath = outputPath ? path.join(outputPath, sessionId) : path.join('./workspaces', sessionId);
|
||||||
try {
|
try {
|
||||||
await injectModelIntoReport(repoPath, effectiveOutputPath, logger);
|
await injectModelIntoReport(repoPath, deliverablesSubdir, effectiveOutputPath, logger);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const err = error as Error;
|
const err = error as Error;
|
||||||
logger.warn(`Error injecting model into report: ${err.message}`);
|
logger.warn(`Error injecting model into report: ${err.message}`);
|
||||||
@@ -593,6 +600,18 @@ export async function restoreGitCheckpoint(
|
|||||||
const logger = createActivityLogger();
|
const logger = createActivityLogger();
|
||||||
logger.info(`Restoring deliverables to ${checkpointHash}...`);
|
logger.info(`Restoring deliverables to ${checkpointHash}...`);
|
||||||
|
|
||||||
|
// Validate hash exists in this clone before attempting reset
|
||||||
|
try {
|
||||||
|
await executeGitCommandWithRetry(
|
||||||
|
['git', 'rev-parse', '--verify', checkpointHash],
|
||||||
|
repoPath,
|
||||||
|
'verify checkpoint hash exists'
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
logger.info(`Checkpoint hash not found in clone, skipping git reset: ${checkpointHash}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
await executeGitCommandWithRetry(
|
await executeGitCommandWithRetry(
|
||||||
['git', 'reset', '--hard', checkpointHash],
|
['git', 'reset', '--hard', checkpointHash],
|
||||||
deliverablesPath,
|
deliverablesPath,
|
||||||
@@ -744,5 +763,42 @@ export async function saveCheckpoint(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const container = getContainer(input.workflowId);
|
const container = getContainer(input.workflowId);
|
||||||
if (!container?.checkpointProvider) return;
|
if (!container?.checkpointProvider) return;
|
||||||
return container.checkpointProvider.onAgentComplete(agentName, phase, state);
|
|
||||||
|
const context: CheckpointContext = {
|
||||||
|
repoPath: input.repoPath,
|
||||||
|
sessionId: input.sessionId,
|
||||||
|
deliverablesSubdir: input.deliverablesSubdir ?? DEFAULT_DELIVERABLES_SUBDIR,
|
||||||
|
...(input.outputPath !== undefined && { outputPath: input.outputPath }),
|
||||||
|
};
|
||||||
|
|
||||||
|
return container.checkpointProvider.onAgentComplete(agentName, phase, state, context);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate an optional additional output alongside the assembled markdown report.
|
||||||
|
*
|
||||||
|
* Delegates to the ReportOutputProvider registered in the DI container.
|
||||||
|
* Default: no-op. Consumers can override this activity at the worker level
|
||||||
|
* to emit derived outputs from the final report.
|
||||||
|
*/
|
||||||
|
export async function generateReportOutputActivity(input: ActivityInput): Promise<void> {
|
||||||
|
const container = getContainer(input.workflowId);
|
||||||
|
if (!container?.reportOutputProvider) return;
|
||||||
|
|
||||||
|
const logger = createActivityLogger();
|
||||||
|
|
||||||
|
// Resolve promptDir against the worker root so providers are cwd-independent.
|
||||||
|
const resolvedInput: ActivityInput = {
|
||||||
|
...input,
|
||||||
|
...(input.promptDir !== undefined && {
|
||||||
|
promptDir: path.isAbsolute(input.promptDir)
|
||||||
|
? input.promptDir
|
||||||
|
: path.resolve(process.env.SHANNON_WORKER_ROOT ?? process.cwd(), input.promptDir),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await container.reportOutputProvider.generate(resolvedInput, logger);
|
||||||
|
if (result.outputPath) {
|
||||||
|
logger.info(`Report output written to ${result.outputPath}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,10 +25,10 @@ export interface PipelineInput {
|
|||||||
deliverablesSubdir?: string; // Override deliverables path (default: '.shannon/deliverables')
|
deliverablesSubdir?: string; // Override deliverables path (default: '.shannon/deliverables')
|
||||||
auditDir?: string; // Override audit log directory (default: './workspaces')
|
auditDir?: string; // Override audit log directory (default: './workspaces')
|
||||||
promptDir?: string; // Override prompt template directory
|
promptDir?: string; // Override prompt template directory
|
||||||
sastSarifPath?: string; // Path to SARIF file (gates SAST-enhanced mode)
|
sastSarifPath?: string; // Optional path for consumer-supplied findings input
|
||||||
checkpointsEnabled?: boolean; // Enable checkpoint activities (default: false)
|
checkpointsEnabled?: boolean; // Enable checkpoint activities (default: false)
|
||||||
skipGitCheck?: boolean; // Skip .git directory validation in preflight (e.g. when .git is removed after clone)
|
skipGitCheck?: boolean; // Skip .git directory validation in preflight (e.g. when .git is removed after clone)
|
||||||
providerConfig?: ProviderConfig; // LLM provider configuration (Bedrock, Vertex, LiteLLM, etc.)
|
providerConfig?: ProviderConfig; // LLM provider configuration (Bedrock, Vertex, etc.)
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ResumeState {
|
export interface ResumeState {
|
||||||
|
|||||||
@@ -332,30 +332,14 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Aggregate results from settled pipeline promises into workflow state
|
// Aggregate errors from settled pipeline promises.
|
||||||
|
// Metrics and completedAgents are updated incrementally inside runVulnExploitPipeline
|
||||||
|
// so that getProgress queries reflect real-time status during execution.
|
||||||
function aggregatePipelineResults(results: PromiseSettledResult<VulnExploitPipelineResult>[]): void {
|
function aggregatePipelineResults(results: PromiseSettledResult<VulnExploitPipelineResult>[]): void {
|
||||||
const failedPipelines: string[] = [];
|
const failedPipelines: string[] = [];
|
||||||
|
|
||||||
for (const result of results) {
|
for (const result of results) {
|
||||||
if (result.status === 'fulfilled') {
|
if (result.status === 'rejected') {
|
||||||
const { vulnType, vulnMetrics, exploitMetrics } = result.value;
|
|
||||||
|
|
||||||
const vulnAgentName = `${vulnType}-vuln`;
|
|
||||||
if (vulnMetrics) {
|
|
||||||
state.agentMetrics[vulnAgentName] = vulnMetrics;
|
|
||||||
state.completedAgents.push(vulnAgentName);
|
|
||||||
} else if (shouldSkip(vulnAgentName)) {
|
|
||||||
state.completedAgents.push(vulnAgentName);
|
|
||||||
}
|
|
||||||
|
|
||||||
const exploitAgentName = `${vulnType}-exploit`;
|
|
||||||
if (exploitMetrics) {
|
|
||||||
state.agentMetrics[exploitAgentName] = exploitMetrics;
|
|
||||||
state.completedAgents.push(exploitAgentName);
|
|
||||||
} else if (shouldSkip(exploitAgentName)) {
|
|
||||||
state.completedAgents.push(exploitAgentName);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||||
failedPipelines.push(errorMsg);
|
failedPipelines.push(errorMsg);
|
||||||
}
|
}
|
||||||
@@ -442,14 +426,17 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
|
|||||||
let vulnMetrics: AgentMetrics | null = null;
|
let vulnMetrics: AgentMetrics | null = null;
|
||||||
if (!shouldSkip(vulnAgentName)) {
|
if (!shouldSkip(vulnAgentName)) {
|
||||||
vulnMetrics = await runVulnAgent();
|
vulnMetrics = await runVulnAgent();
|
||||||
|
state.agentMetrics[vulnAgentName] = vulnMetrics;
|
||||||
|
state.completedAgents.push(vulnAgentName);
|
||||||
if (input.checkpointsEnabled) {
|
if (input.checkpointsEnabled) {
|
||||||
await a.saveCheckpoint(activityInput, vulnAgentName, 'vulnerability-analysis', state);
|
await a.saveCheckpoint(activityInput, vulnAgentName, 'vulnerability-analysis', state);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.info(`Skipping ${vulnAgentName} (already complete)`);
|
log.info(`Skipping ${vulnAgentName} (already complete)`);
|
||||||
|
state.completedAgents.push(vulnAgentName);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1.5. Merge external findings (SAST, SCA, etc.) into exploitation queue
|
// 1.5. Merge external findings from consumer provider into exploitation queue
|
||||||
await a.mergeFindingsIntoQueue(activityInput, vulnType);
|
await a.mergeFindingsIntoQueue(activityInput, vulnType);
|
||||||
|
|
||||||
// 2. Check exploitation queue for actionable findings
|
// 2. Check exploitation queue for actionable findings
|
||||||
@@ -460,11 +447,14 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
|
|||||||
if (decision.shouldExploit) {
|
if (decision.shouldExploit) {
|
||||||
if (!shouldSkip(exploitAgentName)) {
|
if (!shouldSkip(exploitAgentName)) {
|
||||||
exploitMetrics = await runExploitAgent();
|
exploitMetrics = await runExploitAgent();
|
||||||
|
state.agentMetrics[exploitAgentName] = exploitMetrics;
|
||||||
|
state.completedAgents.push(exploitAgentName);
|
||||||
if (input.checkpointsEnabled) {
|
if (input.checkpointsEnabled) {
|
||||||
await a.saveCheckpoint(activityInput, exploitAgentName, 'exploitation', state);
|
await a.saveCheckpoint(activityInput, exploitAgentName, 'exploitation', state);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.info(`Skipping ${exploitAgentName} (already complete)`);
|
log.info(`Skipping ${exploitAgentName} (already complete)`);
|
||||||
|
state.completedAgents.push(exploitAgentName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -526,6 +516,13 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
|
|||||||
state.completedAgents.push('report');
|
state.completedAgents.push('report');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Runs after the skip gate so consumer providers still execute on resume.
|
||||||
|
await a.generateReportOutputActivity(activityInput);
|
||||||
|
|
||||||
|
if (input.checkpointsEnabled) {
|
||||||
|
await a.saveCheckpoint(activityInput, 'report-output', 'reporting', state);
|
||||||
|
}
|
||||||
|
|
||||||
state.status = 'completed';
|
state.status = 'completed';
|
||||||
state.currentPhase = null;
|
state.currentPhase = null;
|
||||||
state.currentAgent = null;
|
state.currentAgent = null;
|
||||||
|
|||||||
@@ -80,7 +80,6 @@ export interface ProviderConfig {
|
|||||||
readonly gcpCredentialsPath?: string;
|
readonly gcpCredentialsPath?: string;
|
||||||
readonly baseUrl?: string;
|
readonly baseUrl?: string;
|
||||||
readonly authToken?: string;
|
readonly authToken?: string;
|
||||||
readonly routerDefault?: string;
|
|
||||||
readonly modelOverrides?: Record<string, string>;
|
readonly modelOverrides?: Record<string, string>;
|
||||||
readonly supportsStructuredOutput?: boolean;
|
readonly supportsStructuredOutput?: boolean;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ export const BILLING_TEXT_PATTERNS = [
|
|||||||
'cap reached',
|
'cap reached',
|
||||||
'budget exceeded',
|
'budget exceeded',
|
||||||
'usage limit',
|
'usage limit',
|
||||||
'resets',
|
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user