feat: extract pipeline core for library consumption (#282)

* feat: extract pipeline core for library consumption

* fix: chmod workspace directory for container write access

* fix: resolve playwright output dir relative to deliverables parent

* feat: add multi-provider LLM support via ProviderConfig

* fix: resolve model overrides via options.model, remove unused model env passthrough

* fix: use ANTHROPIC_AUTH_TOKEN for custom base URL and router auth

* fix: skip env-based credential validation when providerConfig is present

* fix: support large UID/GID values for AD/LDAP users in container

apps/worker/src/services/agent-execution.ts

@@ -46,8 +46,13 @@ export interface AgentExecutionInput {
   repoPath: string;
   deliverablesPath: string;
   configPath?: string | undefined;
+  configData?: import('../types/config.js').DistributedConfig | undefined;
+  configYAML?: string | undefined;
   pipelineTestingMode?: boolean | undefined;
   attemptNumber: number;
+  apiKey?: string | undefined;
+  promptDir?: string | undefined;
+  providerConfig?: import('../types/config.js').ProviderConfig | undefined;
 }
 
 interface FailAgentOpts {
@@ -90,10 +95,10 @@ export class AgentExecutionService {
     auditSession: AuditSession,
     logger: ActivityLogger,
   ): Promise<Result<AgentEndResult, PentestError>> {
-    const { webUrl, repoPath, deliverablesPath, configPath, pipelineTestingMode = false, attemptNumber } = input;
+    const { webUrl, repoPath, deliverablesPath, configPath, configData, configYAML, pipelineTestingMode = false, attemptNumber, apiKey, promptDir, providerConfig } = input;
 
-    // 1. Load config (if provided)
-    const configResult = await this.configLoader.loadOptional(configPath);
+    // 1. Load config (pre-parsed configData → raw YAML → file path)
+    const configResult = await this.configLoader.loadOptional(configPath, configData, configYAML);
     if (isErr(configResult)) {
       return configResult;
     }
@@ -103,7 +108,7 @@ export class AgentExecutionService {
     const promptTemplate = AGENTS[agentName].promptTemplate;
     let prompt: string;
     try {
-      prompt = await loadPrompt(promptTemplate, { webUrl, repoPath }, distributedConfig, pipelineTestingMode, logger);
+      prompt = await loadPrompt(promptTemplate, { webUrl, repoPath }, distributedConfig, pipelineTestingMode, logger, promptDir);
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : String(error);
       return err(
@@ -148,6 +153,9 @@ export class AgentExecutionService {
       logger,
       AGENTS[agentName].modelTier,
       outputFormat,
+      apiKey,
+      path.relative(repoPath, deliverablesPath),
+      providerConfig,
     );
 
     // 6. Spending cap check - defense-in-depth
@@ -184,15 +192,14 @@ export class AgentExecutionService {
     // 8. Write structured output to disk (vuln agents only)
     const queueFilename = getQueueFilename(agentName);
     if (result.structuredOutput !== undefined && queueFilename) {
-      const deliverablesDir = path.join(repoPath, '.shannon', 'deliverables');
-      await fs.ensureDir(deliverablesDir);
-      const queuePath = path.join(deliverablesDir, queueFilename);
+      await fs.ensureDir(deliverablesPath);
+      const queuePath = path.join(deliverablesPath, queueFilename);
       await fs.writeFile(queuePath, JSON.stringify(result.structuredOutput, null, 2), 'utf8');
       logger.info(`Wrote structured output queue to ${queueFilename}`);
     }
 
     // 9. Validate output
-    const validationPassed = await validateAgentOutput(result, agentName, repoPath, logger);
+    const validationPassed = await validateAgentOutput(result, agentName, deliverablesPath, logger);
     if (!validationPassed) {
       return this.failAgent(agentName, deliverablesPath, auditSession, logger, {
         attemptNumber,

apps/worker/src/services/config-loader.ts

@@ -11,7 +11,7 @@
  * Pure service with no Temporal dependencies.
  */
 
-import { distributeConfig, parseConfig } from '../config-parser.js';
+import { distributeConfig, parseConfig, parseConfigYAML } from '../config-parser.js';
 import type { DistributedConfig } from '../types/config.js';
 import { ErrorCode } from '../types/errors.js';
 import { err, ok, type Result } from '../types/result.js';
@@ -60,11 +60,31 @@ export class ConfigLoaderService {
 
   /**
    * Load config if path is provided, otherwise return null config.
+   * If configData is provided (pre-parsed), returns it directly without file I/O.
    *
    * @param configPath - Optional path to the YAML configuration file
+   * @param configData - Optional pre-parsed config (bypasses file loading)
    * @returns Result containing DistributedConfig (or null) on success, PentestError on failure
    */
-  async loadOptional(configPath: string | undefined): Promise<Result<DistributedConfig | null, PentestError>> {
+  async loadOptional(
+    configPath: string | undefined,
+    configData?: DistributedConfig,
+    configYAML?: string,
+  ): Promise<Result<DistributedConfig | null, PentestError>> {
+    if (configData) {
+      return ok(configData);
+    }
+    if (configYAML) {
+      try {
+        const config = parseConfigYAML(configYAML);
+        return ok(distributeConfig(config));
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return err(
+          new PentestError(`Failed to parse config YAML: ${errorMessage}`, 'config', false, { originalError: errorMessage }, ErrorCode.CONFIG_PARSE_ERROR),
+        );
+      }
+    }
     if (!configPath) {
       return ok(null);
     }

apps/worker/src/services/container.ts

@@ -18,6 +18,11 @@
  */
 
 import type { SessionMetadata } from '../audit/utils.js';
+import type { CheckpointProvider } from '../interfaces/checkpoint-provider.js';
+import { NoOpCheckpointProvider } from '../interfaces/checkpoint-provider.js';
+import type { FindingsProvider } from '../interfaces/findings-provider.js';
+import { NoOpFindingsProvider } from '../interfaces/findings-provider.js';
+import type { ContainerConfig } from '../types/config.js';
 import { AgentExecutionService } from './agent-execution.js';
 import { ConfigLoaderService } from './config-loader.js';
 import { ExploitationCheckerService } from './exploitation-checker.js';
@@ -32,6 +37,9 @@ import { ExploitationCheckerService } from './exploitation-checker.js';
  */
 export interface ContainerDependencies {
   readonly sessionMetadata: SessionMetadata;
+  readonly config: ContainerConfig;
+  readonly findingsProvider?: FindingsProvider;
+  readonly checkpointProvider?: CheckpointProvider;
 }
 
 /**
@@ -45,17 +53,25 @@ export interface ContainerDependencies {
  */
 export class Container {
   readonly sessionMetadata: SessionMetadata;
+  readonly config: ContainerConfig;
   readonly agentExecution: AgentExecutionService;
   readonly configLoader: ConfigLoaderService;
   readonly exploitationChecker: ExploitationCheckerService;
+  readonly findingsProvider: FindingsProvider;
+  readonly checkpointProvider: CheckpointProvider;
 
   constructor(deps: ContainerDependencies) {
     this.sessionMetadata = deps.sessionMetadata;
+    this.config = deps.config;
 
     // Wire services with explicit constructor injection
     this.configLoader = new ConfigLoaderService();
     this.exploitationChecker = new ExploitationCheckerService();
     this.agentExecution = new AgentExecutionService(this.configLoader);
+
+    // Wire providers with default no-ops when not provided
+    this.findingsProvider = deps.findingsProvider ?? new NoOpFindingsProvider();
+    this.checkpointProvider = deps.checkpointProvider ?? new NoOpCheckpointProvider();
   }
 }
 
@@ -65,6 +81,12 @@ export class Container {
  */
 const containers = new Map<string, Container>();
 
+/** Default container config — OSS standalone defaults */
+const DEFAULT_CONFIG: ContainerConfig = {
+  deliverablesSubdir: '.shannon/deliverables',
+  auditDir: './workspaces',
+};
+
 /**
  * Get or create a Container for a workflow.
  *
@@ -73,13 +95,18 @@ const containers = new Map<string, Container>();
  *
  * @param workflowId - Unique workflow identifier
  * @param sessionMetadata - Session metadata for audit paths
+ * @param config - Runtime configuration (defaults to OSS standalone config)
  * @returns Container instance for the workflow
  */
-export function getOrCreateContainer(workflowId: string, sessionMetadata: SessionMetadata): Container {
+export function getOrCreateContainer(
+  workflowId: string,
+  sessionMetadata: SessionMetadata,
+  config: ContainerConfig = DEFAULT_CONFIG,
+): Container {
   let container = containers.get(workflowId);
 
   if (!container) {
-    container = new Container({ sessionMetadata });
+    container = new Container({ sessionMetadata, config });
     containers.set(workflowId, container);
   }
 

apps/worker/src/services/index.ts

@@ -16,7 +16,7 @@ export { AgentExecutionService } from './agent-execution.js';
 
 export { ConfigLoaderService } from './config-loader.js';
 export type { ContainerDependencies } from './container.js';
-export { Container, getOrCreateContainer, removeContainer } from './container.js';
+export { Container, getContainer, getOrCreateContainer, removeContainer } from './container.js';
 export { ExploitationCheckerService } from './exploitation-checker.js';
 export { loadPrompt } from './prompt-manager.js';
 export { assembleFinalReport, injectModelIntoReport } from './reporting.js';

apps/worker/src/services/preflight.ts

@@ -39,7 +39,7 @@ function isLoopbackAddress(address: string): boolean {
 
 // === Repository Validation ===
 
-async function validateRepo(repoPath: string, logger: ActivityLogger): Promise<Result<void, PentestError>> {
+async function validateRepo(repoPath: string, logger: ActivityLogger, skipGitCheck?: boolean): Promise<Result<void, PentestError>> {
   logger.info('Checking repository path...', { repoPath });
 
   // 1. Check repo directory exists
@@ -68,10 +68,22 @@ async function validateRepo(repoPath: string, logger: ActivityLogger): Promise<R
     );
   }
 
-  // 2. Check .git directory exists
-  try {
-    const gitStats = await fs.stat(`${repoPath}/.git`);
-    if (!gitStats.isDirectory()) {
+  // 2. Check .git directory exists (skipped when consumer removes .git after clone)
+  if (!skipGitCheck) {
+    try {
+      const gitStats = await fs.stat(`${repoPath}/.git`);
+      if (!gitStats.isDirectory()) {
+        return err(
+          new PentestError(
+            `Not a git repository (no .git directory): ${repoPath}`,
+            'config',
+            false,
+            { repoPath },
+            ErrorCode.REPO_NOT_FOUND,
+          ),
+        );
+      }
+    } catch {
       return err(
         new PentestError(
           `Not a git repository (no .git directory): ${repoPath}`,
@@ -82,16 +94,8 @@ async function validateRepo(repoPath: string, logger: ActivityLogger): Promise<R
         ),
       );
     }
-  } catch {
-    return err(
-      new PentestError(
-        `Not a git repository (no .git directory): ${repoPath}`,
-        'config',
-        false,
-        { repoPath },
-        ErrorCode.REPO_NOT_FOUND,
-      ),
-    );
+  } else {
+    logger.info('Skipping .git check (skipGitCheck enabled)');
   }
 
   logger.info('Repository path OK');
@@ -180,9 +184,21 @@ function classifySdkError(sdkError: SDKAssistantMessageError, authType: string):
 }
 
 /** Validate credentials via a minimal Claude Agent SDK query. */
-async function validateCredentials(logger: ActivityLogger): Promise<Result<void, PentestError>> {
+async function validateCredentials(logger: ActivityLogger, apiKey?: string, providerConfig?: import('../types/config.js').ProviderConfig): Promise<Result<void, PentestError>> {
+  // 0. If providerConfig is present, credentials are managed by the caller.
+  //    The executor will map providerConfig directly to sdkEnv — no process.env needed.
+  if (providerConfig) {
+    logger.info(`Provider config present (type: ${providerConfig.providerType || 'anthropic_api'}) — skipping env-based credential validation`);
+    return ok(undefined);
+  }
+
+  // 0b. If apiKey provided via config, set it in env for SDK validation
+  //     This avoids requiring process.env.ANTHROPIC_API_KEY when key is threaded via input
+  if (apiKey) {
+    process.env.ANTHROPIC_API_KEY = apiKey;
+  }
   // 1. Custom base URL — validate endpoint is reachable via SDK query
-  if (process.env.ANTHROPIC_BASE_URL) {
+  if (process.env.ANTHROPIC_BASE_URL && process.env.ANTHROPIC_AUTH_TOKEN) {
     const baseUrl = process.env.ANTHROPIC_BASE_URL;
     logger.info(`Validating custom base URL: ${baseUrl}`);
 
@@ -289,7 +305,7 @@ async function validateCredentials(logger: ActivityLogger): Promise<Result<void,
   }
 
   // 4. Check that at least one credential is present
-  if (!process.env.ANTHROPIC_API_KEY && !process.env.CLAUDE_CODE_OAUTH_TOKEN) {
+  if (!process.env.ANTHROPIC_API_KEY && !process.env.CLAUDE_CODE_OAUTH_TOKEN && !process.env.ANTHROPIC_AUTH_TOKEN) {
     return err(
       new PentestError(
         'No API credentials found. Set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env (or use CLAUDE_CODE_USE_BEDROCK=1 for AWS Bedrock, or CLAUDE_CODE_USE_VERTEX=1 for Google Vertex AI)',
@@ -457,9 +473,12 @@ export async function runPreflightChecks(
   repoPath: string,
   configPath: string | undefined,
   logger: ActivityLogger,
+  skipGitCheck?: boolean,
+  apiKey?: string,
+  providerConfig?: import('../types/config.js').ProviderConfig,
 ): Promise<Result<void, PentestError>> {
   // 1. Repository check (free — filesystem only)
-  const repoResult = await validateRepo(repoPath, logger);
+  const repoResult = await validateRepo(repoPath, logger, skipGitCheck);
   if (!repoResult.ok) {
     return repoResult;
   }
@@ -472,8 +491,8 @@ export async function runPreflightChecks(
     }
   }
 
-  // 3. Credential check (cheap — 1 SDK round-trip)
-  const credResult = await validateCredentials(logger);
+  // 3. Credential check (cheap — 1 SDK round-trip, skipped when providerConfig present)
+  const credResult = await validateCredentials(logger, apiKey, providerConfig);
   if (!credResult.ok) {
     return credResult;
   }

apps/worker/src/services/prompt-manager.ts

@@ -23,10 +23,14 @@ interface IncludeReplacement {
 }
 
 // Pure function: Build complete login instructions from config
-async function buildLoginInstructions(authentication: Authentication, logger: ActivityLogger): Promise<string> {
+async function buildLoginInstructions(
+  authentication: Authentication,
+  logger: ActivityLogger,
+  promptsBaseDir: string = PROMPTS_DIR,
+): Promise<string> {
   try {
     // 1. Load the login instructions template
-    const loginInstructionsPath = path.join(PROMPTS_DIR, 'shared', 'login-instructions.txt');
+    const loginInstructionsPath = path.join(promptsBaseDir, 'shared', 'login-instructions.txt');
 
     if (!(await fs.pathExists(loginInstructionsPath))) {
       throw new PentestError('Login instructions template not found', 'filesystem', false, { loginInstructionsPath });
@@ -148,6 +152,7 @@ async function interpolateVariables(
   variables: PromptVariables,
   config: DistributedConfig | null = null,
   logger: ActivityLogger,
+  promptsBaseDir: string = PROMPTS_DIR,
 ): Promise<string> {
   try {
     if (!template || typeof template !== 'string') {
@@ -188,7 +193,7 @@ async function interpolateVariables(
 
       // Extract and inject login instructions from config
       if (config.authentication?.login_flow) {
-        const loginInstructions = await buildLoginInstructions(config.authentication, logger);
+        const loginInstructions = await buildLoginInstructions(config.authentication, logger, promptsBaseDir);
         result = result.replace(/{{LOGIN_INSTRUCTIONS}}/g, loginInstructions);
       } else {
         result = result.replace(/{{LOGIN_INSTRUCTIONS}}/g, '');
@@ -223,10 +228,12 @@ export async function loadPrompt(
   config: DistributedConfig | null = null,
   pipelineTestingMode: boolean = false,
   logger: ActivityLogger,
+  promptDir?: string,
 ): Promise<string> {
   try {
-    // 1. Resolve prompt file path
-    const promptsDir = pipelineTestingMode ? path.join(PROMPTS_DIR, 'pipeline-testing') : PROMPTS_DIR;
+    // 1. Resolve prompt file path (promptDir override → default PROMPTS_DIR)
+    const basePromptsDir = promptDir ?? PROMPTS_DIR;
+    const promptsDir = pipelineTestingMode ? path.join(basePromptsDir, 'pipeline-testing') : basePromptsDir;
     const promptPath = path.join(promptsDir, `${promptName}.txt`);
 
     if (pipelineTestingMode) {
@@ -256,7 +263,7 @@ export async function loadPrompt(
     template = await processIncludes(template, promptsDir);
 
     // 5. Interpolate variables and return final prompt
-    return await interpolateVariables(template, enhancedVariables, config, logger);
+    return await interpolateVariables(template, enhancedVariables, config, logger, basePromptsDir);
   } catch (error) {
     if (error instanceof PentestError) {
       throw error;

apps/worker/src/services/queue-validation.ts

@@ -5,6 +5,7 @@
 // as published by the Free Software Foundation.
 
 import { fs, path } from 'zx';
+
 import type { ExploitationDecision, VulnType } from '../types/agents.js';
 import { ErrorCode } from '../types/errors.js';
 import { err, ok, type Result } from '../types/result.js';
@@ -133,8 +134,8 @@ const createPaths = (vulnType: VulnType, sourceDir: string): PathsBase | PathsWi
 
   return Object.freeze({
     vulnType,
-    deliverable: path.join(sourceDir, '.shannon', 'deliverables', config.deliverable),
-    queue: path.join(sourceDir, '.shannon', 'deliverables', config.queue),
+    deliverable: path.join(sourceDir, config.deliverable),
+    queue: path.join(sourceDir, config.queue),
     sourceDir,
   });
 };

apps/worker/src/services/reporting.ts

@@ -5,6 +5,7 @@
 // as published by the Free Software Foundation.
 
 import { fs, path } from 'zx';
+import { deliverablesDir } from '../paths.js';
 import type { ActivityLogger } from '../types/activity-logger.js';
 import { ErrorCode } from '../types/errors.js';
 import { PentestError } from './error-handling.js';
@@ -28,7 +29,7 @@ export async function assembleFinalReport(sourceDir: string, logger: ActivityLog
   const sections: string[] = [];
 
   for (const file of deliverableFiles) {
-    const filePath = path.join(sourceDir, '.shannon', 'deliverables', file.path);
+    const filePath = path.join(deliverablesDir(sourceDir), file.path);
     try {
       if (await fs.pathExists(filePath)) {
         const content = await fs.readFile(filePath, 'utf8');
@@ -55,12 +56,12 @@ export async function assembleFinalReport(sourceDir: string, logger: ActivityLog
   }
 
   const finalContent = sections.join('\n\n');
-  const deliverablesDir = path.join(sourceDir, '.shannon', 'deliverables');
-  const finalReportPath = path.join(deliverablesDir, 'comprehensive_security_assessment_report.md');
+  const outputDir = deliverablesDir(sourceDir);
+  const finalReportPath = path.join(outputDir, 'comprehensive_security_assessment_report.md');
 
   try {
     // Ensure deliverables directory exists
-    await fs.ensureDir(deliverablesDir);
+    await fs.ensureDir(outputDir);
     await fs.writeFile(finalReportPath, finalContent);
     logger.info(`Final report assembled at ${finalReportPath}`);
   } catch (error) {
@@ -117,7 +118,7 @@ export async function injectModelIntoReport(
   logger.info(`Injecting model info into report: ${modelStr}`);
 
   // 3. Read the final report
-  const reportPath = path.join(repoPath, '.shannon', 'deliverables', 'comprehensive_security_assessment_report.md');
+  const reportPath = path.join(deliverablesDir(repoPath), 'comprehensive_security_assessment_report.md');
 
   if (!(await fs.pathExists(reportPath))) {
     logger.warn('Final report not found, skipping model injection');