feat: add model tracking and reporting across pipeline

- Track actual model name from router through audit logs, session.json, and query output
- Add router-utils.ts to resolve model names from ROUTER_DEFAULT env var
- Inject model info into final report's Executive Summary section
- Update documentation with supported providers, pricing, and config examples
- Update router-config.json with latest model versions (GPT-5.2, Gemini 2.5, etc.)
This commit is contained in:
ajmallesh
2026-01-15 18:30:19 -08:00
parent d01980ce4b
commit cd04c7a6d2
16 changed files with 312 additions and 56 deletions
+25 -1
View File
@@ -67,7 +67,7 @@ import {
rollbackGitWorkspace,
getGitCommitHash,
} from '../utils/git-manager.js';
import { assembleFinalReport } from '../phases/reporting.js';
import { assembleFinalReport, injectModelIntoReport } from '../phases/reporting.js';
import { getPromptNameForAgent } from '../types/agents.js';
import { AuditSession } from '../audit/index.js';
import type { WorkflowSummary } from '../audit/workflow-logger.js';
@@ -192,6 +192,7 @@ async function runAgentActivity(
duration_ms: result.duration,
cost_usd: 0,
success: false,
model: result.model,
error: `Spending cap likely reached: ${resultText.slice(0, 100)}`,
});
// Throw as billing error so Temporal retries with long backoff
@@ -207,6 +208,7 @@ async function runAgentActivity(
duration_ms: result.duration,
cost_usd: result.cost || 0,
success: false,
model: result.model,
error: result.error || 'Execution failed',
});
throw new Error(result.error || 'Agent execution failed');
@@ -221,6 +223,7 @@ async function runAgentActivity(
duration_ms: result.duration,
cost_usd: result.cost || 0,
success: false,
model: result.model,
error: 'Output validation failed',
});
@@ -243,6 +246,7 @@ async function runAgentActivity(
duration_ms: result.duration,
cost_usd: result.cost || 0,
success: true,
model: result.model,
...(commitHash && { checkpoint: commitHash }),
});
await commitGitSuccess(repoPath, agentName);
@@ -254,6 +258,7 @@ async function runAgentActivity(
outputTokens: null,
costUsd: result.cost ?? null,
numTurns: result.turns ?? null,
model: result.model,
};
} catch (error) {
// Rollback git workspace before Temporal retry to ensure clean state
@@ -369,6 +374,25 @@ export async function assembleReportActivity(input: ActivityInput): Promise<void
}
}
/**
* Inject model metadata into the final report.
* This must be called AFTER runReportAgent to add the model information to the Executive Summary.
*/
export async function injectReportMetadataActivity(input: ActivityInput): Promise<void> {
const { repoPath, outputPath } = input;
if (!outputPath) {
console.log(chalk.yellow('⚠️ No output path provided, skipping model injection'));
return;
}
try {
await injectModelIntoReport(repoPath, outputPath);
} catch (error) {
const err = error as Error;
console.log(chalk.yellow(`⚠️ Error injecting model into report: ${err.message}`));
// Don't throw - this is a non-critical enhancement
}
}
/**
* Check if exploitation should run for a given vulnerability type.
* Reads the vulnerability queue file and returns the decision.
+3
View File
@@ -35,6 +35,7 @@ interface AgentMetrics {
outputTokens: number | null;
costUsd: number | null;
numTurns: number | null;
model?: string | undefined;
}
interface PipelineProgress {
@@ -123,8 +124,10 @@ async function queryWorkflow(): Promise<void> {
const metrics = progress.agentMetrics[agent];
const duration = metrics ? formatDuration(metrics.durationMs) : 'unknown';
const cost = metrics?.costUsd ? `$${metrics.costUsd.toFixed(4)}` : '';
const model = metrics?.model ? ` [${metrics.model}]` : '';
console.log(
chalk.green(` - ${agent}`) +
chalk.blue(model) +
chalk.gray(` (${duration}${cost ? ', ' + cost : ''})`)
);
}
+1
View File
@@ -17,6 +17,7 @@ export interface AgentMetrics {
outputTokens: number | null;
costUsd: number | null;
numTurns: number | null;
model?: string | undefined;
}
export interface PipelineSummary {
+4
View File
@@ -276,6 +276,10 @@ export async function pentestPipelineWorkflow(
// Then run the report agent to add executive summary and clean up
state.agentMetrics['report'] = await a.runReportAgent(activityInput);
state.completedAgents.push('report');
// Inject model metadata into the final report
await a.injectReportMetadataActivity(activityInput);
await a.logPhaseTransition(activityInput, 'reporting', 'complete');
// === Complete ===