diff --git a/core/src/autonomous/index.ts b/core/src/autonomous/index.ts index ecce928..8e8eaef 100644 --- a/core/src/autonomous/index.ts +++ b/core/src/autonomous/index.ts @@ -18,7 +18,7 @@ export type { BudgetGuardOptions } from "./lib/budget.js"; export type { AutonomousReport } from "./report/types.js"; export { mapRunLogToReport } from "./report/mapRunLog.js"; export { renderReportHtml } from "./report/html.js"; -export { writeAutonomousReport } from "./report/writeReport.js"; +export { writeAutonomousReport, reportDirFor } from "./report/writeReport.js"; export { loadKnowledge } from "./knowledge/load.js"; export type { diff --git a/core/src/autonomous/orchestrator/run.ts b/core/src/autonomous/orchestrator/run.ts index 76b8958..141d176 100644 --- a/core/src/autonomous/orchestrator/run.ts +++ b/core/src/autonomous/orchestrator/run.ts @@ -64,8 +64,13 @@ export interface RunHooks { progress?: ProgressReporter; /** Called once the in-memory RunLog is created — used by the live UI for snapshots. */ onRunLog?: (runLog: import("../state/runLog.js").RunLog) => void; + /** Abort to stop the run early and finalize a partial (truncated) report instead of throwing. */ + signal?: AbortSignal; } +/** Truncation reason recorded when a run is stopped by the user rather than a ceiling/error. */ +const USER_INTERRUPT_REASON = "interrupted by user (Ctrl+C)"; + export async function runAutonomous( options: HuntOptions, runHooks?: RunHooks @@ -79,6 +84,8 @@ export async function runAutonomous( ); } + // Created only after the checks above — a bad config should fail with zero artifacts, not an + // empty report folder. Handed off via `onRunLog` before any progress event can fire. const runLog = createRunLog({ runId: randomUUID(), objective: options.objective, @@ -173,14 +180,35 @@ export async function runAutonomous( const q = query({ prompt: kickoff, options: queryOptions }); const reporter = runHooks?.progress; + const signal = runHooks?.signal; reporter?.onLine("Autonomous assessment started — commander initializing…"); + // Interrupt immediately so an in-flight multi-turn attack doesn't keep the run alive after + // Ctrl+C — truncation flags are set below, this just stops the agent promptly. + const onAbort = (): void => { + reporter?.onLine("⏹ interrupt received — stopping agents, finalizing a partial report…"); + void q.interrupt().catch(() => {}); + }; + if (signal) { + if (signal.aborted) onAbort(); + else signal.addEventListener("abort", onAbort, { once: true }); + } + // Tracks last reported cost threshold so we only emit a cost line every $0.10. let lastReportedCostUsd = 0; try { for await (const message of q) { + // Stop promptly on cancellation while messages are still flowing (the abort listener + // above covers the idle-between-messages case). + if (signal?.aborted && !runLog.completed) { + runLog.truncated = true; + if (!runLog.truncationReason) runLog.truncationReason = USER_INTERRUPT_REASON; + await q.interrupt().catch(() => {}); + break; + } + if (message.type === "assistant") { const text = message.message.content .map((b) => (b.type === "text" ? b.text : "")) @@ -252,20 +280,34 @@ export async function runAutonomous( } } } catch (err) { - // A mid-run failure (e.g. provider usage-policy block, network error) must - // NOT lose the findings already captured in the RunLog. Mark the run - // truncated and fall through to build a partial report. + // A mid-run failure or the abort itself must not lose captured findings — mark truncated + // and fall through to a partial report. const message = err instanceof Error ? err.message : String(err); runLog.truncated = true; - runLog.truncationReason = `run interrupted: ${message.slice(0, 300)}`; - reporter?.onLine( - `⚠️ run interrupted — finalizing partial report from ${runLog.findings.length} finding(s)` - ); - reporter?.onLine(` reason: ${message.slice(0, 200)}`); + if (signal?.aborted) { + runLog.truncationReason = USER_INTERRUPT_REASON; + reporter?.onLine( + `⏹ run interrupted — finalizing partial report from ${runLog.findings.length} finding(s)` + ); + } else { + runLog.truncationReason = `run interrupted: ${message.slice(0, 300)}`; + reporter?.onLine( + `⚠️ run interrupted — finalizing partial report from ${runLog.findings.length} finding(s)` + ); + reporter?.onLine(` reason: ${message.slice(0, 200)}`); + } } finally { + if (signal) signal.removeEventListener("abort", onAbort); q.close(); } + // Safety net: if the caller aborted but neither the loop nor the catch labelled it (e.g. the + // stream ended cleanly right as the interrupt landed), still mark it a user interrupt. + if (signal?.aborted && !runLog.completed && !runLog.truncated) { + runLog.truncated = true; + runLog.truncationReason = USER_INTERRUPT_REASON; + } + if (!runLog.completed && !runLog.truncated) { // Stream ended without a submit_report (e.g. agent stopped early). runLog.truncated = runLog.findings.length === 0 && runLog.threads.size === 0; diff --git a/core/src/autonomous/report/mapRunLog.ts b/core/src/autonomous/report/mapRunLog.ts index 66e94d1..7f18a9a 100644 --- a/core/src/autonomous/report/mapRunLog.ts +++ b/core/src/autonomous/report/mapRunLog.ts @@ -191,6 +191,7 @@ export function mapRunLogToReport(log: RunLog): AutonomousReport { return { reportId: log.runId, + startedAt: log.startedAt, generatedAt: new Date().toISOString(), target: { name: log.targetName, endpoint: log.targetEndpoint }, objective: log.objective, diff --git a/core/src/autonomous/report/types.ts b/core/src/autonomous/report/types.ts index a4c8b1f..c07ac7c 100644 --- a/core/src/autonomous/report/types.ts +++ b/core/src/autonomous/report/types.ts @@ -85,6 +85,7 @@ export interface PersonaTimelineEntry { export interface AutonomousReport { reportId: string; + startedAt: string; generatedAt: string; target: { name: string; endpoint: string }; objective: string; diff --git a/core/src/autonomous/report/writeReport.ts b/core/src/autonomous/report/writeReport.ts index fc8a4de..2f15250 100644 --- a/core/src/autonomous/report/writeReport.ts +++ b/core/src/autonomous/report/writeReport.ts @@ -11,21 +11,39 @@ export interface ReportFiles { dir: string; } +function slugify(name: string): string { + return ( + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40) || "target" + ); +} + +/** Report folder name — derivable at run start, so callers can create it upfront instead of relocating files later. */ +export function reportDirFor( + outputDir: string, + params: { targetName: string; runId: string; startedAt: string } +): string { + const compactTs = params.startedAt.replace(/[-:T.Z]/g, "").slice(0, 14); + const slug = slugify(params.targetName); + const shortId = params.runId.replace(/-/g, "").slice(0, 8); + return path.resolve(outputDir, `hunt-report-${compactTs}-${slug}-${shortId}`); +} + export async function writeAutonomousReport( report: AutonomousReport, outputDir: string ): Promise { - const compactTs = report.generatedAt.replace(/[-:T.Z]/g, "").slice(0, 14); - const slug = - report.target.name - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 40) || "target"; - const shortId = report.reportId.replace(/-/g, "").slice(0, 8); - const dir = path.resolve(outputDir, `hunt-report-${compactTs}-${slug}-${shortId}`); + const dir = reportDirFor(outputDir, { + targetName: report.target.name, + runId: report.reportId, + startedAt: report.startedAt, + }); await mkdir(dir, { recursive: true }); + const slug = slugify(report.target.name); const htmlPath = path.join(dir, `${slug}-report.html`); const jsonPath = path.join(dir, `${slug}-report.json`); diff --git a/docs/hunt.md b/docs/hunt.md index cb8c447..8fccd4d 100644 --- a/docs/hunt.md +++ b/docs/hunt.md @@ -132,6 +132,14 @@ respond before it's killed. | `--ui` | Launch live dashboard | | `--ui-port ` | `3847` | +Each run writes everything into one folder — `hunt-report---/` — containing the live log (`hunt-live.log`), the structured event trail (`run-events.jsonl`), and the final `*-report.html` / `*-report.json`. + +## Stopping a run + +Press **Ctrl+C** once to stop early: the agent is interrupted, and a report is still written from the findings gathered so far (with the live log and event trail already on disk). The report is marked as truncated so it's clear the assessment was cut short. Press **Ctrl+C** a second time to force-quit without writing a report. + +The same applies if a run errors out mid-flight (provider block, network failure) — findings captured up to that point are preserved in a partial report rather than lost. + ## Authentication Credentials are resolved in order: diff --git a/runners/cli/src/commands/hunt.ts b/runners/cli/src/commands/hunt.ts index b24e74a..362556a 100644 --- a/runners/cli/src/commands/hunt.ts +++ b/runners/cli/src/commands/hunt.ts @@ -1,7 +1,7 @@ import type { Command } from "commander"; import path from "node:path"; -import { readFile, mkdir } from "node:fs/promises"; -import { createWriteStream, existsSync, type WriteStream } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { createWriteStream, existsSync, mkdirSync, type WriteStream } from "node:fs"; import { homedir } from "node:os"; import { consola } from "consola"; import type { @@ -11,7 +11,10 @@ import type { import type { RunEvent } from "@keyvaluesystems/agent-opfor-core/autonomous/state/observe.js"; import { parseAgentTarget } from "@keyvaluesystems/agent-opfor-core/config/schema.js"; import { runAutonomous } from "@keyvaluesystems/agent-opfor-core/autonomous/orchestrator/run.js"; -import { writeAutonomousReport } from "@keyvaluesystems/agent-opfor-core/autonomous/report/writeReport.js"; +import { + writeAutonomousReport, + reportDirFor, +} from "@keyvaluesystems/agent-opfor-core/autonomous/report/writeReport.js"; import { startUiServer } from "../ui/server.js"; import { mergeReporters } from "../ui/bridge.js"; @@ -295,6 +298,25 @@ export function registerHuntCommand(program: Command): void { }, }); + // First Ctrl+C aborts a running assessment (partial report via onComplete), or exits if + // idle on the setup form; second Ctrl+C force-quits. + let setupSigintCount = 0; + const onSetupSigint = (): void => { + setupSigintCount++; + if (setupSigintCount >= 2) { + void cleanup(130); + return; + } + if (serverHandle?.abortAssessment()) { + consola.warn( + "\nCaught interrupt — finalizing a partial report… (Ctrl+C again to force quit)" + ); + } else { + void cleanup(0); + } + }; + process.on("SIGINT", onSetupSigint); + // Keep process alive until onComplete is called await new Promise(() => {}); return; @@ -420,25 +442,35 @@ export function registerHuntCommand(program: Command): void { outputDir: path.resolve(opts.output), }; - // Live log file the user can `tail -f` while the run is in progress. - await mkdir(huntOptions.outputDir, { recursive: true }); - const startedAt = new Date() - .toISOString() - .replace(/[-:T.Z]/g, "") - .slice(0, 14); - const liveLogPath = path.join(huntOptions.outputDir, `hunt-live-${startedAt}.log`); - const liveLog: WriteStream = createWriteStream(liveLogPath, { flags: "a" }); + const header = [ + "════════════════════════════════════════════════════════════════", + ` AUTONOMOUS RED-TEAM`, + ` target : ${target.name} (${target.mode}) ${target.endpoint}`, + ` objective : ${objective}`, + ` models : commander=${huntOptions.commanderModel} operator=${huntOptions.operatorModel} scout=${huntOptions.scoutModel}`, + ` limits : operators≤${huntOptions.maxOperators} turns≤${huntOptions.maxTurns} thread-turns≤${huntOptions.maxThreadTurns}${huntOptions.budgetUsd ? ` budget=$${huntOptions.budgetUsd}` : ""}`, + ` verifier : ${huntOptions.verify ? "on" : "off"}`, + "════════════════════════════════════════════════════════════════", + ].join("\n"); + process.stdout.write(header + "\n"); + + // Folder + streams are created from `onRunLog` below, before any progress event can fire. + let reportDir = ""; + let liveLogPath = ""; + let eventLogPath = ""; + let liveLog: WriteStream | undefined; + let eventLog: WriteStream | undefined; + const emit = (line: string): void => { const stamped = `[${clock()}] ${line}`; process.stdout.write(stamped + "\n"); - liveLog.write(stamped + "\n"); + liveLog?.write(stamped + "\n"); }; // Structured event trail (one JSON object per line) — machine-readable for debugging and // the foundation a future "opfor view" web UI consumes. Stamped with a wall-clock time. - const eventLogPath = path.join(huntOptions.outputDir, `run-${startedAt}.jsonl`); - const eventLog: WriteStream = createWriteStream(eventLogPath, { flags: "a" }); const emitEvent = (event: RunEvent): void => { + if (!eventLog) return; // An observability sink must never crash the run. Guard JSON.stringify (data is // Record — a future event could carry a circular ref / BigInt). try { @@ -455,20 +487,6 @@ export function registerHuntCommand(program: Command): void { } }; - const header = [ - "════════════════════════════════════════════════════════════════", - ` AUTONOMOUS RED-TEAM`, - ` target : ${target.name} (${target.mode}) ${target.endpoint}`, - ` objective : ${objective}`, - ` models : commander=${huntOptions.commanderModel} operator=${huntOptions.operatorModel} scout=${huntOptions.scoutModel}`, - ` limits : operators≤${huntOptions.maxOperators} turns≤${huntOptions.maxTurns} thread-turns≤${huntOptions.maxThreadTurns}${huntOptions.budgetUsd ? ` budget=$${huntOptions.budgetUsd}` : ""}`, - ` verifier : ${huntOptions.verify ? "on" : "off"}`, - "════════════════════════════════════════════════════════════════", - ].join("\n"); - process.stdout.write(header + "\n"); - liveLog.write(header + "\n"); - consola.box(`Live log (tail it):\n tail -f ${liveLogPath}`); - let uiHandle: Awaited> | undefined; if (opts.ui) { const uiPort = intOr(opts.uiPort, 3847); @@ -490,8 +508,6 @@ export function registerHuntCommand(program: Command): void { } catch (err) { consola.error(`Failed to start UI server: ${String(err)}`); process.exitCode = 1; - liveLog.end(); - eventLog.end(); return; } } @@ -499,32 +515,78 @@ export function registerHuntCommand(program: Command): void { const fileReporter = { onLine: emit, onEvent: emitEvent }; const progress = uiHandle ? mergeReporters(fileReporter, uiHandle.bridge) : fileReporter; + // First Ctrl+C aborts and finalizes a partial report; second Ctrl+C force-quits. + const ac = new AbortController(); + let sigintCount = 0; + const onSigint = (): void => { + sigintCount++; + if (sigintCount === 1) { + consola.warn("\nCaught interrupt — stopping agents and finalizing a partial report…"); + consola.warn("Press Ctrl+C again to force quit (no report will be written).\n"); + ac.abort(); + } else { + process.exit(130); + } + }; + process.on("SIGINT", onSigint); + let report; try { report = await runAutonomous(huntOptions, { progress, - onRunLog: uiHandle ? (log) => uiHandle!.attachRunLog(log) : undefined, + signal: ac.signal, + onRunLog: (log) => { + reportDir = reportDirFor(huntOptions.outputDir, { + targetName: log.targetName, + runId: log.runId, + startedAt: log.startedAt, + }); + mkdirSync(reportDir, { recursive: true }); + liveLogPath = path.join(reportDir, "hunt-live.log"); + eventLogPath = path.join(reportDir, "run-events.jsonl"); + liveLog = createWriteStream(liveLogPath, { flags: "a" }); + eventLog = createWriteStream(eventLogPath, { flags: "a" }); + liveLog.write(header + "\n"); + consola.box(`Live log (tail it):\n tail -f ${liveLogPath}`); + uiHandle?.attachRunLog(log); + }, }); } finally { - liveLog.end(); - eventLog.end(); + // Stop intercepting SIGINT before the post-run UI wait installs its own handler. + process.off("SIGINT", onSigint); + liveLog?.end(); + eventLog?.end(); } + const interrupted = ac.signal.aborted; + const { html, json, dir } = await writeAutonomousReport(report, huntOptions.outputDir); - uiHandle?.markComplete({ reportDir: dir, outcome: report.objectiveOutcome }); + // `outcome` is the dashboard's final status label (rendered verbatim). On interrupt show + // "interrupted" so a cancelled run isn't mistaken for a normal finish; otherwise show the + // assessment verdict, which is more useful than a generic "completed". + uiHandle?.markComplete({ + reportDir: dir, + outcome: interrupted ? "interrupted" : report.objectiveOutcome, + }); consola.info(""); - consola.success(`Assessment complete — outcome: ${report.objectiveOutcome}`); + if (interrupted) { + consola.warn(`Assessment interrupted — partial report written from work completed so far.`); + } else { + consola.success(`Assessment complete — outcome: ${report.objectiveOutcome}`); + } consola.info( `Vulnerabilities: ${report.summary.confirmed} · Defended: ${report.summary.defended} · Errors: ${report.summary.errors}` ); if (report.totalCostUsd !== undefined) consola.info(`Cost: $${report.totalCostUsd.toFixed(4)}`); - if (report.truncated) consola.warn(`Run truncated: ${report.truncationReason}`); + if (report.truncated && !interrupted) + consola.warn(`Run truncated: ${report.truncationReason}`); consola.success(`Report: ${html}`); consola.info(` JSON: ${json}`); consola.info(` Dir : ${dir}`); + consola.info(` Live log: ${liveLogPath}`); consola.info(` Events: ${eventLogPath}`); if (uiHandle) { consola.info(` UI : ${uiHandle.url} (press Ctrl+C to exit)`); diff --git a/runners/cli/src/commands/setup.ts b/runners/cli/src/commands/setup.ts index 2631a57..8331358 100644 --- a/runners/cli/src/commands/setup.ts +++ b/runners/cli/src/commands/setup.ts @@ -48,6 +48,7 @@ export async function runSetupAndWrite(opts?: { await writeFile(outPath, JSON.stringify(config, null, 2), "utf8"); log.success(`\nConfig written → ${outPath}`); + log.info(`Made a mistake? Edit any field in that JSON file directly before running it.`); return { config, path: outPath }; } diff --git a/runners/cli/src/ui/server.ts b/runners/cli/src/ui/server.ts index 0a889f8..a81f2e1 100644 --- a/runners/cli/src/ui/server.ts +++ b/runners/cli/src/ui/server.ts @@ -2,9 +2,7 @@ import http from "node:http"; import path from "node:path"; -import { existsSync } from "node:fs"; -import { mkdir } from "node:fs/promises"; -import { createWriteStream, type WriteStream } from "node:fs"; +import { existsSync, createWriteStream, mkdirSync, type WriteStream } from "node:fs"; import { fileURLToPath } from "node:url"; import { exec } from "node:child_process"; import express from "express"; @@ -77,6 +75,11 @@ export interface UiServerHandle { url: string; attachRunLog: (runLog: RunLog) => void; markComplete: (payload: { reportDir?: string; outcome?: string }) => void; + /** + * Abort an in-progress setup-mode assessment so it finalizes a partial report. + * Returns true if a run was actually in progress, false if idle (nothing to abort). + */ + abortAssessment: () => boolean; close: () => Promise; } @@ -109,6 +112,9 @@ function openInBrowser(url: string): void { export async function startUiServer(options: UiServerOptions): Promise { const app = express(); const bridge = new UiBridge(); + + // Created fresh per run in the /api/start handler; aborting finalizes a partial report. + let assessmentAbort: AbortController | undefined; bridge.setMeta(options.meta); const staticDir = resolveStaticDir(); @@ -260,8 +266,10 @@ export async function startUiServer(options: UiServerOptions): Promise { options.onComplete?.({ success: true, reportDir }); }) @@ -274,6 +282,7 @@ export async function startUiServer(options: UiServerOptions): Promise { runningAssessment = false; + assessmentAbort = undefined; }); }); } @@ -310,6 +319,11 @@ export async function startUiServer(options: UiServerOptions): Promise { server.close((err) => (err ? reject(err) : resolve())); @@ -322,35 +336,30 @@ export async function startUiServer(options: UiServerOptions): Promise void + onLog?: (line: string) => void, + signal?: AbortSignal ): Promise { const { runAutonomous } = await import("@keyvaluesystems/agent-opfor-core/autonomous/orchestrator/run.js"); - const { writeAutonomousReport } = + const { writeAutonomousReport, reportDirFor } = await import("@keyvaluesystems/agent-opfor-core/autonomous/report/writeReport.js"); - // Set up output directory and log files - await mkdir(autoOptions.outputDir, { recursive: true }); - const startedAt = new Date() - .toISOString() - .replace(/[-:T.Z]/g, "") - .slice(0, 14); - const liveLogPath = path.join(autoOptions.outputDir, `hunt-live-${startedAt}.log`); - const liveLog: WriteStream = createWriteStream(liveLogPath, { flags: "a" }); - const eventLogPath = path.join(autoOptions.outputDir, `run-${startedAt}.jsonl`); - const eventLog: WriteStream = createWriteStream(eventLogPath, { flags: "a" }); - const clock = () => new Date().toISOString().slice(11, 19); + // Folder + streams are created from `onRunLog` below, before any progress event can fire. + let liveLog: WriteStream | undefined; + let eventLog: WriteStream | undefined; + const emit = (line: string): void => { const stamped = `[${clock()}] ${line}`; - liveLog.write(stamped + "\n"); + liveLog?.write(stamped + "\n"); bridge.onLine(line); // Also stream to terminal onLog?.(stamped); }; const emitEvent = (event: RunEvent): void => { + if (!eventLog) return; try { eventLog.write(JSON.stringify({ ...event, wall: clock() }) + "\n"); } catch { @@ -359,15 +368,25 @@ async function runAssessmentInProcess( bridge.onEvent(event); }; - emit(`Starting autonomous assessment against ${autoOptions.target.name}`); - emit(`Objective: ${autoOptions.objective}`); - let reportDir: string | undefined; try { const report = await runAutonomous(autoOptions, { progress: { onLine: emit, onEvent: emitEvent }, + signal, onRunLog: (runLog) => { + reportDir = reportDirFor(autoOptions.outputDir, { + targetName: runLog.targetName, + runId: runLog.runId, + startedAt: runLog.startedAt, + }); + mkdirSync(reportDir, { recursive: true }); + liveLog = createWriteStream(path.join(reportDir, "hunt-live.log"), { flags: "a" }); + eventLog = createWriteStream(path.join(reportDir, "run-events.jsonl"), { flags: "a" }); + + emit(`Starting autonomous assessment against ${autoOptions.target.name}`); + emit(`Objective: ${autoOptions.objective}`); + bridge.attachRunLog(runLog, { objective: autoOptions.objective, targetName: autoOptions.target.name, @@ -380,15 +399,18 @@ async function runAssessmentInProcess( }, }); - emit("Run completed, writing report…"); + const interrupted = signal?.aborted ?? false; + emit( + interrupted ? "Run interrupted, writing partial report…" : "Run completed, writing report…" + ); const reportFiles = await writeAutonomousReport(report, autoOptions.outputDir); reportDir = reportFiles.dir; emit(`Report written to ${reportDir}`); - bridge.markComplete({ reportDir, outcome: "completed" }); + bridge.markComplete({ reportDir, outcome: interrupted ? "interrupted" : "completed" }); } finally { - liveLog.end(); - eventLog.end(); + liveLog?.end(); + eventLog?.end(); } return reportDir;