diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c372fdd..fa396564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Antigravity agent support**: Replaced Gemini CLI initialization and worker-agent support with Antigravity CLI support using the documented `agy` executable, `.agents/skills`, and `.agents/hooks.json` hook surface. + +### Removed + +- **Gemini agent support**: Removed Gemini as a selectable initialization and worker agent target, including obsolete Gemini settings fragments, configurers, command mappings, and logo assets. + ## [3.15.0] - 2026-07-08 ### Added diff --git a/README.md b/README.md index 87da1c06..f06aeb09 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

# Jumbo Context -### Tired of repeating yourself to Claude, Codex, Gemini, Copilot and their cohorts? +### Tired of repeating yourself to Claude, Codex, Antigravity, Copilot and their cohorts? Agents shouldn't have to be re-taught your project every session. Jumbo gives them the context they're missing, so you don't have to. @@ -70,7 +70,7 @@ Those solve the core problems. These make Jumbo pleasant to use: ``` Jumbo will guide you in getting started. -3. Run your agent as normal (claude, codex, gemini, vibe) +3. Run your agent as normal (claude, codex, agy, vibe) ``` bash > claude @@ -212,7 +212,7 @@ Change agents and models at will. Jumbo just picks up where you left off.
What coding agents does jumbo work with? -Jumbo has been battle-tested with Claude Code, GitHub Copilot, and Gemini. More are to be verified soon... +Jumbo has been battle-tested with Claude Code, GitHub Copilot, Codex, and Antigravity. More are to be verified soon...
diff --git a/assets/agent-files/json/antigravity-hooks.fragment.json b/assets/agent-files/json/antigravity-hooks.fragment.json new file mode 100644 index 00000000..ff87cee4 --- /dev/null +++ b/assets/agent-files/json/antigravity-hooks.fragment.json @@ -0,0 +1,11 @@ +{ + "jumbo-session-bootstrap": { + "PreInvocation": [ + { + "type": "command", + "command": "node .agents/jumbo/antigravity-hook.mjs pre-invocation-bootstrap", + "timeout": 30 + } + ] + } +} diff --git a/assets/agent-files/json/gemini-settings.fragment.json b/assets/agent-files/json/gemini-settings.fragment.json deleted file mode 100644 index fbef99ad..00000000 --- a/assets/agent-files/json/gemini-settings.fragment.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "hooks": { - "SessionStart": [ - { - "matcher": "startup", - "hooks": [ - { - "type": "command", - "command": "jumbo session start" - } - ] - }, - { - "matcher": "compress", - "hooks": [ - { - "type": "command", - "command": "jumbo work resume" - } - ] - } - ], - "PreCompress": [ - { - "matcher": "auto", - "hooks": [ - { - "type": "command", - "command": "jumbo work pause" - } - ] - } - ] - }, - "tools": { - "allowed": [ - "run_shell_command(jumbo --help)" - ] - } -} - diff --git a/assets/agent-files/scripts/antigravity-hook.mjs b/assets/agent-files/scripts/antigravity-hook.mjs new file mode 100644 index 00000000..5673df05 --- /dev/null +++ b/assets/agent-files/scripts/antigravity-hook.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; + +const MANAGED_MARKER = "JUMBO_MANAGED_ANTIGRAVITY_HOOK_RUNNER_V1"; +const MAX_INJECTED_MESSAGE_LENGTH = 12_000; + +void MANAGED_MARKER; + +const mode = process.argv[2] ?? ""; +const input = parseJson(await readStdin()); + +try { + if (mode === "pre-invocation-bootstrap") { + await handlePreInvocationBootstrap(input); + } else { + writeJson({ injectSteps: [] }); + } +} catch (error) { + writeJson({ + injectSteps: [ + { + ephemeralMessage: `Jumbo Antigravity hook failed: ${getErrorMessage(error)}`, + }, + ], + }); +} + +async function handlePreInvocationBootstrap(input) { + if (input.invocationNum !== 0) { + writeJson({ injectSteps: [] }); + return; + } + + const cwd = getWorkspaceCwd(input); + const result = await runCommand("jumbo", ["session", "start", "--format", "text"], cwd); + const output = [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); + + if (!output) { + writeJson({ injectSteps: [] }); + return; + } + + const prefix = result.exitCode === 0 ? "" : `jumbo session start exited with code ${result.exitCode}.\n\n`; + writeJson({ + injectSteps: [ + { + ephemeralMessage: limitTextTail(`${prefix}${output}`, MAX_INJECTED_MESSAGE_LENGTH), + }, + ], + }); +} + +function runCommand(command, args, cwd) { + return new Promise((resolve) => { + const isWin = process.platform === "win32"; + let child; + if (isWin) { + const commandLine = [command, ...args].map(arg => { + return /[ \"]/.test(arg) ? `"${arg.replace(/"/g, '\\"')}"` : arg; + }).join(" "); + child = spawn(commandLine, [], { + cwd, + shell: true, + stdio: ["ignore", "pipe", "pipe"], + }); + } else { + child = spawn(command, args, { + cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + } + + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", (chunk) => { + stdout = limitTextTail(`${stdout}${chunk.toString()}`, MAX_INJECTED_MESSAGE_LENGTH); + }); + + child.stderr?.on("data", (chunk) => { + stderr = limitTextTail(`${stderr}${chunk.toString()}`, MAX_INJECTED_MESSAGE_LENGTH); + }); + + child.on("close", (code) => { + resolve({ exitCode: code ?? 1, stdout, stderr }); + }); + + child.on("error", (error) => { + resolve({ exitCode: 1, stdout, stderr: stderr || error.message }); + }); + }); +} + +function getWorkspaceCwd(input) { + if (Array.isArray(input.workspacePaths) && typeof input.workspacePaths[0] === "string") { + return input.workspacePaths[0]; + } + + return process.cwd(); +} + +function readStdin() { + return new Promise((resolve) => { + let input = ""; + process.stdin.setEncoding("utf-8"); + process.stdin.on("data", (chunk) => { + input += chunk; + }); + process.stdin.on("end", () => resolve(input)); + }); +} + +function parseJson(value) { + try { + return value.trim() ? JSON.parse(value) : {}; + } catch { + return {}; + } +} + +function writeJson(value) { + process.stdout.write(JSON.stringify(value)); +} + +function getErrorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +function limitTextTail(value, maxLength) { + return value.length > maxLength ? value.slice(-maxLength) : value; +} diff --git a/assets/gemini-logo.svg b/assets/gemini-logo.svg deleted file mode 100644 index f1cf3575..00000000 --- a/assets/gemini-logo.svg +++ /dev/null @@ -1 +0,0 @@ -Gemini \ No newline at end of file diff --git a/docs/getting-started/what-jumbo-creates.md b/docs/getting-started/what-jumbo-creates.md index f0860812..4179f86c 100644 --- a/docs/getting-started/what-jumbo-creates.md +++ b/docs/getting-started/what-jumbo-creates.md @@ -47,7 +47,7 @@ Jumbo creates or updates several files outside `.jumbo/` to integrate with AI co | `JUMBO.md` | Bootstrap-only Jumbo instruction file | | `AGENTS.md` | Points agents to `JUMBO.md` | | `CLAUDE.md` | Points Claude Code to `JUMBO.md` | -| `GEMINI.md` | Points Gemini CLI to `JUMBO.md` | +| `GEMINI.md` | Points Antigravity-compatible Gemini context loading to `JUMBO.md` | | `.github/copilot-instructions.md` | Points GitHub Copilot to `../JUMBO.md` | | `.cursor/rules/jumbo.mdc` | Points Cursor to `JUMBO.md` with always-apply rules frontmatter | @@ -59,17 +59,18 @@ Jumbo creates or updates several files outside `.jumbo/` to integrate with AI co |---|---| | `.claude/settings.json` | Claude Code hooks for session start and compaction | | `.codex/hooks.json` | Codex hooks for session start and compaction using text-mode Jumbo output | -| `.gemini/settings.json` | Gemini CLI hooks for session start and compression | +| `.agents/hooks.json` | Antigravity hooks for session bootstrap | +| `.agents/jumbo/antigravity-hook.mjs` | Antigravity hook runner that returns documented JSON hook envelopes | | `.github/hooks/hooks.json` | GitHub Copilot hooks for session start | | `.cursor/hooks.json` | Cursor hook for session start | -These hooks load the session router when an agent session begins and preserve work state before compaction or compression. +These hooks load the session router when an agent session begins and preserve work state before supported lifecycle events. **Managed skills:** Jumbo copies workflow and maintenance skills from `assets/skills` into the selected agents' skill directories, including bootstrap/session use, lifecycle hooks, command discovery, context maintenance, and correction capture. Additive initialization does not overwrite existing managed skill directories; repair refreshes Jumbo-managed skills from assets while preserving user-created skills. -Codex skills are installed to `.agents/skills`. Jumbo may remove obsolete Codex skill copies from `.codex/skills` during repair or evolve, but only when the obsolete directory is byte-identical to the current managed template and contains no extra user files. +Codex and Antigravity skills are installed to `.agents/skills`. Jumbo may remove obsolete Codex skill copies from `.codex/skills` during repair or evolve, but only when the obsolete directory is byte-identical to the current managed template and contains no extra user files. --- diff --git a/docs/reference/commands/work.md b/docs/reference/commands/work.md index c8af3003..30c39cc9 100644 --- a/docs/reference/commands/work.md +++ b/docs/reference/commands/work.md @@ -23,7 +23,7 @@ Long-running daemon that continuously polls for goals in `defined` state and del | Option | Required | Default | Description | |--------|----------|---------|-------------| -| `--agent ` | Yes | — | Agent to delegate refinement to. Supported: `claude`, `gemini`, `copilot`, `codex`, `cursor`, `vibe` | +| `--agent ` | Yes | — | Agent to delegate refinement to. Supported: `claude`, `antigravity`, `copilot`, `codex`, `cursor`, `vibe` | | `--poll-interval ` | No | `30` | Seconds to wait between polling for new goals | | `--max-retries ` | No | `3` | Max retry attempts per goal before skipping | @@ -69,7 +69,7 @@ Long-running daemon that continuously polls for goals in `submitted` state and d | Option | Required | Default | Description | |--------|----------|---------|-------------| -| `--agent ` | Yes | — | Agent to delegate review to. Supported: `claude`, `gemini`, `copilot`, `codex`, `cursor`, `vibe` | +| `--agent ` | Yes | — | Agent to delegate review to. Supported: `claude`, `antigravity`, `copilot`, `codex`, `cursor`, `vibe` | | `--poll-interval ` | No | `30` | Seconds to wait between polling for new goals | | `--max-retries ` | No | `3` | Max retry attempts per goal before skipping | diff --git a/docs/reference/project-initialization.md b/docs/reference/project-initialization.md index 729fc57a..6a1116c2 100644 --- a/docs/reference/project-initialization.md +++ b/docs/reference/project-initialization.md @@ -89,7 +89,7 @@ Jumbo configures hooks for popular AI coding assistants: | **Claude Code** | `CLAUDE.md`, `.claude/settings.json`, and `.claude/skills` | | **Codex** | `.codex/hooks.json` and `.agents/skills` | | **GitHub Copilot** | `.github/copilot-instructions.md`, `.github/hooks/hooks.json`, and `.agents/skills` | -| **Gemini CLI** | `GEMINI.md`, `.gemini/settings.json`, and `.gemini/skills` | +| **Antigravity CLI** | `GEMINI.md`, `.agents/hooks.json`, `.agents/jumbo/antigravity-hook.mjs`, and `.agents/skills` | | **Cursor** | `.cursor/rules/jumbo.mdc` and `.cursor/hooks.json` | | **Vibe** | `.vibe/skills` | | **All agents** | `JUMBO.md` and `AGENTS.md` | @@ -102,6 +102,8 @@ Jumbo-owned markdown files and JSON hook/settings fragments are loaded from `ass For Codex, Jumbo uses the repository skill directory `.agents/skills` and keeps `.codex` for documented Codex hooks/configuration. During repair and evolve, obsolete Jumbo-managed skill copies under `.codex/skills` are removed only when they exactly match the current managed templates; customized skills and extra user files are preserved. +For Antigravity, Jumbo uses the documented `agy` CLI target, keeps `GEMINI.md` as a thin compatibility reference to `JUMBO.md`, and manages workspace hooks through `.agents/hooks.json`. Antigravity repair removes obsolete Jumbo-managed Gemini hook/settings files while preserving unrelated user-owned `.gemini` content. + --- ## Update project settings diff --git a/evals/.gitignore b/evals/.gitignore deleted file mode 100644 index f29edf32..00000000 --- a/evals/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -.jumbo/ -.jumbo/jumbo.db -node_modules/ -dist/ -.env* -*.tsbuildinfo -.eval-store -.agents -.claude -.gemini -.github -.vibe -AGENTS.md -CLAUDE.md -GEMINI.md -VIBE.md -JUMBO.md diff --git a/evals/CHANGELOG.md b/evals/CHANGELOG.md deleted file mode 100644 index 664b6d3d..00000000 --- a/evals/CHANGELOG.md +++ /dev/null @@ -1,26 +0,0 @@ -# Changelog - -## Unreleased - -- `eval run` now auto-prints the full `ReportGenerator` output (divergence curve, lift %, divergence onset, disruption-impact ranking, per-comparison audit trail) after a successful comparison loop, mirroring `eval report`'s default of excluding tampered comparisons from aggregates with a notice. `eval report --scenario ` continues to work unchanged for post-hoc reporting. -- Codify methodology framing: a new decision (`Treatment under test is the whole Jumbo system, not memory in isolation`) names the active methodology as framework-as-developer orchestration with agent-driven lifecycle. README gains a `Methodology` section; the byte-identical claim is qualified to apply to the scenario prompt before variant-specific wrapping; `formatFullReport` terminal output now carries explicit `Treatment` / `Methodology` / `Lift attribution` lines; the `AuditTrail` TSDoc adds a methodology note. Headline lift attributes to the whole Jumbo system (init + pre-seeded memory + lifecycle protocol + agent capture + audit), not to memory delivery in isolation. -- Redefine baseline-arm parity: the baseline arm now receives only the scenario prompt — no Jumbo collaboration block, no `jumbo` binary reachable on PATH. `LocalExecutor.installJumboShim` writes a fail-loud `jumbo` / `jumbo.cmd` / `jumbo.bat` shim into the baseline workdir's `.eval-bin` and returns a PATH-prefixed env that ABRunner threads into every baseline subprocess (probe + harness via `runSession`). A new pre-session probe asserts jumbo is unreachable from baseline; any leak aborts the run with `JumboBaselineLeakError` before session work begins. `LocalExecutor.exec` now accepts an `env` option that merges onto `process.env`; `runSession` forwards it. Treatment under test is documented as the whole Jumbo system (orchestration + memory + protocol), with explicit invariants for scenario-prompt byte-equality across arms and Jumbo-collaboration-block exclusivity. -- Score agent-driven Jumbo usage and protocol adherence: `JumboMemoryCaptureScorer` now credits only entities the agent registered during each session — the diff between a pre-harness and a post-harness Jumbo memory snapshot — across decisions, guidelines, invariants, components, relations, and dependencies. Pre-seeded memory and entities from prior sessions no longer inflate recall. New `ProtocolAdherenceScorer` emits per-step pass/fail across the six prescribed lifecycle steps (session start, goal start, in-session captures, progress updates, goal submit, session end) and an aggregate mean per Jumbo arm; both scorers contribute to `ComparisonResult.jumboScores`/`baselineScores`/`deltas`/timelines. `FullReport` and `formatComparisonOutput` gain a `PROTOCOL ADHERENCE EVIDENCE` section so non-adherence is first-class signal. `SessionRecord` gains `jumboMemorySnapshotBefore`; `JumboLifecycleAudit` gains `inSessionCapturesExecuted`, `progressUpdatesExecuted`, `goalVersionBefore/After`, and `newEntityCount`. Baseline arm remains `maxScore=0` (N/A) for both new dimensions. -- Agent-driven Jumbo lifecycle: ABRunner no longer issues `jumbo session start` or `jumbo session end` on the agent's behalf. The Jumbo arm prompt now composes (a) the scenario task, (b) the framework-assigned active goal-id, and (c) an explicit lifecycle protocol the agent must follow (`jumbo session start`, `jumbo goal start --id `, capture decisions/components/relations, `jumbo goal submit --id `, `jumbo session end`). After each session the framework verifies execution via `jumbo goal show` / `jumbo sessions list` / `jumbo decisions list` and stores the result as `SessionRecord.jumboLifecycleAudit` (four booleans + raw evidence). `buildEffectivePrompt`, `JumboSessionEndError`, `JumboSessionStartError`, and `SessionRecord.jumboSessionEnd` are removed; `SessionPhaseTimings` is now `{ harnessExec, lifecycleAudit? }` and `SessionHeartbeatPhase` is `'harness-exec' | 'lifecycle-audit' | 'scoring'`. -- Add `TestScenario.jumboPlan` for progressive backlog release: scenarios can now declare `preSeededMemory` (decisions/components/invariants/dependencies/relations) that ABRunner registers via real `jumbo` CLI commands after `jumbo init` and before session 1, and `goals[]` tagged with `sessionAvailableFrom`. At each session boundary the framework runs `jumbo goal add` for goals due that session and threads the active goal-id into the prompt so the agent is told which goal to start — the harness picks the active goal, the agent executes the lifecycle. Scenarios without a `jumboPlan` keep today's behavior. Seeder/goal-add failures abort the run before sessions begin (lifecycle-setup invariant). -- `eval status --watch ` now renders via an ink-based TUI in interactive shells (in-place updates, clean teardown on completion or SIGINT) and falls back to a single final snapshot in non-TTY environments (Git Bash on Windows, piped output, the Claude Code `!` shell), fixing the stacked-output bug where the heartbeat block was appended on every poll. -- Fix `eval run` hanging before session execution: ABRunner now invokes `jumbo init` with `--non-interactive --name --yolo` and fails fast on non-zero init exits instead of silently continuing or waiting on prompts. -- Add live read-only run visibility: `eval run` now creates `.eval-store/runs//run.json`, writes heartbeat state to `.eval-store/runs//state.json`, prints the run ID immediately, records per-session phase timings, and `eval status --watch ` renders per-harness, per-variant, per-session progress without feeding state back into the runner. -- Wire `eval report`: action handler now loads scored comparison metadata through the injected `storeProvider`, validates harness and dimension filters before store I/O, generates `FullReport`, prints terminal output, emits the `jumbo-eval-report` v1 JSON envelope under `--json`, and prints an empty-state message when no comparisons exist for the scenario. -- Wire `eval score`: action handler now loads a scenario and completed `TestResult`s via the injected `storeProvider`, supports `--result` narrowing, runs the five deterministic scoring dimensions, prints `formatScoreOutput`, applies baseline memory capture as N/A (`maxScore=0`), and persists scored result metadata/comparison data through the existing `ResultStore.saveTestResult` path. -- Wire `eval run`: action handler now loads the scenario from the store, validates `--harness` names before any execution, applies `--sessions` override onto `scenario.sessionCount`, and invokes `runABComparison` once per harness with a per-harness adapter and a shared `LocalExecutor`. Per-harness `formatComparisonOutput` is printed; when 2+ harnesses run, `formatCrossHarnessComparison` is appended. `createProgram` and `registerRunCommand` accept an optional `abRunner` override for unit-testing the action handler without invoking real harnesses. -- Fix broken CLI Quick Start: `scenario create --from-template` now persists the scenario via `JsonResultStore` and prints its UUID; `scenario list` now reads from the store and prints registered scenarios (with `--json` for machine-readable output). Previously both action handlers were stub placeholders. -- Fix Windows entry-point guard in `src/cli/index.ts`: replaced brittle `process.argv[1]?.includes('cli/index')` substring check with `isMainModule(__filename, process.argv[1])` using `path.resolve`, so the CLI runs correctly when invoked from PowerShell with backslash paths. -- `createProgram(deps?)` is now the CLI composition root, accepting an optional `storeProvider` factory (default: lazy `JsonResultStore` at `EVAL_STORE_PATH` or `.eval-store/`). Action handlers receive dependencies via `register*Command(parent, deps)` rather than constructing infra inline — enables in-memory `ResultStore` substitution in tests. -- Add deterministic end-to-end validation suite (`tests/unit/end-to-end-validation.test.ts`) that uses a fake harness whose output depends only on the delivered Jumbo context, proving positive lift only when the prior-session fact is delivered, zero lift in the no-fact control, separate temp workdir isolation, and full-comparison invalidation on either Jumbo session-end or harness failure. -- Add `HarnessExecutionError`: `runSession` now throws on non-zero harness exit codes, invalidating the comparison before any partial result is persisted. -- Extend `FullReport` with an `auditTrails` section (per-harness effective context per session, final workspace snapshot, and per-dimension scoring evidence with deltas) — surfaced in the terminal report and the JSON v1 export so every measured delta is explainable from inputs and outputs alone. -- Security: `LocalExecutor.captureWorkspaceSnapshot` now excludes secret-prone filenames (`.env*`, `*.key`, `*.pem`, `*.p12`, `*.pfx`, `*.crt`, `*.cert`, `id_rsa/dsa/ecdsa/ed25519`, `credentials.*`, `secrets.*`, `api_keys.*`) before reading file content, preventing accidental secret capture in workspace snapshots. -- Add Jumbo memory capture compliance dimension: after each Jumbo session ABRunner runs five `jumbo {kind} list --format json` commands and stores a `JumboMemorySnapshot` on the `SessionRecord`. `JumboMemoryCaptureScorer` computes precision, recall, and F1 against `expectedJumboMemoryCaptures` declared in `TestScenario`. Baseline runs are always scored N/A (maxScore=0). Evidence (matched/missing/spurious) surfaces in comparison display and full reports. -- Derive `jumbo session end` focus and summary from observed session output, modified files, delivered context, and scheduled disruptions. -- Store `jumboSessionEnd` command diagnostics on Jumbo `SessionRecord` transcripts and fail the comparison before scoring when session-end fails. diff --git a/evals/GOAL.md b/evals/GOAL.md deleted file mode 100644 index 7dd1c2e6..00000000 --- a/evals/GOAL.md +++ /dev/null @@ -1,166 +0,0 @@ -# Eval Framework - -## Objective - -Produce an evaluation framework comparing coding-agent performance when working in a project with Jumbo CLI initialized versus a plain no-Jumbo setup, and yields durable, statistically valid, comparable side-by-side metrics across repeatable scenarios. - -## Scope - -### In scope -- Multiple evaluation scenarios, including at least one null-hypothesis scenario (see Scenarios). -- Durable persistence of per-run metrics with replication support. -- Structural artefact-based scoring (workspace file content), not transcript keyword matching. - -### Out of scope -- API-based agent interactions (explicitly disallowed — harnesses only). - -## Baseline - -The "without Jumbo" setup is **plain agent execution**: -- Use the same agent harness as the with-Jumbo setup. -- No `JUMBO.md`, `AGENTS.md`, or `.claude/settings.json` present in the working tree. -- No `.jumbo/` directory present. -- `jumbo` restricted in the agent's permission set. -- The baseline workdir is asserted clean of Jumbo instruction files before session 1 begins. - -The "with Jumbo" setup uses the same agent in the same starting repo state, with Jumbo initialized and `jumbo` available. - -## Outcomes - -A run of this goal succeeds when each outcome below is true. - -1. **A scenario can be specified in a durable, reusable format.** - - A schema defines starting repo state, instructions, invariants, and expected outcomes. - - The framework supports addition of new scenarios without modifying runner or scoring code. - - Every scenario declares `retentionPatterns` as the primary retention signal. - -2. **A scenario can be executed end-to-end against a configured agent in both setups.** - - At least one authored scenario with A/B variants runs to completion in both setups. - - Each run is identified by a stable run ID. - - The workspace snapshot captures `.jumbo/events/` (event file names and aggregate counts) in addition to user-authored files, so the Jumbo event log is part of the scoring evidence. - -3. **Every run produces durably persisted, retrievable metrics.** - - Metrics are retrievable by run ID after the harness exits. - - Token usage is recorded per run and per session. - - Per-session workspace snapshots are stored (not only the final snapshot) to support per-session scoring. - -4. **Each run is scored on both rule-based and model-reviewed dimensions.** - - Knowledge retention is scored against structural assertions on workspace artefacts, not keyword presence in transcripts. - - Jumbo arm quality scores (file accuracy, retention, disruption recovery) are only computed for sessions where lifecycle adherence meets the minimum threshold (see Methodology). - - Token efficiency is only reported when both arms produced functionally equivalent outputs (see Scoring). - - Rule-based and model-based review are produced by a single scoring pipeline. - -5. **The two setups can be compared side-by-side with statistical confidence.** - - A comparison artifact is produced from K≥5 replicated runs per arm per scenario. - - Lift numbers are reported as mean ± standard deviation, not as single-point estimates. - - A result is considered a valid signal only when the lift exceeds one standard deviation. - -## Methodology - -### Terms -- **Scenario:** The durable task specification shared across both setups: starting repo state, prompt/goal descriptions, invariants, structural assertions, expected outcomes, and any scripted disruptions. -- **Run:** One end-to-end attempt to execute a scenario in one setup, producing a stable run ID and a durable record of outputs and metrics. -- **Replication:** K independent runs of the same scenario in the same setup, used to estimate variance and compute confidence intervals. K≥5 is the minimum for any claim. -- **Session:** One contiguous agent interaction period within a run. A run contains one or more sessions. -- **Setup:** One execution condition for a scenario. In v1, the two setups are Jumbo and baseline. -- **Adherence:** The degree to which the Jumbo arm agent executed the prescribed lifecycle (session start, goal start, in-session captures, goal submit, session end), verified post-session from Jumbo's. - -### Replication requirement - -A single run per arm is an anecdote, not a measurement. LLM outputs are stochastic: the same prompt produces different code, different files, and different token counts across runs. Without replication, there is no variance estimate, no standard error, and no significance test. Any lift number derived from a single run cannot be distinguished from noise. - -**Minimum K:** 5 replications per arm per scenario. With K=5, a t-test at α=0.05 (one-tailed, df=4) requires t > 2.13 to claim a significant result. This is the minimum credible threshold. - -**Reporting standard:** All lift numbers are reported as mean(Jumbo) − mean(baseline) ± pooled standard deviation. A lift is reported as a signal only when it exceeds one standard deviation of the baseline distribution. - -**Cost acknowledgement:** K=5 at 6–7 sessions per run is 30–35 harness invocations per arm per scenario. This is the unavoidable cost of a valid measurement. The framework must support batch replication (launching K runs automatically and aggregating results) to make this tractable. - -Reference: AMA-Bench: Evaluating Long-Horizon Memory for Agentic Applications https://arxiv.org/pdf/2602.22769 - -### Keeping both setups comparable - -The Jumbo arm's effective prompt wraps the scenario task with a framework-assigned active goal-id and an explicit lifecycle protocol. This is an intentional structural difference — the agent is told what to do and given a tool to do it. The baseline arm receives only the scenario task. - -**This is an acknowledged prompt asymmetry, not a controlled comparison.** The measured lift therefore attributes to the entire Jumbo system (memory + protocol + goal handoff), not to memory in isolation. Any published lift claim must carry this attribution explicitly. - -A future ablation arm (protocol-only, no Jumbo binary or memory) would allow separating prompt structure effects from memory effects. This is however a required step before claiming that memory specifically drives the lift. - - -### Workspace evidence: capturing `.jumbo/` - -The workspace snapshot must include Jumbo's own state as scoring evidence. Specifically: -- `.jumbo/events/` — a structured summary of event files by aggregate type and count. Full file content is not required; filenames and counts are sufficient to verify that the agent registered entities during the session. -- The pre-session and post-session entity diff (`JumboMemorySnapshot`) is the primary evidence for in-session captures. The event log summary provides a secondary, non-CLI-mediated confirmation. - -The event log is the ground truth of what happened inside Jumbo. CLI command outputs are a derived view; if a CLI command fails silently, the event log is the fallback. Without it, the audit trail claim of full explainability is false. - -### Comparing token usage fairly - -Token efficiency is only valid when both arms produced functionally equivalent outputs. Equivalence is defined operationally as: both arms produced all expected files, and both arms' structural assertion scores meet a minimum threshold (≥0.8 of the maximum possible score on the structural assertion for this scenario). - -Without output equivalence, token efficiency compares the cost of producing different things — a meaningless ratio. - -**When equivalence is not met:** token efficiency is reported as N/A for that replication. Across K replications, report the fraction of replications where equivalence was achieved, and compute token efficiency only over that subset. - -Computing quality score from the same sessions used to compute token count is resolved by using the structural assertion score — which is computed independently of token counts — as the quality denominator. - -### Structural assertions vs. keyword matching - -Knowledge retention is not measurable by checking whether a term appears in the agent's output text. An agent can write "we are not using discriminated unions here" and score full marks on the term "discriminated union." Furthermore, several retention patterns appear verbatim in the scenario's initial prompt — a stateless agent echoing the prompt scores identical retention to one that correctly implemented the domain. - -**Retention is measured on artefacts.** Each scenario declares `structuralAssertions`: per-session checks on the actual files in the workspace. - -`retentionPatterns` (keyword strings) may be retained as a lightweight pre-filter to decide whether to run the more expensive structural assertion. They are not the primary signal. - -### Scenarios - -The scenario set must include at minimum: - -1. **Complexity scenarios (existing):** Multi-session, multi-disruption domain-modeling tasks where Jumbo is expected to lift. These test the hypothesis under favourable conditions. - -2. **Null-hypothesis scenario:** A single-session task where Jumbo should provide zero or negligible lift. Jumbo cannot improve cross-session retention if there is only one session. If Jumbo shows statistically significant lift on a single-session scenario, the measurement is detecting an artefact (prompt structure, scoring bias, or test contamination), not memory. This scenario is a validity check on the measurement framework itself. A passing eval should show near-zero lift here. - -3. **File-reconstruction scenario:** A multi-session task on a codebase where all architectural decisions are legible in the existing files — an agent that carefully reads prior-session output can reconstruct sufficient context without Jumbo. This tests whether Jumbo's advantage is specific to context that cannot be recovered from file artefacts, or whether it is simply a better file-reading aid. Expected result: lower lift than the complexity scenarios, because the agent has an alternative recovery path. - -## Scoring - -Two tiers. - -### Rule-based checks - -- **File Accuracy** — Were the expected files produced, and are they present in the workspace snapshot? Rubric: exact set, superset allowed, or path-prefix match — declared per scenario. Scored per session using per-session snapshots. Files exceeding the snapshot size cap are recorded as `skipped` (not `absent`) and treated as present for scoring purposes. - -- **Knowledge Retention** — Do structural assertions on workspace artefacts pass in the sessions they are declared for? Each scenario declares assertions as structured checks on file content (AST-level or pattern-level, not keyword presence in transcripts). A session's retention score is the fraction of assertions due in that session that pass. - -- **Token Efficiency** — Tokens consumed per arm, normalised by the structural assertion score (not by file accuracy or keyword retention). Only reported when both arms achieved output equivalence (≥0.8 structural assertion score). Not a standalone dimension — reported as a ratio with explicit equivalence status. - -- **Adherence Rate** — Fraction of Jumbo arm sessions meeting minimum lifecycle adherence. Reported separately from quality scores, not averaged into the lift. A low adherence rate is a diagnostic finding, not a quality failure of Jumbo — it indicates the agent is not following the protocol, which is a separate signal. - -### Model-based review - -- **Consistency** — Naming, architectural, and styling conventions held across sessions. Review prompt and rubric versioned alongside the scenario. Judge model must be from a different provider than the agent under test. Rubric questions are atomic and binary-anchored (not holistic 1–5 scales) to reduce self-preference bias. -- **Adaptability** — When the scenario injects a disruption, does the agent's output register and fully integrate it, including retrofitting prior artefacts? - -## Constraints - -- Must run against coding-agent harnesses; no direct API-based interactions. -- Scenarios must be repeatable: same scenario + same agent + same seed (where seedable) yields comparable runs within measurement variance. -- The framework must be extensible to additional agent harnesses without redesign. -- Replication (K≥5 runs per arm) must be automatable as a single `eval run --replicate K` command — not a manual multi-step process. -- All lift claims in reports must carry their replication count, mean, standard deviation, and significance status explicitly. - -## Challenges - -- **Cost of replication:** K=5 per arm per scenario is expensive. The framework must support parallel execution of replications (two arms × K runs, not sequentially) and interleaved session execution (alternating Jumbo and baseline sessions within each replication) to reduce temporal confounding. -- **Structural assertion authoring:** Writing per-scenario TypeScript structural assertions is expert work. Each new scenario requires a domain expert to specify what correct implementation looks like at the file level. -- **Adherence variance:** Agents may adhere to the lifecycle inconsistently across replications. The adherence rate is itself a stochastic variable. The report must distinguish low-lift-due-to-non-adherence from low-lift-due-to-memory-not-helping. - -## Open Questions - -- **Ablation arm for prompt structure:** A protocol-only, no Jumbo binary or memory to separate prompt structure effects from memory effects? -- **Structural assertion tooling:** Should assertions use the TypeScript compiler API (precise but fragile), a regexp-based structural match (approximate but robust), or a combination? What is the calibration process for new assertions? -- **Parallel replication infrastructure:** Can the host machine sustain K parallel harness invocations without resource contention affecting measurement? What is the right K given the available budget per scenario? -- Should this framework live in the Jumbo CLI repo, or remain a sibling project? -- Can an evals sandbox be containerized while still driving non-API agents (Claude Code CLI, Codex CLI, Gemini CLI)? [Need to verify with Docker Sandboxes (in early access preview)] -- Can defining anti-patterns as desired invariants provide an easy measure of adherence? Models should implement established patterns and best practices by default - if not they should improve over time. Does instructing a model to deviate from best practices and established patterns provide clear signal for measuring the level of adherence? -- What is the durable persistence backend? diff --git a/evals/JUMPSTART.md b/evals/JUMPSTART.md deleted file mode 100644 index d689c0b6..00000000 --- a/evals/JUMPSTART.md +++ /dev/null @@ -1,240 +0,0 @@ -# Jumbo Evals — Project Jumpstart - -## Project Purpose - -Prove that Jumbo's workflow — systematic knowledge capture, curated context delivery, and review-driven correction — measurably improves coding agent performance over time, across sessions, and across agent harnesses. The eval simulates realistic multi-session projects with and without Jumbo active, measuring divergence as agent amnesia compounds in the non-Jumbo runs. - -## Hypothesis - -Over the lifecycle of a multi-goal project spanning multiple sessions, agents using Jumbo's workflow will produce measurably better outcomes than agents relying on native harness capabilities alone. The advantage compounds over time as knowledge accumulates and sessions reset. - -## `jumbo init` Purpose (copy-paste ready) - -``` -Prove that Jumbo's workflow measurably improves coding agent performance over time by simulating realistic multi-session projects with and without Jumbo active across agent harnesses, measuring how agent amnesia compounds and quantifying Jumbo's lift in knowledge retention, consistency, and efficiency. -``` - -## Key Design Decisions - -### Harness-based execution, not direct API calls - -Jumbo integrates with coding agent harnesses (Claude Code CLI, Codex CLI, Gemini CLI, etc.), not raw LLM APIs. Evals must test through the same harnesses to produce apples-to-apples comparisons. The system invokes harness CLIs via subprocess — it does not call LLM APIs directly. - -### Longitudinal multi-session simulation, not single-prompt evals - -Jumbo's core value is solving agent amnesia across sessions. Within a single session, any agent can track its own plan. The differentiation emerges across session boundaries when agents lose their context window. Each eval run simulates multiple sessions with fresh agent invocations against a persistent workspace. - -### Same container, different configuration - -Both Jumbo and non-Jumbo runs use an identical container image with all tools installed (including Jumbo). The only difference is whether `jumbo init` is run at project setup. This eliminates any "unfair advantage" argument — both environments have identical tooling available. - -### Persistent container, fresh agent per session - -The container persists across sessions within a test run (filesystem, including Jumbo's repo-local data, carries forward). Each session is a fresh harness CLI invocation — the agent starts with a clean context window. This simulates real-world agent amnesia: the Jumbo agent recovers via `jumbo session start`; the non-Jumbo agent has only whatever it wrote to files it knows to look for. - -### Hybrid scoring - -- **Deterministic checks**: file accuracy, pattern presence/absence, test pass/fail — objective, reproducible. -- **LLM-as-judge**: convention adherence, architectural quality — using locked rubrics with evidence anchoring. The judge LLM must differ from the runner to avoid self-model bias. -- **Task prompts are self-contained**: both runners receive identical task descriptions with sufficient information to succeed. No scoring criterion references information only available via Jumbo entities. What Jumbo proves is that structured, automatic context delivery produces better adherence than relying on the agent to extract and retain everything from natural language across sessions. - -### Technology stack - -TypeScript (Node.js) orchestrator running on the host. Containerized workspaces (Docker) for isolated, reproducible test environments. JSON file storage for results, with a SQLite graduation path if query patterns become complex. - -## Measurement Dimensions - -| Dimension | What it captures | -|-----------|-----------------| -| **Knowledge retention** | Does session N remember decisions from session 1? | -| **Consistency** | Do patterns established early hold across later goals? | -| **Error correction** | Are mistakes caught and not reintroduced after session boundaries? | -| **Rework rate** | How often does work need to be redone due to lost context? | -| **Efficiency** | Tokens consumed, iterations needed, goals completed per session | -| **Disruption recovery** | When mid-project corrections are introduced, do they persist? | -| **Token efficiency** | Tokens consumed to achieve equivalent or better outcomes — does Jumbo's curated context reduce total token spend? | - -## Goal Scaffold — Tracer Bullet Decomposition - -Each goal delivers a thin, end-to-end vertical slice. After each goal, the system is buildable, runnable, and demonstrates a new traced capability. Later goals widen the path established by earlier ones. - -Run these commands after `jumbo init` to set up the project backlog. Goals are chained in dependency order. - -### Goal 1 — Single-Session Smoke Test - -```bash -jumbo goal add \ - --title "[EVAL-1/9] Single-session smoke test" \ - --objective "Deliver the thinnest possible end-to-end path: define minimal domain types (TestScenario, SessionRecord, TestResult with just enough fields), build a Dockerfile with Claude Code CLI and Jumbo installed, implement a ContainerManager that can create/start/exec/destroy a container, implement a minimal HarnessAdapter for Claude Code CLI ('claude -p'), and wire it all together so that a single session of a single scenario runs inside a container, captures the agent's output, and stores a SessionRecord to a JSON-backed ResultStore. This is the 'hello world' of the eval system — one scenario, one session, one harness, stored result. TypeScript/Node.js project initialized with build tooling." \ - --criteria \ - "TypeScript project initialized with tsconfig, package.json, and build script" \ - "Minimal TestScenario type: initial prompt, session count" \ - "Minimal SessionRecord type: session number, harness, agent output, files modified, transcript" \ - "Minimal TestResult type: scenario reference, list of session records" \ - "ResultStore interface with JSON file implementation" \ - "Dockerfile with Node.js, Claude Code CLI, and Jumbo installed" \ - "ContainerManager: create, start, exec, stop, destroy lifecycle" \ - "Claude Code HarnessAdapter: invoke via 'claude -p', capture output" \ - "End-to-end: define scenario -> run one session in container -> store SessionRecord" \ - "Works on Docker Desktop for Windows 11" \ - --scope-in "Project setup" "Minimal domain types" "Dockerfile" "ContainerManager" "Claude Code adapter" "ResultStore" \ - --scope-out "Jumbo vs non-Jumbo comparison" "Multi-session" "Scoring" "Disruptions" "Reporting" "CLI" -``` - -### Goal 2 — A/B Comparison - -```bash -jumbo goal add \ - --title "[EVAL-2/9] A/B comparison" \ - --objective "Extend the smoke test to run the same scenario twice: once with JUMBO_ENABLED=true (jumbo init runs at setup) and once with JUMBO_ENABLED=false (Jumbo installed but not initialized). Still single-session. Add a minimal deterministic scorer that compares the two runs on file accuracy (did the agent modify the expected files?). Produce a simple side-by-side terminal output showing the Jumbo vs non-Jumbo results and the file accuracy score for each. This is the first actual measurement — proof that the A/B comparison pipeline works end-to-end." \ - --criteria \ - "JUMBO_ENABLED env var controls whether jumbo init runs at container setup" \ - "Same scenario executed in both Jumbo and non-Jumbo containers" \ - "Deterministic file accuracy scorer: compares files modified against expected list" \ - "TestResult extended with per-run scores and delta" \ - "Side-by-side terminal output showing both results and score delta" \ - "End-to-end: define scenario -> run both variants -> score -> display comparison" \ - --scope-in "A/B run orchestration" "JUMBO_ENABLED configuration" "File accuracy scorer" "Terminal comparison output" \ - --scope-out "Multi-session" "Disruptions" "LLM-as-judge" "Token tracking" "Additional harnesses" \ - --prerequisite-goals "" -``` - -### Goal 3 — Multi-Session Orchestration - -```bash -jumbo goal add \ - --title "[EVAL-3/9] Multi-session orchestration" \ - --objective "Extend the A/B comparison to run across N sessions with fresh agent invocations per session. The container persists (filesystem and Jumbo data carry forward) but each session is a new harness CLI invocation — simulating agent amnesia. Session 1 receives the initial broad objective; subsequent sessions receive a continuation prompt. For Jumbo runs, the session prompt instructs the agent to run 'jumbo session start' first and 'jumbo session end' at close. Add knowledge retention scoring: after session N, check whether decisions/patterns established in session 1 are still reflected in the workspace. Produce a per-session timeline in terminal output showing the knowledge retention trajectory — this is where the amnesia signal first becomes visible." \ - --criteria \ - "SessionOrchestrator runs N sessions per scenario with fresh agent invocations" \ - "Container persists across sessions — only agent context resets" \ - "Session 1 delivers initial prompt; subsequent sessions deliver continuation prompt" \ - "Jumbo runs include jumbo session start/end lifecycle" \ - "TestScenario extended with session count and session boundary definitions" \ - "Knowledge retention scorer: checks if session 1 decisions persist through session N" \ - "Per-session timeline output showing knowledge retention trajectory for both runs" \ - "End-to-end: multi-session scenario -> run both variants -> score retention -> show timeline" \ - --scope-in "Multi-session orchestration" "Continuation prompts" "Jumbo session lifecycle" "Knowledge retention scorer" "Timeline output" \ - --scope-out "Disruptions" "LLM-as-judge" "Token tracking" "Additional harnesses" "JSON export" \ - --prerequisite-goals "" -``` - -### Goal 4 — Disruption Injection - -```bash -jumbo goal add \ - --title "[EVAL-4/9] Disruption injection" \ - --objective "Add the ability to inject disruptions (mid-project corrections, scope changes, new constraints) at scheduled session boundaries. A Disruption is defined as part of the TestScenario: type (correction, scope change, new constraint), injection session number, and content. At the scheduled session, the disruption text is prepended to the session prompt. Add disruption recovery scoring: in sessions after a disruption, check whether the correction persisted or was lost at the next session boundary. This is where compounding divergence becomes measurable — the non-Jumbo agent forgets corrections while the Jumbo agent retains them." \ - --criteria \ - "Disruption type defined: correction, scope change, new constraint with injection point and content" \ - "TestScenario extended with disruption schedule" \ - "Disruptions injected by prepending to session prompt at scheduled session" \ - "Disruption recovery scorer: checks if corrections persist in subsequent sessions" \ - "Timeline output extended to show disruption points and recovery trajectories" \ - "End-to-end: scenario with disruptions -> run both variants -> score recovery -> show divergence" \ - --scope-in "Disruption types" "Disruption injection" "Disruption recovery scorer" "Extended timeline" \ - --scope-out "LLM-as-judge" "Token tracking" "Additional harnesses" "JSON export" \ - --prerequisite-goals "" -``` - -### Goal 5 — Token Efficiency Tracking - -```bash -jumbo goal add \ - --title "[EVAL-5/9] Token efficiency" \ - --objective "Capture token usage (input and output) per session from harness output and add token efficiency as a measurement dimension. Extend SessionRecord with token usage fields. Parse token counts from harness CLI output (each harness reports usage differently — the adapter is responsible for extracting it). Add a token efficiency scorer that compares total token spend across Jumbo vs non-Jumbo runs, normalized by outcome quality (tokens-per-quality-point). Add token usage to the per-session timeline and comparison output. This reveals whether Jumbo's curated context reduces total token spend while maintaining or improving outcomes." \ - --criteria \ - "SessionRecord extended with input/output token counts" \ - "Claude Code adapter extracts token usage from CLI output" \ - "Token efficiency scorer: total tokens normalized by outcome quality score" \ - "Per-session token usage shown in timeline output" \ - "Cumulative token comparison in A/B output" \ - "End-to-end: run scenario -> capture tokens -> score efficiency -> display in report" \ - --scope-in "Token capture" "Token efficiency scorer" "Adapter token extraction" "Extended output" \ - --scope-out "LLM-as-judge" "Additional harnesses" "JSON export" \ - --prerequisite-goals "" -``` - -### Goal 6 — LLM-as-Judge Scoring - -```bash -jumbo goal add \ - --title "[EVAL-6/9] LLM-as-judge scoring" \ - --objective "Add qualitative scoring via LLM-as-judge for dimensions that deterministic checks cannot reach: consistency (do patterns established early hold across later sessions?), error correction (are mistakes caught and not reintroduced?), and architectural quality. For each session pair (Jumbo vs non-Jumbo), submit transcripts and workspace snapshots to a judge LLM with a locked rubric containing specific questions and defined scoring scales. The judge LLM must be a different model than the runner to avoid self-model bias — make the judge model configurable. Combine deterministic and LLM-judge scores into a unified TestResult. This completes the hybrid scoring engine across all seven measurement dimensions." \ - --criteria \ - "LLM-as-judge scorer with locked rubric and defined scoring scales" \ - "Judge scores consistency, error correction, and architectural quality" \ - "Judge LLM configurable and enforced to differ from runner model" \ - "Rubric uses evidence anchoring — judge must cite specific transcript/file evidence" \ - "Deterministic and LLM-judge scores unified in TestResult" \ - "All seven measurement dimensions now scored" \ - "Unit tests with mock transcripts verifying rubric application" \ - --scope-in "LLM-as-judge scorer" "Locked rubrics" "Evidence anchoring" "Unified scoring" \ - --scope-out "Additional harnesses" "JSON export" "CLI commands" \ - --prerequisite-goals "" -``` - -### Goal 7 — Second Harness Adapter - -```bash -jumbo goal add \ - --title "[EVAL-7/9] Second harness adapter" \ - --objective "Add a second HarnessAdapter (Codex CLI or Gemini CLI — whichever is most readily available and testable). Verify that the same TestScenario produces valid results through the new harness. Add cross-harness comparison to the terminal report: for the same scenario, show side-by-side results across harnesses to reveal which harnesses benefit most from Jumbo. This validates that the HarnessAdapter abstraction works and that results are comparable across harnesses." \ - --criteria \ - "Second HarnessAdapter implemented (Codex CLI or Gemini CLI)" \ - "Adapter captures transcript, files modified, and token usage" \ - "Same TestScenario runnable through both harnesses" \ - "Cross-harness comparison added to terminal report" \ - "HarnessAdapter abstraction validated — no orchestrator changes needed" \ - --scope-in "Second harness adapter" "Cross-harness comparison" "Adapter validation" \ - --scope-out "Third harness" "JSON export" "CLI commands" \ - --prerequisite-goals "" -``` - -### Goal 8 — Full Reporting - -```bash -jumbo goal add \ - --title "[EVAL-8/9] Full reporting" \ - --objective "Build the complete reporting suite. Add the third harness adapter. Produce structured reports: session-by-session divergence curve, per-dimension lift percentages, divergence onset detection (the session number where amnesia impact becomes statistically significant), disruption impact analysis (which corrections caused the largest delta), and cross-harness aggregation. Output in both terminal-formatted human-readable and JSON machine-readable formats. The JSON format is designed for external consumption in documentation and marketing." \ - --criteria \ - "Third HarnessAdapter implemented" \ - "Session-by-session divergence curve in terminal output" \ - "Per-dimension lift percentages" \ - "Divergence onset detection: identifies session where amnesia impact becomes significant" \ - "Disruption impact analysis: ranks corrections by delta size" \ - "Cross-harness aggregation: lift by harness type" \ - "JSON export with complete structured data" \ - "Terminal report polished for human readability" \ - --scope-in "Third harness" "Divergence analysis" "Lift quantification" "JSON export" "Terminal polish" \ - --scope-out "CLI commands" \ - --prerequisite-goals "" -``` - -### Goal 9 — CLI Command Surface - -```bash -jumbo goal add \ - --title "[EVAL-9/9] CLI command surface" \ - --objective "Wire up the CLI command surface. 'eval scenario create' — define a new test scenario interactively or from a JSON template. 'eval scenario list' — show available scenarios. 'eval run' — execute a scenario against specified harnesses with --harness and --sessions flags, producing both Jumbo and non-Jumbo runs. 'eval score' — run the scoring engine against completed runs. 'eval report' — generate lift reports with optional --harness and --dimension filters. 'eval status' — show in-progress and completed runs with summary scores. Each command follows existing Jumbo CLI patterns for output formatting, error handling, and help text." \ - --criteria \ - "eval scenario create command with interactive and --from-template modes" \ - "eval scenario list command showing available scenarios" \ - "eval run command with --harness and --sessions filters" \ - "eval score command to trigger scoring on completed runs" \ - "eval report command with --harness and --dimension filters" \ - "eval status command showing run progress and summary scores" \ - "All commands include --help with usage examples" \ - --scope-in "CLI commands" "Command registration" "Output formatting" \ - --scope-out "Core simulation logic" "Scoring logic" "Report aggregation" \ - --prerequisite-goals "" -``` - -## Notes - -- Goals use `--prerequisite-goals` to enforce ordering. Replace `` placeholders with actual goal IDs as each goal is created. -- Each goal is a tracer bullet: it delivers a vertical slice through the full stack and leaves the system in a buildable, runnable state. -- Harness adapters invoke agent CLIs via subprocess inside containers — no direct LLM API calls from the orchestrator. -- The judge LLM for scoring (Goal 6) is the one place where an LLM API call may be needed, unless the judge can also be invoked via a harness CLI. This is an implementation detail to resolve in Goal 6. -- Adding new harness adapters should be trivial — implement the HarnessAdapter interface and register it. -- Test scenarios should be designed so that both Jumbo and non-Jumbo runs receive identical initial prompts with sufficient information to succeed. The advantage Jumbo provides is retention and curation across sessions, not access to hidden information. diff --git a/evals/README.md b/evals/README.md deleted file mode 100644 index d78a1f61..00000000 --- a/evals/README.md +++ /dev/null @@ -1,233 +0,0 @@ -# Jumbo Evals - -Longitudinal evaluation system that measures whether Jumbo's workflow (knowledge capture, context delivery, review-driven correction) measurably improves coding agent performance over multi-session projects. - -## Methodology - -The treatment under test is **the whole Jumbo system** — `jumbo init`, pre-seeded memory, the lifecycle protocol handed to the agent, agent-driven capture of decisions/components/relations, and the post-session audit — **not** memory delivery in isolation. The execution model is **framework-as-developer orchestration with agent-driven lifecycle**: the framework plays the developer role (it picks the active goal-id per session and hands the agent a `(scenario, goal-id, lifecycle-protocol)` bundle) and the agent itself runs every `jumbo` lifecycle call. The framework never issues lifecycle calls on the agent's behalf; it verifies post-session by snapshotting Jumbo state. - -The comparison measures the same agent running the same scenario twice — once armed with Jumbo (real binary on PATH, project initialised, developer-grade goal handoff) and once with Jumbo unreachable (a PATH-shadowing fail-loud shim ensures the baseline arm cannot invoke `jumbo`). Any published lift number therefore attributes to **the entire Jumbo system armed against the same agent**, not to one isolated feature. - -## How It Works - -The system runs **A/B comparisons**: for the same test scenario, it executes N sessions through an agent harness twice — once with Jumbo enabled, once without. The provider-neutral **scenario prompt** for each session is byte-identical across variants before variant-specific wrapping (this is an invariant of the harness, including any scheduled disruptions). The intentional A/B difference is that the Jumbo variant's effective prompt wraps the scenario with the framework-assigned active goal-id and an explicit lifecycle protocol the agent must follow (run `jumbo session start`, `jumbo goal start`, capture decisions/components/relations as it works, `jumbo goal submit`, `jumbo session end`). The baseline variant receives only the raw scenario prompt and cannot reach the `jumbo` binary. The framework never issues lifecycle calls on the agent's behalf — it verifies execution post-session by reading Jumbo's own state (`jumbo goal show` / `jumbo sessions list` / `jumbo decisions list`). - -After execution, 7 scoring dimensions measure the delta: - -| Dimension | Type | What it measures | -|-----------|------|-----------------| -| File Accuracy | Deterministic | Were the right files modified? (F1 score) | -| Knowledge Retention | Deterministic | Do patterns from early sessions persist? | -| Disruption Recovery | Deterministic | Do mid-project corrections stick? | -| Token Efficiency | Deterministic | Tokens consumed per quality point | -| Consistency | LLM-as-Judge | Naming, architectural, style coherence across sessions | -| Error Correction | LLM-as-Judge | Mistakes caught and not reintroduced | -| Architectural Quality | LLM-as-Judge | Separation of concerns, dependency direction | - -## Prerequisites - -- **Node.js 22+** -- **Agent CLI tools** installed and authenticated on your machine (claude, codex, gemini — whichever harnesses you plan to test) -- **Jumbo CLI** installed globally -- **API keys** for the harnesses you want to test (set as environment variables, never in code) - -> **Note:** Evals run locally, not in Docker containers. Agent CLIs require host-level authentication (OAuth flows, config files in `~/`) that cannot be provisioned inside fresh containers. Each eval variant gets its own temp working directory for isolation. - -## Prompt Auditability - -Every `SessionRecord` stores the provider-neutral `scenarioPrompt`, the exact `effectivePrompt` passed to the harness adapter, and any `deliveredContext`. For baseline records, `effectivePrompt` is the scenario prompt unchanged. For Jumbo records with an active goal-id, `effectivePrompt` is the scenario prompt wrapped with the active goal-id and the lifecycle protocol the agent must execute. This makes the measured lift attributable to the whole Jumbo system (init + pre-seeded memory + developer-grade goal handoff + agent-driven lifecycle + audit) rather than to hidden task information, prompt drift across harness providers, or a single isolated feature. - -Jumbo records also store a `jumboLifecycleAudit` derived from post-session Jumbo state (`jumbo goal show`, `jumbo sessions list`, `jumbo decisions list`): four booleans (`sessionStartExecuted`, `goalStartExecuted`, `goalSubmitExecuted`, `sessionEndExecuted`) and the raw CLI evidence used to compute them. Lifecycle non-adherence is recorded as audit signal, not a framework error. - -## Setup - -```bash -npm install -npm run build -npm link # optional — exposes the `eval` binary on your PATH -``` - -Examples below use `node dist/cli/index.js`. If you ran `npm link`, you can substitute `eval` (e.g. `eval scenario list`). - -## Test Scenarios - -Three curated scenarios ship in `scenarios/`. Each one is designed to expose specific amnesia patterns across multi-session projects: - -| Scenario | Sessions | Disruptions | What it tests | -|----------|----------|-------------|---------------| -| **Event-Sourced Inventory** | 7 | 2 | Domain modeling with compounding complexity. Disruptions add event metadata and optimistic concurrency — patterns that must propagate to all existing code. Tests whether the agent remembers its own event types and projection logic. | -| **Plugin Architecture** | 6 | 2 | Interface contract adherence over time. Disruptions change execution order (priority) and add dependency resolution — both require updating existing plugins. Tests whether the agent maintains the plugin contract across sessions. | -| **CLI Task Tracker** | 6 | 2 | State machine consistency. Disruptions add a new state and an audit trail — both touch the core transition logic. Tests whether the agent remembers valid transitions and the append-only history constraint. | - -Each scenario includes: -- **Retention patterns** — specific terms the agent should reference if it remembers prior sessions -- **Disruptions** — mid-project corrections injected at specific sessions to test recovery -- **Expected files** — the files that should exist by the end - -## Quick Start - -### 1. Register a scenario - -```bash -node dist/cli/index.js scenario create --from-template scenarios/event-sourced-inventory.json -``` - -This prints the new scenario's UUID — that's the `` used by every other command. You can also list registered scenarios at any time: - -```bash -node dist/cli/index.js scenario list -``` - -### 2. Run the evaluation - -```bash -# Single harness (default: claude-code) -node dist/cli/index.js run --scenario - -# Multiple harnesses -node dist/cli/index.js run --scenario --harness claude-code codex-cli gemini-cli - -# Override session count -node dist/cli/index.js run --scenario --sessions 10 -``` - -`eval run` prints a `Run ID` immediately after creating the run record. Use it from another terminal to watch live, read-only progress without affecting the running eval: - -```bash -node dist/cli/index.js status --watch -``` - -The watcher reads `.eval-store/runs//state.json` once per second in a TTY and renders per-harness, per-variant, per-session status, phase, and elapsed time. When output is piped, it performs a single read for scripting. - -**Authentication.** Each adapter shells out to the corresponding agent CLI and inherits whatever auth that CLI is already using on your machine — you do **not** need to set API keys in your shell if the CLIs are already logged in. Examples: - -- **Claude Code** — a Claude Max/Pro subscription authenticated via `claude /login` (OAuth) is sufficient. No `ANTHROPIC_API_KEY` required. -- **Codex CLI** — uses the auth configured by `codex login` (or `OPENAI_API_KEY` if you prefer key-based auth). -- **Gemini CLI** — uses the auth configured by `gemini auth` (or `GOOGLE_API_KEY` if you prefer key-based auth). - -If a harness CLI isn't logged in, run its native login command before invoking `eval run`. This is why evals run on the host rather than in containers (see the note under Prerequisites). - -### 3. Score completed runs - -```bash -node dist/cli/index.js score --scenario -``` - -Scoring writes deterministic dimension scores and comparison metadata back to the result store. Reports read those scored comparisons. - -### 4. View results - -```bash -# Check run status -node dist/cli/index.js status - -# Generate terminal report -node dist/cli/index.js report --scenario - -# Filter by harness or dimension -node dist/cli/index.js report --scenario --harness claude-code --dimension file-accuracy knowledge-retention - -# Export as JSON (for docs/marketing) -node dist/cli/index.js report --scenario --json -``` - -## CLI Commands - -| Command | Description | -|---------|-------------| -| `eval scenario create --from-template ` | Create a scenario from JSON template | -| `eval scenario list` | List available scenarios | -| `eval run --scenario [--harness ] [--sessions ]` | Run A/B comparison | -| `eval status --watch ` | Watch a live run heartbeat | -| `eval score --scenario ` | Score completed runs | -| `eval report --scenario ` | Generate lift report | -| `eval status` | Show run progress and results | - -All commands support `--help` for full usage details. - -## Supported Harnesses - -| Harness | CLI | Adapter | -|---------|-----|---------| -| Claude Code | `claude -p --output-format json` | `ClaudeCodeAdapter` | -| Codex CLI | `codex --ask-for-approval never --sandbox workspace-write exec --json --skip-git-repo-check` | `CodexCliAdapter` | -| Gemini CLI | `gemini --json` | `GeminiCliAdapter` | - -## Report Output - -The reporting suite produces: - -- **Divergence Curve** — session-by-session delta between Jumbo and baseline -- **Lift Percentages** — per-dimension absolute and percentage improvement -- **Divergence Onset** — the session number where amnesia impact becomes significant -- **Disruption Impact** — which mid-project corrections caused the largest delta (ranked) -- **Cross-Harness Aggregation** — which harnesses benefit most from Jumbo -- **Audit Trail** — per-harness explanation of every measured delta: the effective context delivered each session, the final workspace snapshot evidence, and per-dimension scoring evidence - -Output formats: terminal (human-readable) and JSON (machine-readable, versioned as `jumbo-eval-report` v1). - -Reports are generated from scored comparison metadata stored on completed `TestResult`s. If a scenario has no scored comparisons yet, `eval report --scenario ` prints an empty-state message instead of failing. - -## Deterministic E2E Validation - -`tests/unit/end-to-end-validation.test.ts` is the harness's own self-check. It runs a fake deterministic harness whose output depends solely on whether a known prior-session fact appears in the delivered context. The test suite proves that: - -- Jumbo lift is positive **only** when injected Jumbo context contains the needed prior-session fact. -- The same fake harness produces **zero** lift when Jumbo memory is empty (control). -- Each variant gets its own real temp working directory; cross-variant filesystem state cannot leak. -- A failed Jumbo session-end (`JumboSessionEndError`) or a failed harness exec (`HarnessExecutionError`) on either variant invalidates the entire comparison — partial results are never persisted or scored. -- Reports include the audit trail above, so a measured delta can be explained from inputs (effective context) and outputs (workspace snapshot, scoring evidence) alone. - -This is what makes a measured Jumbo lift attributable to the whole Jumbo system armed against a parity-matched agent — not to prompt drift, filesystem leakage, scorer artifacts, partial-run persistence, or any single feature in isolation. - -## Testing - -```bash -# Unit tests — fully deterministic, no live agent CLI or API calls -npm test - -# Integration tests — live smoke tests that exercise real agent CLIs -npm run test:integration -``` - -Default unit tests are deterministic by design. Any test that invokes a live agent CLI or external API belongs in `tests/integration/` and is excluded from `npm test`. - -## Creating Custom Scenarios - -You can also create your own scenario templates. A scenario JSON file has this structure: - -```json -{ - "name": "Scenario Name", - "initialPrompt": "The first session prompt — sets up the project.", - "continuationPrompt": "Prompt for sessions 2+. Should ask the agent to review and continue.", - "sessionCount": 5, - "expectedFiles": ["files/that/should/exist.ts"], - "retentionPatterns": ["terms", "the agent", "should remember"], - "disruptions": [ - { - "type": "correction", - "sessionNumber": 3, - "content": "A mid-project correction the agent must integrate.", - "recoveryPatterns": ["terms", "proving", "the correction stuck"] - } - ] -} -``` - -Disruption types: `correction` (fix something), `scope-change` (pivot direction), `new-constraint` (add a requirement). - -## Architecture - -``` -src/ - cli/ CLI command surface (Commander) - domain/ Pure types: TestScenario, SessionRecord, TestResult, ComparisonResult - harness/ HarnessAdapter interface + 3 implementations - infrastructure/ LocalExecutor (per-variant temp working directories) - scoring/ 4 deterministic scorers + LLM-as-judge (3 rubrics) - output/ Terminal display, cross-harness comparison, report generator, JSON export - storage/ ResultStore interface + JSON file implementation - ab-runner.ts A/B comparison orchestrator - run-session.ts Single session execution -``` diff --git a/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt b/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt deleted file mode 100644 index 4fd043bb..00000000 --- a/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt +++ /dev/null @@ -1,274 +0,0 @@ -════════════════════════════════════════════════════════════ - A/B Comparison: 7ac1ef79-0105-411e-a604-62c85ab3c8a4 - Harness: codex-cli -════════════════════════════════════════════════════════════ - - JUMBO RUN (JUMBO_ENABLED=true) -──────────────────────────────────────────────────────────── - Session 1: 3 files modified - Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - Session 2: 2 files modified - Files: src/events/types.ts, src/projections/inventory.ts - Session 3: 4 files modified - Files: src/events/factory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - Session 4: 1 files modified - Files: src/queries/inventoryAvailability.ts - Session 5: 3 files modified - Files: src/commands/inventory.ts, src/projections/inventory.ts, src/queries/inventoryAvailability.ts - Session 6: 4 files modified - Files: src/commands/inventory.ts, src/events/factory.ts, src/events/types.ts, src/projections/inventory.ts - Session 7: 1 files modified - Files: src/queries/inventoryAvailability.ts - - BASELINE RUN (JUMBO_ENABLED=false) -──────────────────────────────────────────────────────────── - Session 1: 3 files modified - Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - Session 2: 2 files modified - Files: src/events/types.ts, src/projections/inventory.ts - Session 3: 3 files modified - Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - Session 4: 2 files modified - Files: src/events/types.ts, src/projections/inventory.ts - Session 5: 4 files modified - Files: src/commands/inventory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - Session 6: 3 files modified - Files: src/commands/inventory.ts, src/events/types.ts, src/projections/inventory.ts - Session 7: 3 files modified - Files: src/commands/inventory.ts, src/events/types.ts, src/projections/inventory.ts - - SCORES -════════════════════════════════════════════════════════════ - Dimension Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - file-accuracy 0.55/1.00 0.67/1.00 -0.12 - knowledge-retention 0.57/1.00 0.57/1.00 0.00 - structural-retention 1.00/1.00 1.00/1.00 0.00 - disruption-recovery 1.00/1.00 1.00/1.00 0.00 - jumbo-memory-capture 0.25/1.00 0.00/0.00 +0.25 - jumbo-event-capture 1.00/1.00 0.00/0.00 +1.00 - protocol-adherence 0.33/1.00 0.00/0.00 +0.33 - token-efficiency 0.00 (ref) — - └─ N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). - - JUMBO MEMORY CAPTURE EVIDENCE -──────────────────────────────────────────────────────────── - Jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} - Baseline: Not applicable: baseline runs do not use Jumbo project memory. - - PROTOCOL ADHERENCE EVIDENCE -──────────────────────────────────────────────────────────── - Jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) - Baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. - - TIMELINE (per-session scores) -════════════════════════════════════════════════════════════ - file-accuracy - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 0.75/1.00 0.75/1.00 0.00 - 2 0.57/1.00 0.57/1.00 0.00 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 0.67/1.00 0.75/1.00 -0.08 - 4 0.00/1.00 0.57/1.00 -0.57 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 0.25/1.00 0.67/1.00 -0.42 - 6 0.44/1.00 0.50/1.00 -0.06 - 7 0.00/1.00 0.50/1.00 -0.50 - - knowledge-retention - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 0.57/1.00 0.57/1.00 0.00 - 2 0.57/1.00 0.57/1.00 0.00 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 0.57/1.00 0.57/1.00 0.00 - 4 0.57/1.00 0.57/1.00 0.00 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 0.57/1.00 0.57/1.00 0.00 - 6 0.57/1.00 0.57/1.00 0.00 - 7 0.57/1.00 0.57/1.00 0.00 - - structural-retention - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 1.00/1.00 1.00/1.00 0.00 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 1.00/1.00 1.00/1.00 0.00 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 1.00/1.00 1.00/1.00 0.00 - 7 1.00/1.00 1.00/1.00 0.00 - - disruption-recovery - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 1.00/1.00 1.00/1.00 0.00 - 2 1.00/1.00 1.00/1.00 0.00 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 1.00/1.00 1.00/1.00 0.00 - 4 1.00/1.00 1.00/1.00 0.00 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 1.00/1.00 1.00/1.00 0.00 - 6 1.00/1.00 1.00/1.00 0.00 - 7 1.00/1.00 1.00/1.00 0.00 - - jumbo-memory-capture - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 0.22/1.00 0.00/0.00 +0.22 - 2 0.22/1.00 0.00/0.00 +0.22 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 0.17/1.00 0.00/0.00 +0.17 - 4 0.15/1.00 0.00/0.00 +0.15 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 0.25/1.00 0.00/0.00 +0.25 - 6 0.25/1.00 0.00/0.00 +0.25 - 7 0.25/1.00 0.00/0.00 +0.25 - - jumbo-event-capture - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 1.00/1.00 0.00/0.00 +1.00 - 2 1.00/1.00 0.00/0.00 +1.00 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 1.00/1.00 0.00/0.00 +1.00 - 4 1.00/1.00 0.00/0.00 +1.00 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 1.00/1.00 0.00/0.00 +1.00 - 6 1.00/1.00 0.00/0.00 +1.00 - 7 1.00/1.00 0.00/0.00 +1.00 - - protocol-adherence - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 0.33/1.00 0.00/0.00 +0.33 - 2 0.33/1.00 0.00/0.00 +0.33 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 0.33/1.00 0.00/0.00 +0.33 - 4 0.33/1.00 0.00/0.00 +0.33 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 0.33/1.00 0.00/0.00 +0.33 - 6 0.33/1.00 0.00/0.00 +0.33 - 7 0.33/1.00 0.00/0.00 +0.33 - - token-usage - Session Jumbo Baseline Delta -──────────────────────────────────────────────────────────── - 1 744030 250088 +493942 - 2 1017547 232222 +785325 - >>> [CORRECTION] The event types need a 'metadata' field containing... - 3 2049125 453763 +1595362 - 4 1128233 464308 +663925 - >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... - 5 2703659 752580 +1951079 - 6 1868112 536532 +1331580 - 7 1019923 443985 +575938 - -════════════════════════════════════════════════════════════ -════════════════════════════════════════════════════════════════════════ - JUMBO EVALUATION REPORT - Scenario: 7ac1ef79-0105-411e-a604-62c85ab3c8a4 - Harnesses: codex-cli - Treatment: whole Jumbo system vs Jumbo-unreachable baseline - Methodology: framework-as-developer orchestration, agent-driven lifecycle - Lift attribution: init + pre-seeded memory + lifecycle protocol + agent capture + audit -════════════════════════════════════════════════════════════════════════ - - LIFT BY DIMENSION -──────────────────────────────────────────────────────────────────────── - Dimension Jumbo Base Lift Lift % -──────────────────────────────────────────────────────────────────────── - file-accuracy 0.55 0.67 -0.120 -17.9% - knowledge-retention 0.57 0.57 0.000 0.0% - structural-retention 1.00 1.00 0.000 0.0% - disruption-recovery 1.00 1.00 0.000 0.0% - jumbo-memory-capture 0.25 0.00 +0.250 n/a - jumbo-event-capture 1.00 0.00 +1.000 n/a - protocol-adherence 0.33 0.00 +0.330 n/a - token-efficiency 0.00 0.00 0.000 n/a - - DIVERGENCE ONSET (session where amnesia impact exceeds threshold) -──────────────────────────────────────────────────────────────────────── - file-accuracy session 4 (delta: -0.570, threshold: 0.1) - knowledge-retention no significant divergence detected - structural-retention no significant divergence detected - disruption-recovery no significant divergence detected - jumbo-memory-capture session 1 (delta: +0.220, threshold: 0.1) - jumbo-event-capture session 1 (delta: +1.000, threshold: 0.1) - protocol-adherence session 1 (delta: +0.330, threshold: 0.1) - token-usage session 1 (delta: +493942.000, threshold: 0.1) - - DIVERGENCE CURVE (session-by-session delta) -──────────────────────────────────────────────────────────────────────── - file-accuracy S1:0.00 S2:0.00 S3:-0.08 S4:-0.57 S5:-0.42 S6:-0.06 S7:-0.50 - knowledge-retention S1:0.00 S2:0.00 S3:0.00 S4:0.00 S5:0.00 S6:0.00 S7:0.00 - structural-retention S1:0.00 S3:0.00 S5:0.00 S7:0.00 - disruption-recovery S1:0.00 S2:0.00 S3:0.00 S4:0.00 S5:0.00 S6:0.00 S7:0.00 - jumbo-memory-capture S1:+0.22 S2:+0.22 S3:+0.17 S4:+0.15 S5:+0.25 S6:+0.25 S7:+0.25 - jumbo-event-capture S1:+1.00 S2:+1.00 S3:+1.00 S4:+1.00 S5:+1.00 S6:+1.00 S7:+1.00 - protocol-adherence S1:+0.33 S2:+0.33 S3:+0.33 S4:+0.33 S5:+0.33 S6:+0.33 S7:+0.33 - token-usage S1:+493942.00 S2:+785325.00 S3:+1595362.00 S4:+663925.00 S5:+1951079.00 S6:+1331580.00 S7:+575938.00 - - JUMBO MEMORY CAPTURE EVIDENCE -──────────────────────────────────────────────────────────────────────── - codex-cli score: 0.25 - Jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} - Baseline: Not applicable: baseline runs do not use Jumbo project memory. - - PROTOCOL ADHERENCE EVIDENCE (per-step lifecycle execution) -──────────────────────────────────────────────────────────────────────── - codex-cli score: 0.33 - Jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) - Baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. - S1: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S2: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S3: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S4: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S5: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S6: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - S7: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) - - DISRUPTION IMPACT (ranked by magnitude) -──────────────────────────────────────────────────────────────────────── - [new-constraint] S5 token-usage before:+663925.000 after:+1951079.000 magnitude:1287154.000 - [correction] S3 token-usage before:+785325.000 after:+1595362.000 magnitude:810037.000 - [new-constraint] S5 file-accuracy before:-0.570 after:-0.420 magnitude:0.150 - [new-constraint] S5 jumbo-memory-capture before:+0.150 after:+0.250 magnitude:0.100 - [correction] S3 file-accuracy before:+0.000 after:-0.080 magnitude:0.080 - [correction] S3 jumbo-memory-capture before:+0.220 after:+0.170 magnitude:0.050 - [correction] S3 knowledge-retention before:+0.000 after:+0.000 magnitude:0.000 - [correction] S3 structural-retention before:+0.000 after:+0.000 magnitude:0.000 - [correction] S3 disruption-recovery before:+0.000 after:+0.000 magnitude:0.000 - [correction] S3 jumbo-event-capture before:+1.000 after:+1.000 magnitude:0.000 - - AUDIT TRAIL (effective context, final snapshot, scoring evidence) -──────────────────────────────────────────────────────────────────────── - codex-cli - Jumbo final session 7 delivered context: none - Baseline final session 7 effective prompt: Continue building the inventory system. Review the existing code and event ty... - Jumbo final snapshot files: .agents/skills/codify-jumbo-goal/SKILL.md, .agents/skills/decompose-architecture-aggregate/SKILL.md, .agents/skills/define-jumbo-goals/SKILL.md, .agents/skills/formalize-objectives/SKILL.md, .agents/skills/jumbo-add-component/SKILL.md, .agents/skills/jumbo-add-decision/SKILL.md, .agents/skills/jumbo-add-dependency/SKILL.md, .agents/skills/jumbo-add-guideline/SKILL.md, .agents/skills/jumbo-add-invariant/SKILL.md, .agents/skills/jumbo-bootstrap-session/SKILL.md, .agents/skills/jumbo-command-discovery/SKILL.md, .agents/skills/jumbo-context-maintenance/SKILL.md, .agents/skills/jumbo-correction-capture/SKILL.md, .agents/skills/jumbo-design-goal/SKILL.md, .agents/skills/refine-jumbo-goals/SKILL.md, .agents/skills/reject-jumbo-goal/SKILL.md, .agents/skills/review-jumbo-goal/SKILL.md, .agents/skills/start-jumbo-goal/SKILL.md, .claude/settings.json, .claude/skills/codify-jumbo-goal/SKILL.md, .claude/skills/decompose-architecture-aggregate/SKILL.md, .claude/skills/define-jumbo-goals/SKILL.md, .claude/skills/formalize-objectives/SKILL.md, .claude/skills/jumbo-add-component/SKILL.md, .claude/skills/jumbo-add-decision/SKILL.md, .claude/skills/jumbo-add-dependency/SKILL.md, .claude/skills/jumbo-add-guideline/SKILL.md, .claude/skills/jumbo-add-invariant/SKILL.md, .claude/skills/jumbo-bootstrap-session/SKILL.md, .claude/skills/jumbo-command-discovery/SKILL.md, .claude/skills/jumbo-context-maintenance/SKILL.md, .claude/skills/jumbo-correction-capture/SKILL.md, .claude/skills/jumbo-design-goal/SKILL.md, .claude/skills/refine-jumbo-goals/SKILL.md, .claude/skills/reject-jumbo-goal/SKILL.md, .claude/skills/review-jumbo-goal/SKILL.md, .claude/skills/start-jumbo-goal/SKILL.md, .codex/config.toml, .codex/hooks.json, .cursor/hooks.json, .cursor/rules/jumbo.mdc, .gemini/settings.json, .gemini/skills/codify-jumbo-goal/SKILL.md, .gemini/skills/decompose-architecture-aggregate/SKILL.md, .gemini/skills/define-jumbo-goals/SKILL.md, .gemini/skills/formalize-objectives/SKILL.md, .gemini/skills/jumbo-add-component/SKILL.md, .gemini/skills/jumbo-add-decision/SKILL.md, .gemini/skills/jumbo-add-dependency/SKILL.md, .gemini/skills/jumbo-add-guideline/SKILL.md, .gemini/skills/jumbo-add-invariant/SKILL.md, .gemini/skills/jumbo-bootstrap-session/SKILL.md, .gemini/skills/jumbo-command-discovery/SKILL.md, .gemini/skills/jumbo-context-maintenance/SKILL.md, .gemini/skills/jumbo-correction-capture/SKILL.md, .gemini/skills/jumbo-design-goal/SKILL.md, .gemini/skills/refine-jumbo-goals/SKILL.md, .gemini/skills/reject-jumbo-goal/SKILL.md, .gemini/skills/review-jumbo-goal/SKILL.md, .gemini/skills/start-jumbo-goal/SKILL.md, .github/copilot-instructions.md, .github/hooks/hooks.json, .gitignore, .vibe/skills/codify-jumbo-goal/SKILL.md, .vibe/skills/decompose-architecture-aggregate/SKILL.md, .vibe/skills/define-jumbo-goals/SKILL.md, .vibe/skills/formalize-objectives/SKILL.md, .vibe/skills/jumbo-add-component/SKILL.md, .vibe/skills/jumbo-add-decision/SKILL.md, .vibe/skills/jumbo-add-dependency/SKILL.md, .vibe/skills/jumbo-add-guideline/SKILL.md, .vibe/skills/jumbo-add-invariant/SKILL.md, .vibe/skills/jumbo-bootstrap-session/SKILL.md, .vibe/skills/jumbo-command-discovery/SKILL.md, .vibe/skills/jumbo-context-maintenance/SKILL.md, .vibe/skills/jumbo-correction-capture/SKILL.md, .vibe/skills/jumbo-design-goal/SKILL.md, .vibe/skills/refine-jumbo-goals/SKILL.md, .vibe/skills/reject-jumbo-goal/SKILL.md, .vibe/skills/review-jumbo-goal/SKILL.md, .vibe/skills/start-jumbo-goal/SKILL.md, AGENTS.md, CLAUDE.md, GEMINI.md, JUMBO.md, src/commands/inventory.ts, src/events/factory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts, src/queries/inventoryAvailability.ts - Baseline final snapshot files: .codex/config.toml, .eval-bin/jumbo.bat, .eval-bin/jumbo.cmd, src/commands/inventory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts - file-accuracy jumbo=0.55 baseline=0.67 delta=-0.120 - jumbo: 3/5 expected files modified; missed: src/commands/handlers.ts, src/domain/product.ts; unexpected: src/events/factory.ts, src/queries/inventoryAvailability.ts, src/commands/inventory.ts; precision=0.50 recall=0.60 f1=0.55 - baseline: 3/5 expected files modified; missed: src/commands/handlers.ts, src/domain/product.ts; unexpected: src/commands/inventory.ts; precision=0.75 recall=0.60 f1=0.67 - knowledge-retention jumbo=0.57 baseline=0.57 delta=0.000 - jumbo: 4/7 patterns retained in session 7; lost: discriminated union, append-only, event store - baseline: 4/7 patterns retained in session 7; lost: discriminated union, append-only, event store - structural-retention jumbo=1.00 baseline=1.00 delta=0.000 - jumbo: 11/11 structural assertions passed - baseline: 11/11 structural assertions passed - disruption-recovery jumbo=1.00 baseline=1.00 delta=0.000 - jumbo: 20/20 recovery checks passed - baseline: 20/20 recovery checks passed - jumbo-memory-capture jumbo=0.25 baseline=0.00 delta=+0.250 - jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} - baseline: Not applicable: baseline runs do not use Jumbo project memory. - jumbo-event-capture jumbo=1.00 baseline=0.00 delta=+1.000 - jumbo: 3/3 expected entity kinds evidenced in .jumbo/events; missing: none - baseline: Not applicable: baseline runs have no .jumbo/events log. - protocol-adherence jumbo=0.33 baseline=0.00 delta=+0.330 - jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) - baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. - token-efficiency jumbo=0.00 baseline=0.00 delta=0.000 - jumbo: N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). - baseline: N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). \ No newline at end of file diff --git a/evals/docker/Dockerfile b/evals/docker/Dockerfile deleted file mode 100644 index e21cee89..00000000 --- a/evals/docker/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM node:22-bookworm - -# Install common dev tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - curl \ - jq \ - && rm -rf /var/lib/apt/lists/* - -# Install Claude Code CLI -RUN npm install -g @anthropic-ai/claude-code - -# Install Jumbo CLI (from local build — mount or copy during image build) -# For now, install placeholder — real path configured at build time -ARG JUMBO_CLI_PATH="" -RUN if [ -n "$JUMBO_CLI_PATH" ]; then npm install -g "$JUMBO_CLI_PATH"; fi - -# Workspace directory -RUN mkdir -p /workspace -WORKDIR /workspace - -# API keys come from environment variables at runtime — never baked into the image -# See invariant: inv_a8c50a2b - No API keys in source code, config files, or test fixtures - -# Keep container running for exec commands -CMD ["tail", "-f", "/dev/null"] diff --git a/evals/docs/parallel-replication-design.md b/evals/docs/parallel-replication-design.md deleted file mode 100644 index 62abb5cf..00000000 --- a/evals/docs/parallel-replication-design.md +++ /dev/null @@ -1,152 +0,0 @@ -# Parallel & Interleaved Replication Execution — Design - -_Design deliverable for Jumbo goal `fa0394d4` (Outcome 5 challenge). This is a -design, not an implementation: it defines the execution model, resolves the -resource-contention question, gives an interleaving go/no-go, and decomposes the -follow-on implementation goals._ - -## 1. Context & problem - -`eval run --replicate K` (Outcome 5, goal 2/2) runs K A/B comparisons -**sequentially**. Each comparison is one `runABComparison`, which runs the -**jumbo arm fully, then the baseline arm fully** (`ab-runner.ts` → -`runMultiSession` for jumbo, then baseline), and each session spawns a real -coding-agent CLI subprocess (claude / codex / gemini) that performs LLM -inference and file I/O. - -So one scenario/harness at K≥5 with 6–7 sessions ≈ **60–70+ agent invocations**, -run end to end → wall-time on the order of hours. GOAL.md Challenges require the -framework to support **parallel execution of replications** and **interleaved -session execution** to make K≥5 tractable and to reduce temporal confounding. - -## 2. Key insight: parallelism does not bias the measurement - -Every scored dimension is **artefact- or count-based**, never wall-clock based: - -- file-accuracy, knowledge-retention, **structural-retention** — checks on - workspace files; -- jumbo-memory-capture, **jumbo-event-capture** — entity / `.jumbo/events` - counts; -- token-efficiency — **token counts** (not latency); -- protocol-adherence — lifecycle booleans from Jumbo state. - -`phaseTimings` are recorded but **not scored**. Therefore CPU/IO contention from -running replications concurrently changes **wall-time and failure rate, not the -measured quality or token metrics**. Parallelism is safe for measurement -*validity*; its risks are purely **operational** (rate limits, OOM, process -failures). - -Corollary: `aggregateReplications` is **order-independent** (mean, sample SD, -and counts do not depend on completion order), so the order in which concurrent -replications finish cannot change the `ReplicationReport`. - -## 3. Execution model: bounded worker pool - -Add `--concurrency ` (default **1**) to `eval run`. The K replications per -harness are dispatched through a bounded pool of size N. - -### 3.1 Worker-pool interface (the primitive the implementation builds) - -```ts -// Pure concurrency primitive — no eval knowledge. -// Runs `tasks` with at most `n` in flight; resolves with results in INPUT order. -// Fail-fast: on the first rejection, stop scheduling new tasks, let in-flight -// tasks settle, then reject with that first error. -export function runWithConcurrency( - tasks: ReadonlyArray<() => Promise>, - n: number, -): Promise; -``` - -- **Ordering:** results are returned positionally (result[i] ↔ tasks[i]), - independent of completion order. -- **Error propagation:** fail-fast in v1 (a failed replication aborts the batch - with a clear error) — simpler and safer than partial aggregation; partial-batch - tolerance can be a later enhancement. -- **Bound:** at most `n` tasks are ever in flight; `n=1` is exactly the current - sequential behaviour. -- **Testability:** unit-tested with fake async tasks asserting (a) max observed - in-flight ≤ n, (b) output order matches input order, (c) fail-fast on rejection. - -### 3.2 Wiring into the replicate loop - -`run.ts` builds K task closures per harness (`() => abRunner({...})`) and awaits -`runWithConcurrency(tasks, concurrency)`. Aggregation is unchanged because it is -order-independent. `--concurrency 1` reproduces today's output byte-for-byte. - -## 4. Resource-contention analysis (non-API harnesses) - -Running N replications concurrently means up to **N concurrent agent-CLI -subprocesses** (each spawning its own runtime + LLM calls), plus the jumbo CLI -subprocesses they invoke. - -| Contention vector | Effect | Mitigation | -|---|---|---| -| **Provider rate limits** (one API key/account, N concurrent agents) | 429 throttling → retries/failures | The binding constraint; cap N to the account's concurrency headroom | -| **Host CPU / memory** (N agent CLIs + node runtimes) | slowdown / OOM | Cap N to host capacity; fail-fast on OOM | -| **Disk I/O** (workspace snapshots; per-workdir `.jumbo` SQLite) | I/O load; **no cross-contention** (each replication has an isolated workdir) | Acceptable; isolated state | -| **Agent CLI auth/session limits** | some CLIs disallow concurrent sessions | Detected as agent failures; document per-harness | - -**Recommended default: N = 1.** Rationale: the binding constraint (provider rate -limits) is **user-specific and external**; a higher default risks mass 429 -failures that masquerade as eval failures and silently corrupt a batch. N is -**opt-in** for users who know their headroom. - -**Method to pick N for a host:** start at N=2 with a K=2 smoke; watch for non-zero -agent exit codes, 429s in transcripts, and memory pressure; increase N until -failures/pressure appear, then back off by one. Ship a short guidance table -(rate-limit tier → suggested N) alongside `--concurrency`. - -## 5. Interleaving (temporal confounding) — GO - -**Today:** within a comparison the jumbo arm runs to completion, then baseline. -Over a long batch, time-varying external factors (provider model/load drift) -could bias the always-jumbo-first ordering. - -Two axes: - -- **(a) Arm interleaving within a comparison** — alternate jumbo/baseline per - session (`jumbo s1, baseline s1, jumbo s2, …`). **Feasible:** the two arms have - independent workdirs; sessions *within* an arm remain sequential (each - continues from the prior), but the two arms are independent, so alternating by - session index is a contained `ab-runner` refactor (interleave the two - `runMultiSession` loops). **GO** — low risk, removes within-comparison temporal - skew. Behind a flag, default-on once validated; a test asserts the session - execution order alternates by arm. -- **(b) Replication interleaving** — with the worker pool (N>1), replications - already overlap in time, naturally de-correlating order. At N=1, alternate the - arm-order per replication. - -Arm interleaving is **independent of** the worker pool and can ship separately. - -## 6. Decomposition into follow-on implementation goals - -1. **`runWithConcurrency` worker-pool primitive** (pure, fully tested). _Foundation._ -2. **Wire `--concurrency N` into `eval run --replicate`** — dispatch replications - through the pool; default 1; aggregation unchanged. _Prereq: goal 1 + Outcome 5 - goal 2/2._ -3. **Arm interleaving in `ab-runner`** — per-session jumbo/baseline alternation, - flagged, with an ordering test. _Independent (parallel) of goals 1–2._ -4. _(optional)_ **`--concurrency` guidance + host smoke** to recommend N. _Prereq: goal 2._ - -Goals 1 → 2 are prerequisite-chained; goal 3 is independent and can run in -parallel; goal 4 is optional. - -## 7. Resolving GOAL.md open questions - -- _"Can the host sustain K parallel harness invocations without resource - contention affecting measurement?"_ — **The measurement is not affected** - (all metrics are artefact/count-based, not wall-clock); only wall-time and - failure rate are. Parallelism is therefore valid; bound N by provider rate - limits and host resources (default 1, opt-in higher). -- _"Interleaved session execution?"_ — **Feasible and recommended (GO)** via - per-session arm alternation, shipped as an independent goal. - -## 8. Decisions (registered in Jumbo) - -- **Default replication concurrency = 1; opt-in via `--concurrency N`.** Provider - rate limits are the binding, user-specific constraint; a higher default risks - mass throttling failures that corrupt a batch. -- **Arm interleaving = GO** (per-session alternation), as an independent - follow-on goal — it reduces temporal confounding at low risk and is orthogonal - to the worker pool. diff --git a/evals/jest.config.ts b/evals/jest.config.ts deleted file mode 100644 index 655d818d..00000000 --- a/evals/jest.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { Config } from 'jest'; - -const config: Config = { - testEnvironment: 'node', - roots: ['/tests'], - testMatch: ['**/*.test.ts'], - testPathIgnorePatterns: ['/node_modules/', '/dist/', '/tests/integration/'], - transform: { - '^.+\\.ts$': ['@swc/jest', { - module: { type: 'commonjs', ignoreDynamic: true }, - }], - }, - moduleNameMapper: { - '^(\\.{1,2}/.*)\\.js$': '$1', - }, -}; - -export default config; diff --git a/evals/package-lock.json b/evals/package-lock.json deleted file mode 100644 index a196dc93..00000000 --- a/evals/package-lock.json +++ /dev/null @@ -1,5361 +0,0 @@ -{ - "name": "jumbo-evals", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "jumbo-evals", - "version": "0.1.0", - "license": "ISC", - "dependencies": { - "commander": "^14.0.3", - "ink": "^7.0.1", - "react": "^19.2.5" - }, - "bin": { - "eval": "dist/cli/index.js" - }, - "devDependencies": { - "@jest/globals": "^30.3.0", - "@swc/core": "^1.15.18", - "@swc/jest": "^0.2.39", - "@types/jest": "^30.0.0", - "@types/node": "^25.5.0", - "@types/react": "^19.2.14", - "ink-testing-library": "^4.0.0", - "jest": "^30.3.0", - "ts-node": "^10.9.2", - "typescript": "^5.9.3" - } - }, - "node_modules/@alcalzone/ansi-tokenize": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@alcalzone/ansi-tokenize/-/ansi-tokenize-0.3.0.tgz", - "integrity": "sha512-p+CMKJ93HFmLkjXKlXiVGlMQEuRb6H0MokBSwUsX+S6BRX8eV5naFZpQJFfJHjRZY0Hmnqy1/r6UWl3x+19zYA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "is-fullwidth-code-point": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@alcalzone/ansi-tokenize/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@alcalzone/ansi-tokenize/node_modules/is-fullwidth-code-point": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", - "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.3.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", - "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", - "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.0" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", - "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", - "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@emnapi/core": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.1.tgz", - "integrity": "sha512-mukuNALVsoix/w1BJwFzwXBN/dHeejQtuVzcDsfOEsdpCumXb/E9j8w11h5S54tT1xhifGfbbSm/ICrObRb3KA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.0", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.1.tgz", - "integrity": "sha512-VYi5+ZVLhpgK4hQ0TAjiQiZ6ol0oe4mBx7mVv7IflsiEp0OWoVsp/+f9Vc1hOhE0TtkORVrI1GvzyreqpgWtkA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz", - "integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.3.0.tgz", - "integrity": "sha512-PAwCvFJ4696XP2qZj+LAn1BWjZaJ6RjG6c7/lkMaUJnkyMS34ucuIsfqYvfskVNvUI27R/u4P1HMYFnlVXG/Ww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "@types/node": "*", - "chalk": "^4.1.2", - "jest-message-util": "30.3.0", - "jest-util": "30.3.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/core": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.3.0.tgz", - "integrity": "sha512-U5mVPsBxLSO6xYbf+tgkymLx+iAhvZX43/xI1+ej2ZOPnPdkdO1CzDmFKh2mZBn2s4XZixszHeQnzp1gm/DIxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.3.0", - "@jest/pattern": "30.0.1", - "@jest/reporters": "30.3.0", - "@jest/test-result": "30.3.0", - "@jest/transform": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "ansi-escapes": "^4.3.2", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "exit-x": "^0.2.2", - "graceful-fs": "^4.2.11", - "jest-changed-files": "30.3.0", - "jest-config": "30.3.0", - "jest-haste-map": "30.3.0", - "jest-message-util": "30.3.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.3.0", - "jest-resolve-dependencies": "30.3.0", - "jest-runner": "30.3.0", - "jest-runtime": "30.3.0", - "jest-snapshot": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0", - "jest-watcher": "30.3.0", - "pretty-format": "30.3.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/create-cache-key-function": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/create-cache-key-function/-/create-cache-key-function-30.3.0.tgz", - "integrity": "sha512-hTupmOWylzeyqbMNeSNi7ZDprpjrcroAOOG+qCEW66st3+Z5RnYHVYkUt+zjIcLmrTUi2lPY79hJz8mB3L2oXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/diff-sequences": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.3.0.tgz", - "integrity": "sha512-cG51MVnLq1ecVUaQ3fr6YuuAOitHK1S4WUJHnsPFE/quQr33ADUx1FfrTCpMCRxvy0Yr9BThKpDjSlcTi91tMA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/environment": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.3.0.tgz", - "integrity": "sha512-SlLSF4Be735yQXyh2+mctBOzNDx5s5uLv88/j8Qn1wH679PDcwy67+YdADn8NJnGjzlXtN62asGH/T4vWOkfaw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "jest-mock": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/expect": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.3.0.tgz", - "integrity": "sha512-76Nlh4xJxk2D/9URCn3wFi98d2hb19uWE1idLsTt2ywhvdOldbw3S570hBgn25P4ICUZ/cBjybrBex2g17IDbg==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "30.3.0", - "jest-snapshot": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.3.0.tgz", - "integrity": "sha512-j0+W5iQQ8hBh7tHZkTQv3q2Fh/M7Je72cIsYqC4OaktgtO7v1So9UTjp6uPBHIaB6beoF/RRsCgMJKvti0wADA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.3.0.tgz", - "integrity": "sha512-WUQDs8SOP9URStX1DzhD425CqbN/HxUYCTwVrT8sTVBfMvFqYt/s61EK5T05qnHu0po6RitXIvP9otZxYDzTGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "@sinonjs/fake-timers": "^15.0.0", - "@types/node": "*", - "jest-message-util": "30.3.0", - "jest-mock": "30.3.0", - "jest-util": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.3.0.tgz", - "integrity": "sha512-+owLCBBdfpgL3HU+BD5etr1SvbXpSitJK0is1kiYjJxAAJggYMRQz5hSdd5pq1sSggfxPbw2ld71pt4x5wwViA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/expect": "30.3.0", - "@jest/types": "30.3.0", - "jest-mock": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/pattern": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.0.1.tgz", - "integrity": "sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "jest-regex-util": "30.0.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.3.0.tgz", - "integrity": "sha512-a09z89S+PkQnL055bVj8+pe2Caed2PBOaczHcXCykW5ngxX9EWx/1uAwncxc/HiU0oZqfwseMjyhxgRjS49qPw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.3.0", - "@jest/test-result": "30.3.0", - "@jest/transform": "30.3.0", - "@jest/types": "30.3.0", - "@jridgewell/trace-mapping": "^0.3.25", - "@types/node": "*", - "chalk": "^4.1.2", - "collect-v8-coverage": "^1.0.2", - "exit-x": "^0.2.2", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^5.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "30.3.0", - "jest-util": "30.3.0", - "jest-worker": "30.3.0", - "slash": "^3.0.0", - "string-length": "^4.0.2", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/schemas": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.34.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/snapshot-utils": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.3.0.tgz", - "integrity": "sha512-ORbRN9sf5PP82v3FXNSwmO1OTDR2vzR2YTaR+E3VkSBZ8zadQE6IqYdYEeFH1NIkeB2HIGdF02dapb6K0Mj05g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "natural-compare": "^1.4.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/source-map": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.1.tgz", - "integrity": "sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "callsites": "^3.1.0", - "graceful-fs": "^4.2.11" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.3.0.tgz", - "integrity": "sha512-e/52nJGuD74AKTSe0P4y5wFRlaXP0qmrS17rqOMHeSwm278VyNyXE3gFO/4DTGF9w+65ra3lo3VKj0LBrzmgdQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.3.0", - "@jest/types": "30.3.0", - "@types/istanbul-lib-coverage": "^2.0.6", - "collect-v8-coverage": "^1.0.2" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/test-sequencer": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.3.0.tgz", - "integrity": "sha512-dgbWy9b8QDlQeRZcv7LNF+/jFiiYHTKho1xirauZ7kVwY7avjFF6uTT0RqlgudB5OuIPagFdVtfFMosjVbk1eA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "30.3.0", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.3.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.3.0.tgz", - "integrity": "sha512-TLKY33fSLVd/lKB2YI1pH69ijyUblO/BQvCj566YvnwuzoTNr648iE0j22vRvVNk2HsPwByPxATg3MleS3gf5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.27.4", - "@jest/types": "30.3.0", - "@jridgewell/trace-mapping": "^0.3.25", - "babel-plugin-istanbul": "^7.0.1", - "chalk": "^4.1.2", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.3.0", - "jest-regex-util": "30.0.1", - "jest-util": "30.3.0", - "pirates": "^4.0.7", - "slash": "^3.0.0", - "write-file-atomic": "^5.0.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/types": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.3.0.tgz", - "integrity": "sha512-JHm87k7bA33hpBngtU8h6UBub/fqqA9uXfw+21j5Hmk7ooPHlboRNxHq0JcMtC+n8VJGP1mcfnD3Mk+XKe1oSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.1", - "@jest/schemas": "30.0.5", - "@types/istanbul-lib-coverage": "^2.0.6", - "@types/istanbul-reports": "^3.0.4", - "@types/node": "*", - "@types/yargs": "^17.0.33", - "chalk": "^4.1.2" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" - } - }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@pkgr/core": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", - "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/pkgr" - } - }, - "node_modules/@sinclair/typebox": { - "version": "0.34.48", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.48.tgz", - "integrity": "sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "15.1.1", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.1.1.tgz", - "integrity": "sha512-cO5W33JgAPbOh07tvZjUOJ7oWhtaqGHiZw+11DPbyqh2kHTBc3eF/CjJDeQ4205RLQsX6rxCuYOroFQwl7JDRw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.1" - } - }, - "node_modules/@swc/core": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.15.18.tgz", - "integrity": "sha512-z87aF9GphWp//fnkRsqvtY+inMVPgYW3zSlXH1kJFvRT5H/wiAn+G32qW5l3oEk63KSF1x3Ov0BfHCObAmT8RA==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@swc/counter": "^0.1.3", - "@swc/types": "^0.1.25" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/swc" - }, - "optionalDependencies": { - "@swc/core-darwin-arm64": "1.15.18", - "@swc/core-darwin-x64": "1.15.18", - "@swc/core-linux-arm-gnueabihf": "1.15.18", - "@swc/core-linux-arm64-gnu": "1.15.18", - "@swc/core-linux-arm64-musl": "1.15.18", - "@swc/core-linux-x64-gnu": "1.15.18", - "@swc/core-linux-x64-musl": "1.15.18", - "@swc/core-win32-arm64-msvc": "1.15.18", - "@swc/core-win32-ia32-msvc": "1.15.18", - "@swc/core-win32-x64-msvc": "1.15.18" - }, - "peerDependencies": { - "@swc/helpers": ">=0.5.17" - }, - "peerDependenciesMeta": { - "@swc/helpers": { - "optional": true - } - } - }, - "node_modules/@swc/core-darwin-arm64": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.15.18.tgz", - "integrity": "sha512-+mIv7uBuSaywN3C9LNuWaX1jJJ3SKfiJuE6Lr3bd+/1Iv8oMU7oLBjYMluX1UrEPzwN2qCdY6Io0yVicABoCwQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-darwin-x64": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.15.18.tgz", - "integrity": "sha512-wZle0eaQhnzxWX5V/2kEOI6Z9vl/lTFEC6V4EWcn+5pDjhemCpQv9e/TDJ0GIoiClX8EDWRvuZwh+Z3dhL1NAg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm-gnueabihf": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.15.18.tgz", - "integrity": "sha512-ao61HGXVqrJFHAcPtF4/DegmwEkVCo4HApnotLU8ognfmU8x589z7+tcf3hU+qBiU1WOXV5fQX6W9Nzs6hjxDw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-gnu": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.15.18.tgz", - "integrity": "sha512-3xnctOBLIq3kj8PxOCgPrGjBLP/kNOddr6f5gukYt/1IZxsITQaU9TDyjeX6jG+FiCIHjCuWuffsyQDL5Ew1bg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-musl": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.15.18.tgz", - "integrity": "sha512-0a+Lix+FSSHBSBOA0XznCcHo5/1nA6oLLjcnocvzXeqtdjnPb+SvchItHI+lfeiuj1sClYPDvPMLSLyXFaiIKw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-x64-gnu": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.15.18.tgz", - "integrity": "sha512-wG9J8vReUlpaHz4KOD/5UE1AUgirimU4UFT9oZmupUDEofxJKYb1mTA/DrMj0s78bkBiNI+7Fo2EgPuvOJfuAA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-x64-musl": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.15.18.tgz", - "integrity": "sha512-4nwbVvCphKzicwNWRmvD5iBaZj8JYsRGa4xOxJmOyHlMDpsvvJ2OR2cODlvWyGFH6BYL1MfIAK3qph3hp0Az6g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-arm64-msvc": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.15.18.tgz", - "integrity": "sha512-zk0RYO+LjiBCat2RTMHzAWaMky0cra9loH4oRrLKLLNuL+jarxKLFDA8xTZWEkCPLjUTwlRN7d28eDLLMgtUcQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-ia32-msvc": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.15.18.tgz", - "integrity": "sha512-yVuTrZ0RccD5+PEkpcLOBAuPbYBXS6rslENvIXfvJGXSdX5QGi1ehC4BjAMl5FkKLiam4kJECUI0l7Hq7T1vwg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-x64-msvc": { - "version": "1.15.18", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.15.18.tgz", - "integrity": "sha512-7NRmE4hmUQNCbYU3Hn9Tz57mK9Qq4c97ZS+YlamlK6qG9Fb5g/BB3gPDe0iLlJkns/sYv2VWSkm8c3NmbEGjbg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/counter": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/@swc/jest": { - "version": "0.2.39", - "resolved": "https://registry.npmjs.org/@swc/jest/-/jest-0.2.39.tgz", - "integrity": "sha512-eyokjOwYd0Q8RnMHri+8/FS1HIrIUKK/sRrFp8c1dThUOfNeCWbLmBP1P5VsKdvmkd25JaH+OKYwEYiAYg9YAA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/create-cache-key-function": "^30.0.0", - "@swc/counter": "^0.1.3", - "jsonc-parser": "^3.2.0" - }, - "engines": { - "npm": ">= 7.0.0" - }, - "peerDependencies": { - "@swc/core": "*" - } - }, - "node_modules/@swc/types": { - "version": "0.1.25", - "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.25.tgz", - "integrity": "sha512-iAoY/qRhNH8a/hBvm3zKj9qQ4oc2+3w1unPJa2XvTK3XjeLXtzcCingVPw/9e5mn1+0yPqxcBGp9Jf0pkfMb1g==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@swc/counter": "^0.1.3" - } - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.27.0", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", - "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.2" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", - "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", - "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", - "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/jest": { - "version": "30.0.0", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-30.0.0.tgz", - "integrity": "sha512-XTYugzhuwqWjws0CVz8QpM36+T+Dz5mTEBKhNs/esGLnCIlGdRy+Dq78NRjd7ls7r8BC8ZRMOrKlkO1hU0JOwA==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^30.0.0", - "pretty-format": "^30.0.0" - } - }, - "node_modules/@types/node": { - "version": "25.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", - "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "csstype": "^3.2.2" - } - }, - "node_modules/@types/stack-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", - "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/yargs": { - "version": "17.0.35", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", - "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.3", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", - "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@ungap/structured-clone": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", - "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", - "dev": true, - "license": "ISC" - }, - "node_modules/@unrs/resolver-binding-android-arm-eabi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", - "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@unrs/resolver-binding-android-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", - "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@unrs/resolver-binding-darwin-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", - "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@unrs/resolver-binding-darwin-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", - "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@unrs/resolver-binding-freebsd-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", - "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", - "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", - "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", - "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", - "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", - "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", - "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", - "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", - "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", - "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", - "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-wasm32-wasi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", - "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^0.2.11" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", - "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", - "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-x64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", - "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.3.5", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", - "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true, - "license": "MIT" - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/auto-bind": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/auto-bind/-/auto-bind-5.0.1.tgz", - "integrity": "sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg==", - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/babel-jest": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.3.0.tgz", - "integrity": "sha512-gRpauEU2KRrCox5Z296aeVHR4jQ98BCnu0IO332D/xpHNOsIH/bgSRk9k6GbKIbBw8vFeN6ctuu6tV8WOyVfYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/transform": "30.3.0", - "@types/babel__core": "^7.20.5", - "babel-plugin-istanbul": "^7.0.1", - "babel-preset-jest": "30.3.0", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "slash": "^3.0.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.11.0 || ^8.0.0-0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.1.tgz", - "integrity": "sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA==", - "dev": true, - "license": "BSD-3-Clause", - "workspaces": [ - "test/babel-8" - ], - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-instrument": "^6.0.2", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.3.0.tgz", - "integrity": "sha512-+TRkByhsws6sfPjVaitzadk1I0F5sPvOVUH5tyTSzhePpsGIVrdeunHSw/C36QeocS95OOk8lunc4rlu5Anwsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/babel__core": "^7.20.5" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" - } - }, - "node_modules/babel-preset-jest": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.3.0.tgz", - "integrity": "sha512-6ZcUbWHC+dMz2vfzdNwi87Z1gQsLNK2uLuK1Q89R11xdvejcivlYYwDlEv0FHX3VwEXpbBQ9uufB/MUNpZGfhQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "babel-plugin-jest-hoist": "30.3.0", - "babel-preset-current-node-syntax": "^1.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.11.0 || ^8.0.0-beta.1" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/baseline-browser-mapping": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.10.tgz", - "integrity": "sha512-sUoJ3IMxx4AyRqO4MLeHlnGDkyXRoUG0/AI9fjK+vS72ekpV0yWVY7O0BVjmBcRtkNcsAO2QDZ4tdKKGoI6YaQ==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/browserslist": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", - "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.9.0", - "caniuse-lite": "^1.0.30001759", - "electron-to-chromium": "^1.5.263", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.2.0" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001780", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001780.tgz", - "integrity": "sha512-llngX0E7nQci5BPJDqoZSbuZ5Bcs9F5db7EtgfwBerX9XGtkkiO4NwfDDIRzHTTwcYC8vC7bmeUEPGrKlR/TkQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/ci-info": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", - "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/cli-boxes": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-4.0.1.tgz", - "integrity": "sha512-5IOn+jcCEHEraYolBPs/sT4BxYCe2nHg374OPiItB1O96KZFseS2gthU4twyYzeDcFew4DaUM/xwc5BQf08JJw==", - "license": "MIT", - "engines": { - "node": ">=18.20 <19 || >=20.10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-cursor": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", - "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", - "license": "MIT", - "dependencies": { - "restore-cursor": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-truncate": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-6.0.0.tgz", - "integrity": "sha512-3+YKIUFsohD9MIoOFPFBldjAlnfCmCDcqe6aYGFqlDTRKg80p4wg35L+j83QQ63iOlKRccEkbn8IuM++HsgEjA==", - "license": "MIT", - "dependencies": { - "slice-ansi": "^9.0.0", - "string-width": "^8.2.0" - }, - "engines": { - "node": ">=22" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-truncate/node_modules/string-width": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", - "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.5.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/code-excerpt": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", - "integrity": "sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==", - "license": "MIT", - "dependencies": { - "convert-to-spaces": "^2.0.1" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", - "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/commander": { - "version": "14.0.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", - "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", - "license": "MIT", - "engines": { - "node": ">=20" - } - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-to-spaces": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", - "integrity": "sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==", - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, - "node_modules/create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "devOptional": true, - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dedent": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", - "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/diff": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", - "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, - "license": "MIT" - }, - "node_modules/electron-to-chromium": { - "version": "1.5.321", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.321.tgz", - "integrity": "sha512-L2C7Q279W2D/J4PLZLk7sebOILDSWos7bMsMNN06rK482umHUrh/3lM8G7IlHFOYip2oAg5nha1rCMxr/rs6ZQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, - "license": "MIT" - }, - "node_modules/environment": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", - "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/es-toolkit": { - "version": "1.46.1", - "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.46.1.tgz", - "integrity": "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ==", - "license": "MIT", - "workspaces": [ - "docs", - "benchmarks" - ] - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/execa/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/exit-x": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/exit-x/-/exit-x-0.2.2.tgz", - "integrity": "sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.3.0.tgz", - "integrity": "sha512-1zQrciTiQfRdo7qJM1uG4navm8DayFa2TgCSRlzUyNkhcJ6XUZF3hjnpkyr3VhAqPH7i/9GkG7Tv5abz6fqz0Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "30.3.0", - "@jest/get-type": "30.1.0", - "jest-matcher-utils": "30.3.0", - "jest-message-util": "30.3.0", - "jest-mock": "30.3.0", - "jest-util": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-east-asian-width": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", - "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/import-local": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", - "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/indent-string": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", - "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ink": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/ink/-/ink-7.0.1.tgz", - "integrity": "sha512-o6LAC268PLawlGVYrXTyaTfke4VtJftEheuwbgkQf7yvSXyWp1nRwBbAyKEkWXFZZsW/la5wrMuNbuBvZK2C1w==", - "license": "MIT", - "dependencies": { - "@alcalzone/ansi-tokenize": "^0.3.0", - "ansi-escapes": "^7.3.0", - "ansi-styles": "^6.2.3", - "auto-bind": "^5.0.1", - "chalk": "^5.6.2", - "cli-boxes": "^4.0.1", - "cli-cursor": "^4.0.0", - "cli-truncate": "^6.0.0", - "code-excerpt": "^4.0.0", - "es-toolkit": "^1.45.1", - "indent-string": "^5.0.0", - "is-in-ci": "^2.0.0", - "patch-console": "^2.0.0", - "react-reconciler": "^0.33.0", - "scheduler": "^0.27.0", - "signal-exit": "^3.0.7", - "slice-ansi": "^9.0.0", - "stack-utils": "^2.0.6", - "string-width": "^8.2.0", - "terminal-size": "^4.0.1", - "type-fest": "^5.5.0", - "widest-line": "^6.0.0", - "wrap-ansi": "^10.0.0", - "ws": "^8.20.0", - "yoga-layout": "~3.2.1" - }, - "engines": { - "node": ">=22" - }, - "peerDependencies": { - "@types/react": ">=19.2.0", - "react": ">=19.2.0", - "react-devtools-core": ">=6.1.2" - }, - "peerDependenciesMeta": { - "@types/react": { - "optional": true - }, - "react-devtools-core": { - "optional": true - } - } - }, - "node_modules/ink-testing-library": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ink-testing-library/-/ink-testing-library-4.0.0.tgz", - "integrity": "sha512-yF92kj3pmBvk7oKbSq5vEALO//o7Z9Ck/OaLNlkzXNeYdwfpxMQkSowGTFUCS5MSu9bWfSZMewGpp7bFc66D7Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/react": ">=18.0.0" - }, - "peerDependenciesMeta": { - "@types/react": { - "optional": true - } - } - }, - "node_modules/ink/node_modules/ansi-escapes": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", - "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", - "license": "MIT", - "dependencies": { - "environment": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ink/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/ink/node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/ink/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "license": "ISC" - }, - "node_modules/ink/node_modules/string-width": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", - "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.5.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ink/node_modules/type-fest": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.6.0.tgz", - "integrity": "sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA==", - "license": "(MIT OR CC0-1.0)", - "dependencies": { - "tagged-tag": "^1.0.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ink/node_modules/wrap-ansi": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.0.tgz", - "integrity": "sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.3", - "string-width": "^8.2.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/is-in-ci": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-in-ci/-/is-in-ci-2.0.0.tgz", - "integrity": "sha512-cFeerHriAnhrQSbpAxL37W1wcJKUUX07HyLWZCW1URJT/ra3GyUTzBgUnh24TMVfNTV2Hij2HLxkPHFZfOZy5w==", - "license": "MIT", - "bin": { - "is-in-ci": "cli.js" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", - "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.23", - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/jest": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.3.0.tgz", - "integrity": "sha512-AkXIIFcaazymvey2i/+F94XRnM6TsVLZDhBMLsd1Sf/W0wzsvvpjeyUrCZD6HGG4SDYPgDJDBKeiJTBb10WzMg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "30.3.0", - "@jest/types": "30.3.0", - "import-local": "^3.2.0", - "jest-cli": "30.3.0" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.3.0.tgz", - "integrity": "sha512-B/7Cny6cV5At6M25EWDgf9S617lHivamL8vl6KEpJqkStauzcG4e+WPfDgMMF+H4FVH4A2PLRyvgDJan4441QA==", - "dev": true, - "license": "MIT", - "dependencies": { - "execa": "^5.1.1", - "jest-util": "30.3.0", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-circus": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.3.0.tgz", - "integrity": "sha512-PyXq5szeSfR/4f1lYqCmmQjh0vqDkURUYi9N6whnHjlRz4IUQfMcXkGLeEoiJtxtyPqgUaUUfyQlApXWBSN1RA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/expect": "30.3.0", - "@jest/test-result": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "chalk": "^4.1.2", - "co": "^4.6.0", - "dedent": "^1.6.0", - "is-generator-fn": "^2.1.0", - "jest-each": "30.3.0", - "jest-matcher-utils": "30.3.0", - "jest-message-util": "30.3.0", - "jest-runtime": "30.3.0", - "jest-snapshot": "30.3.0", - "jest-util": "30.3.0", - "p-limit": "^3.1.0", - "pretty-format": "30.3.0", - "pure-rand": "^7.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.6" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-cli": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.3.0.tgz", - "integrity": "sha512-l6Tqx+j1fDXJEW5bqYykDQQ7mQg+9mhWXtnj+tQZrTWYHyHoi6Be8HPumDSA+UiX2/2buEgjA58iJzdj146uCw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "30.3.0", - "@jest/test-result": "30.3.0", - "@jest/types": "30.3.0", - "chalk": "^4.1.2", - "exit-x": "^0.2.2", - "import-local": "^3.2.0", - "jest-config": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0", - "yargs": "^17.7.2" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-config": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.3.0.tgz", - "integrity": "sha512-WPMAkMAtNDY9P/oKObtsRG/6KTrhtgPJoBTmk20uDn4Uy6/3EJnnaZJre/FMT1KVRx8cve1r7/FlMIOfRVWL4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.27.4", - "@jest/get-type": "30.1.0", - "@jest/pattern": "30.0.1", - "@jest/test-sequencer": "30.3.0", - "@jest/types": "30.3.0", - "babel-jest": "30.3.0", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "deepmerge": "^4.3.1", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", - "jest-circus": "30.3.0", - "jest-docblock": "30.2.0", - "jest-environment-node": "30.3.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.3.0", - "jest-runner": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0", - "parse-json": "^5.2.0", - "pretty-format": "30.3.0", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "esbuild-register": ">=3.4.0", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "esbuild-register": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-diff": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.3.0.tgz", - "integrity": "sha512-n3q4PDQjS4LrKxfWB3Z5KNk1XjXtZTBwQp71OP0Jo03Z6V60x++K5L8k6ZrW8MY8pOFylZvHM0zsjS1RqlHJZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/diff-sequences": "30.3.0", - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "pretty-format": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-docblock": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.2.0.tgz", - "integrity": "sha512-tR/FFgZKS1CXluOQzZvNH3+0z9jXr3ldGSD8bhyuxvlVUwbeLOGynkunvlTMxchC5urrKndYiwCFC0DLVjpOCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "detect-newline": "^3.1.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-each": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.3.0.tgz", - "integrity": "sha512-V8eMndg/aZ+3LnCJgSm13IxS5XSBM22QSZc9BtPK8Dek6pm+hfUNfwBdvsB3d342bo1q7wnSkC38zjX259qZNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "@jest/types": "30.3.0", - "chalk": "^4.1.2", - "jest-util": "30.3.0", - "pretty-format": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", - "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/fake-timers": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "jest-mock": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.3.0.tgz", - "integrity": "sha512-mMi2oqG4KRU0R9QEtscl87JzMXfUhbKaFqOxmjb2CKcbHcUGFrJCBWHmnTiUqi6JcnzoBlO4rWfpdl2k/RfLCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "@types/node": "*", - "anymatch": "^3.1.3", - "fb-watchman": "^2.0.2", - "graceful-fs": "^4.2.11", - "jest-regex-util": "30.0.1", - "jest-util": "30.3.0", - "jest-worker": "30.3.0", - "picomatch": "^4.0.3", - "walker": "^1.0.8" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.3" - } - }, - "node_modules/jest-leak-detector": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.3.0.tgz", - "integrity": "sha512-cuKmUUGIjfXZAiGJ7TbEMx0bcqNdPPI6P1V+7aF+m/FUJqFDxkFR4JqkTu8ZOiU5AaX/x0hZ20KaaIPXQzbMGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "pretty-format": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-matcher-utils": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.3.0.tgz", - "integrity": "sha512-HEtc9uFQgaUHkC7nLSlQL3Tph4Pjxt/yiPvkIrrDCt9jhoLIgxaubo1G+CFOnmHYMxHwwdaSN7mkIFs6ZK8OhA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "jest-diff": "30.3.0", - "pretty-format": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-message-util": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.3.0.tgz", - "integrity": "sha512-Z/j4Bo+4ySJ+JPJN3b2Qbl9hDq3VrXmnjjGEWD/x0BCXeOXPTV1iZYYzl2X8c1MaCOL+ewMyNBcm88sboE6YWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@jest/types": "30.3.0", - "@types/stack-utils": "^2.0.3", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "picomatch": "^4.0.3", - "pretty-format": "30.3.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.6" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-mock": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.3.0.tgz", - "integrity": "sha512-OTzICK8CpE+t4ndhKrwlIdbM6Pn8j00lvmSmq5ejiO+KxukbLjgOflKWMn3KE34EZdQm5RqTuKj+5RIEniYhog==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "@types/node": "*", - "jest-util": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.0.1.tgz", - "integrity": "sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.3.0.tgz", - "integrity": "sha512-NRtTAHQlpd15F9rUR36jqwelbrDV/dY4vzNte3S2kxCKUJRYNd5/6nTSbYiak1VX5g8IoFF23Uj5TURkUW8O5g==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.3.0", - "jest-pnp-resolver": "^1.2.3", - "jest-util": "30.3.0", - "jest-validate": "30.3.0", - "slash": "^3.0.0", - "unrs-resolver": "^1.7.11" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.3.0.tgz", - "integrity": "sha512-9ev8s3YN6Hsyz9LV75XUwkCVFlwPbaFn6Wp75qnI0wzAINYWY8Fb3+6y59Rwd3QaS3kKXffHXsZMziMavfz/nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-regex-util": "30.0.1", - "jest-snapshot": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-runner": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.3.0.tgz", - "integrity": "sha512-gDv6C9LGKWDPLia9TSzZwf4h3kMQCqyTpq+95PODnTRDO0g9os48XIYYkS6D236vjpBir2fF63YmJFtqkS5Duw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.3.0", - "@jest/environment": "30.3.0", - "@jest/test-result": "30.3.0", - "@jest/transform": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "chalk": "^4.1.2", - "emittery": "^0.13.1", - "exit-x": "^0.2.2", - "graceful-fs": "^4.2.11", - "jest-docblock": "30.2.0", - "jest-environment-node": "30.3.0", - "jest-haste-map": "30.3.0", - "jest-leak-detector": "30.3.0", - "jest-message-util": "30.3.0", - "jest-resolve": "30.3.0", - "jest-runtime": "30.3.0", - "jest-util": "30.3.0", - "jest-watcher": "30.3.0", - "jest-worker": "30.3.0", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-runtime": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.3.0.tgz", - "integrity": "sha512-CgC+hIBJbuh78HEffkhNKcbXAytQViplcl8xupqeIWyKQF50kCQA8J7GeJCkjisC6hpnC9Muf8jV5RdtdFbGng==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/fake-timers": "30.3.0", - "@jest/globals": "30.3.0", - "@jest/source-map": "30.0.1", - "@jest/test-result": "30.3.0", - "@jest/transform": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "chalk": "^4.1.2", - "cjs-module-lexer": "^2.1.0", - "collect-v8-coverage": "^1.0.2", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.3.0", - "jest-message-util": "30.3.0", - "jest-mock": "30.3.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.3.0", - "jest-snapshot": "30.3.0", - "jest-util": "30.3.0", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.3.0.tgz", - "integrity": "sha512-f14c7atpb4O2DeNhwcvS810Y63wEn8O1HqK/luJ4F6M4NjvxmAKQwBUWjbExUtMxWJQ0wVgmCKymeJK6NZMnfQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.27.4", - "@babel/generator": "^7.27.5", - "@babel/plugin-syntax-jsx": "^7.27.1", - "@babel/plugin-syntax-typescript": "^7.27.1", - "@babel/types": "^7.27.3", - "@jest/expect-utils": "30.3.0", - "@jest/get-type": "30.1.0", - "@jest/snapshot-utils": "30.3.0", - "@jest/transform": "30.3.0", - "@jest/types": "30.3.0", - "babel-preset-current-node-syntax": "^1.2.0", - "chalk": "^4.1.2", - "expect": "30.3.0", - "graceful-fs": "^4.2.11", - "jest-diff": "30.3.0", - "jest-matcher-utils": "30.3.0", - "jest-message-util": "30.3.0", - "jest-util": "30.3.0", - "pretty-format": "30.3.0", - "semver": "^7.7.2", - "synckit": "^0.11.8" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/jest-util": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.3.0.tgz", - "integrity": "sha512-/jZDa00a3Sz7rdyu55NLrQCIrbyIkbBxareejQI315f/i8HjYN+ZWsDLLpoQSiUIEIyZF/R8fDg3BmB8AtHttg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.3.0", - "@types/node": "*", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "graceful-fs": "^4.2.11", - "picomatch": "^4.0.3" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-validate": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.3.0.tgz", - "integrity": "sha512-I/xzC8h5G+SHCb2P2gWkJYrNiTbeL47KvKeW5EzplkyxzBRBw1ssSHlI/jXec0ukH2q7x2zAWQm7015iusg62Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "@jest/types": "30.3.0", - "camelcase": "^6.3.0", - "chalk": "^4.1.2", - "leven": "^3.1.0", - "pretty-format": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-watcher": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.3.0.tgz", - "integrity": "sha512-PJ1d9ThtTR8aMiBWUdcownq9mDdLXsQzJayTk4kmaBRHKvwNQn+ANveuhEBUyNI2hR1TVhvQ8D5kHubbzBHR/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "ansi-escapes": "^4.3.2", - "chalk": "^4.1.2", - "emittery": "^0.13.1", - "jest-util": "30.3.0", - "string-length": "^4.0.2" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-worker": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.3.0.tgz", - "integrity": "sha512-DrCKkaQwHexjRUFTmPzs7sHQe0TSj9nvDALKGdwmK5mW9v7j90BudWirKAJHt3QQ9Dhrg1F7DogPzhChppkJpQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@ungap/structured-clone": "^1.3.0", - "jest-util": "30.3.0", - "merge-stream": "^2.0.0", - "supports-color": "^8.1.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsonc-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-dir/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true, - "license": "ISC" - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true, - "license": "MIT" - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", - "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.2" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/napi-postinstall": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", - "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", - "dev": true, - "license": "MIT", - "bin": { - "napi-postinstall": "lib/cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/napi-postinstall" - } - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.36", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", - "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", - "dev": true, - "license": "MIT" - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/patch-console": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/patch-console/-/patch-console-2.0.0.tgz", - "integrity": "sha512-0YNdUceMdaQwoKce1gatDScmMo5pu/tfABfnzEqeG0gtTmd7mh/WcwgUjtAeOU7N8nFFlbQBnFK2gXW5fGvmMA==", - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pretty-format": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.3.0.tgz", - "integrity": "sha512-oG4T3wCbfeuvljnyAzhBvpN45E8iOTXCU/TD3zXW80HA3dQ4ahdqMkWGiPWZvjpQwlbyHrPTWUAqUzGzv4l1JQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.5", - "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/pure-rand": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", - "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ], - "license": "MIT" - }, - "node_modules/react": { - "version": "19.2.5", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", - "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/react-reconciler": { - "version": "0.33.0", - "resolved": "https://registry.npmjs.org/react-reconciler/-/react-reconciler-0.33.0.tgz", - "integrity": "sha512-KetWRytFv1epdpJc3J4G75I4WrplZE5jOL7Yq0p34+OVOKF4Se7WrdIdVC45XsSSmUTlht2FM/fM1FZb1mfQeA==", - "license": "MIT", - "dependencies": { - "scheduler": "^0.27.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "peerDependencies": { - "react": "^19.2.0" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/restore-cursor": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", - "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", - "license": "MIT", - "dependencies": { - "onetime": "^5.1.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/restore-cursor/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "license": "ISC" - }, - "node_modules/scheduler": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", - "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/slice-ansi": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-9.0.0.tgz", - "integrity": "sha512-SO/3iYL5S3W57LLEniscOGPZgOqZUPCx6d3dB+52B80yJ0XstzsC/eV8gnA4tM3MHDrKz+OCFSLNjswdSC+/bA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.3", - "is-fullwidth-code-point": "^5.1.0" - }, - "engines": { - "node": ">=22" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", - "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.3.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-length/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-length/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/synckit": { - "version": "0.11.12", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.12.tgz", - "integrity": "sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@pkgr/core": "^0.2.9" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/synckit" - } - }, - "node_modules/tagged-tag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", - "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", - "license": "MIT", - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/terminal-size": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/terminal-size/-/terminal-size-4.0.1.tgz", - "integrity": "sha512-avMLDQpUI9I5XFrklECw1ZEUPJhqzcwSWsyyI8blhRLT+8N1jLJWLWWYQpB2q2xthq8xDvjZPISVh53T/+CLYQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/test-exclude": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-8.0.0.tgz", - "integrity": "sha512-ZOffsNrXYggvU1mDGHk54I96r26P8SyMjO5slMKSc7+IWmtB/MQKnEC2fP51imB3/pT6YK5cT5E8f+Dd9KdyOQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^13.0.6", - "minimatch": "^10.2.2" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/test-exclude/node_modules/lru-cache": { - "version": "11.2.7", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", - "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/test-exclude/node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/ts-node": { - "version": "10.9.2", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", - "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/unrs-resolver": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", - "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "napi-postinstall": "^0.3.0" - }, - "funding": { - "url": "https://opencollective.com/unrs-resolver" - }, - "optionalDependencies": { - "@unrs/resolver-binding-android-arm-eabi": "1.11.1", - "@unrs/resolver-binding-android-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-x64": "1.11.1", - "@unrs/resolver-binding-freebsd-x64": "1.11.1", - "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", - "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", - "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-musl": "1.11.1", - "@unrs/resolver-binding-wasm32-wasi": "1.11.1", - "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", - "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", - "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true, - "license": "MIT" - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/widest-line": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-6.0.0.tgz", - "integrity": "sha512-U89AsyEeAsyoF0zVJBkG9zBgekjgjK7yk9sje3F4IQpXBJ10TF6ByLlIfjMhcmHMJgHZI4KHt4rdNfktzxIAMA==", - "license": "MIT", - "dependencies": { - "string-width": "^8.1.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/widest-line/node_modules/string-width": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", - "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.5.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/write-file-atomic": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", - "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, - "node_modules/ws": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", - "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/yoga-layout": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/yoga-layout/-/yoga-layout-3.2.1.tgz", - "integrity": "sha512-0LPOt3AxKqMdFBZA3HBAt/t/8vIKq7VaQYbuA8WxCgung+p9TVyKRYdpvCb80HcdTN2NkbIKbhNwKUfm3tQywQ==", - "license": "MIT" - } - } -} diff --git a/evals/package.json b/evals/package.json deleted file mode 100644 index b5ee7b7c..00000000 --- a/evals/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "jumbo-evals", - "version": "0.1.0", - "description": "Longitudinal eval system for measuring Jumbo's workflow efficacy across agent harnesses and sessions", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "bin": { - "eval": "dist/cli/index.js" - }, - "scripts": { - "build": "tsc", - "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathIgnorePatterns=/tests/integration/", - "test:integration": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathIgnorePatterns=/node_modules/ --testPathPatterns=/tests/integration/", - "build:jumbo": "npm --prefix .. run build", - "test:integration:source": "node scripts/with-jumbo-source.mjs npm run test:integration", - "smoke:jumbo": "npm run build && node scripts/with-jumbo-source.mjs node scripts/jumbo-contract-smoke.mjs", - "clean": "rimraf dist" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@jest/globals": "^30.3.0", - "@swc/core": "^1.15.18", - "@swc/jest": "^0.2.39", - "@types/jest": "^30.0.0", - "@types/node": "^25.5.0", - "@types/react": "^19.2.14", - "ink-testing-library": "^4.0.0", - "jest": "^30.3.0", - "ts-node": "^10.9.2", - "typescript": "^5.9.3" - }, - "overrides": { - "test-exclude": "^8.0.0" - }, - "dependencies": { - "commander": "^14.0.3", - "ink": "^7.0.1", - "react": "^19.2.5" - } -} diff --git a/evals/scenarios/cli-task-tracker.json b/evals/scenarios/cli-task-tracker.json deleted file mode 100644 index fe408f95..00000000 --- a/evals/scenarios/cli-task-tracker.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "CLI Task Tracker with State Machine", - "initialPrompt": "Build a CLI task tracker in TypeScript using a state machine for task lifecycle. Tasks move through states: draft -> active -> blocked | completed, with blocked -> active allowed. Use a state machine pattern where each transition is validated — invalid transitions throw a TransitionError with the current state, attempted state, and task ID. Storage is a single JSON file (~/.tasks.json). Commands: 'task add ', 'task list [--state <state>]', 'task start <id>', 'task complete <id>', 'task block <id> --reason <reason>'. Use Commander for CLI. File structure: src/domain/task.ts (Task type + state machine), src/domain/transitions.ts (valid transitions map), src/storage/json-store.ts, src/cli/commands.ts.", - "continuationPrompt": "Continue building the task tracker. Review the existing state machine, commands, and storage layer. Add the next feature while maintaining the state machine pattern and established conventions.", - "sessionCount": 6, - "expectedFiles": [ - "src/domain/task.ts", - "src/domain/transitions.ts", - "src/storage/json-store.ts", - "src/cli/commands.ts" - ], - "retentionPatterns": [ - "TransitionError", - "draft", - "active", - "blocked", - "completed", - "state machine", - "transition", - "~/.tasks.json" - ], - "expectedJumboMemoryCaptures": [ - { - "kind": "decision", - "match": "Commander" - }, - { - "kind": "component", - "match": "state machine" - }, - { - "kind": "invariant", - "match": "append only", - "sessionNumber": 4 - } - ], - "disruptions": [ - { - "type": "correction", - "sessionNumber": 3, - "content": "Add an 'archived' state. Completed tasks can be archived (completed -> archived). Archived tasks are excluded from 'task list' by default but shown with '--include-archived'. The archive transition must record an archivedAt timestamp.", - "recoveryPatterns": ["archived", "archivedAt", "include-archived"] - }, - { - "type": "new-constraint", - "sessionNumber": 4, - "content": "Tasks must now track their full state history as an audit trail. Each transition appends to a 'history' array: { fromState, toState, timestamp, reason? }. The 'task show <id>' command displays the history timeline. Never mutate history entries — append only.", - "recoveryPatterns": ["history", "audit trail", "fromState", "toState", "append only"] - } - ] -} diff --git a/evals/scenarios/event-sourced-inventory.json b/evals/scenarios/event-sourced-inventory.json deleted file mode 100644 index 72e35c56..00000000 --- a/evals/scenarios/event-sourced-inventory.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "name": "Event-Sourced Inventory System", - "initialPrompt": "Build an event-sourced inventory management system in TypeScript. Domain: a warehouse that tracks products, stock levels, and reservations. Start with the event store foundation — an append-only log of domain events (ProductAdded, StockReceived, StockReserved, ReservationConfirmed, ReservationCancelled). Use immutable event types with discriminated unions. Project current state by replaying events. No database — in-memory event store for now. File structure: src/events/types.ts, src/events/store.ts, src/projections/inventory.ts.", - "continuationPrompt": "Continue building the inventory system. Review the existing code and event types, then add the next logical capability. Maintain the established patterns, naming conventions, and architectural decisions from prior sessions.", - "sessionCount": 7, - "expectedFiles": [ - "src/events/types.ts", - "src/events/store.ts", - "src/projections/inventory.ts", - "src/commands/handlers.ts", - "src/domain/product.ts" - ], - "retentionPatterns": [ - "ProductAdded", - "StockReceived", - "StockReserved", - "discriminated union", - "append-only", - "event store", - "projection" - ], - "structuralAssertions": [ - { - "id": "s1-event-types-file", - "description": "Event type definitions exist", - "file": "src/events/types.ts", - "sessionNumber": 1, - "matcher": { "kind": "fileExists" } - }, - { - "id": "s1-discriminated-union", - "description": "Domain events modelled as a discriminated union (a union type listing the event variants)", - "file": "src/events/types.ts", - "sessionNumber": 1, - "matcher": { "kind": "matchesRegex", "pattern": "type\\s+[A-Za-z]+\\s*=[^;]*\\|" } - }, - { - "id": "s1-core-events", - "description": "The foundational event variants are declared", - "file": "src/events/**/*.ts", - "sessionNumber": 1, - "matcher": { "kind": "containsAll", "substrings": ["ProductAdded", "StockReceived", "StockReserved"] } - }, - { - "id": "s1-append-only-store", - "description": "Append-only event store with an append operation", - "file": "src/events/store.ts", - "sessionNumber": 1, - "matcher": { "kind": "matchesRegex", "pattern": "append\\s*\\(" } - }, - { - "id": "s1-projection", - "description": "State is derived via a projection that replays events", - "file": "src/projections/inventory.ts", - "sessionNumber": 1, - "matcher": { "kind": "fileExists" } - }, - { - "id": "s3-event-metadata-retrofit", - "description": "Disruption (session 3): all events carry metadata with correlationId, causationId, timestamp", - "file": "src/events/types.ts", - "sessionNumber": 3, - "matcher": { "kind": "containsAll", "substrings": ["correlationId", "causationId", "timestamp"] } - }, - { - "id": "s5-optimistic-concurrency", - "description": "Disruption (session 5): optimistic concurrency control with a ConcurrencyError", - "file": "src/**/*.ts", - "sessionNumber": 5, - "matcher": { "kind": "containsAll", "substrings": ["ConcurrencyError"] } - }, - { - "id": "s5-expected-version", - "description": "Disruption (session 5): commands specify an expected version", - "file": "src/**/*.ts", - "sessionNumber": 5, - "matcher": { "kind": "containsAny", "substrings": ["expectedVersion", "expected_version", "expectedversion"] } - }, - { - "id": "s7-events-persist", - "description": "Retention: the original event variants survive to the final session", - "file": "src/events/**/*.ts", - "sessionNumber": 7, - "matcher": { "kind": "containsAll", "substrings": ["ProductAdded", "StockReceived", "StockReserved"] } - }, - { - "id": "s7-metadata-persists", - "description": "Retention: the session-3 metadata decision is not regressed by the final session", - "file": "src/events/types.ts", - "sessionNumber": 7, - "matcher": { "kind": "containsAll", "substrings": ["correlationId", "causationId"] } - }, - { - "id": "s7-concurrency-persists", - "description": "Retention: the session-5 concurrency control is not regressed by the final session", - "file": "src/**/*.ts", - "sessionNumber": 7, - "matcher": { "kind": "containsAll", "substrings": ["ConcurrencyError"] } - } - ], - "expectedJumboMemoryCaptures": [ - { - "kind": "decision", - "match": "event-sourced" - }, - { - "kind": "invariant", - "match": "append-only" - }, - { - "kind": "component", - "match": "event store" - }, - { - "kind": "invariant", - "match": "optimistic concurrency", - "sessionNumber": 5 - } - ], - "disruptions": [ - { - "type": "correction", - "sessionNumber": 3, - "content": "The event types need a 'metadata' field containing timestamp, correlationId, and causationId. Retrofit all existing events and ensure new events include metadata. The projection must track the latest event timestamp per aggregate.", - "recoveryPatterns": ["metadata", "correlationId", "causationId"] - }, - { - "type": "new-constraint", - "sessionNumber": 5, - "content": "Add optimistic concurrency control. Each aggregate has a version number. Commands must specify the expected version. If the stored version differs, reject with a ConcurrencyError. This prevents lost updates when two commands target the same product.", - "recoveryPatterns": ["concurrency", "version", "ConcurrencyError", "expectedVersion"] - } - ] -} diff --git a/evals/scenarios/file-reconstruction-expense-tracker.json b/evals/scenarios/file-reconstruction-expense-tracker.json deleted file mode 100644 index 7c7ee8d1..00000000 --- a/evals/scenarios/file-reconstruction-expense-tracker.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "name": "File Reconstruction: Expense Tracker with Decision Log", - "initialPrompt": "Build an expense-tracker library in TypeScript with a layered architecture: domain types in src/domain/expense.ts, business logic in src/services/expense-service.ts, and persistence behind an in-memory store in src/storage/memory-store.ts. The service depends on the store through an interface, never concretely. IMPORTANT PROJECT RULE: maintain a DECISIONS.md file at the repo root; every time you make or change an architectural decision (layering, data representation, naming, error handling), append a dated entry to DECISIONS.md explaining the decision and why. Start with expenses (id, description, amount, category, date) and basic add/list operations.", - "continuationPrompt": "Continue building the expense tracker. Before writing any code, carefully re-read DECISIONS.md and the existing source files to reconstruct the project's conventions and prior decisions, then continue where the previous session left off, maintaining those decisions. Keep appending new decisions to DECISIONS.md.", - "sessionCount": 5, - "expectedFiles": [ - "src/domain/expense.ts", - "src/services/expense-service.ts", - "src/storage/memory-store.ts", - "DECISIONS.md" - ], - "retentionPatterns": [ - "ExpenseService", - "MemoryStore", - "DECISIONS" - ], - "structuralAssertions": [ - { - "id": "s1-domain-exists", - "description": "Domain layer established in session 1", - "file": "src/domain/expense.ts", - "sessionNumber": 1, - "matcher": { "kind": "fileExists" } - }, - { - "id": "s1-store-interface", - "description": "Service depends on a store abstraction, not a concrete store", - "file": "src/**/*.ts", - "sessionNumber": 1, - "matcher": { "kind": "matchesRegex", "pattern": "interface\\s+\\w*Store" } - }, - { - "id": "s1-decisions-log-started", - "description": "The in-repo decision log exists from session 1 — the baseline's recovery path", - "file": "DECISIONS.md", - "sessionNumber": 1, - "matcher": { "kind": "fileExists" } - }, - { - "id": "s3-minor-units-retrofit", - "description": "Disruption (session 2): amounts stored as integer minor units, visible in code by session 3", - "file": "src/**/*.ts", - "sessionNumber": 3, - "matcher": { "kind": "containsAny", "substrings": ["minorUnits", "amountMinor", "cents"] } - }, - { - "id": "s3-minor-units-logged", - "description": "The minor-units decision is legible in the decision log (file-recoverable context)", - "file": "DECISIONS.md", - "sessionNumber": 3, - "matcher": { "kind": "containsAny", "substrings": ["minor unit", "minorUnits", "cents", "integer"] } - }, - { - "id": "s5-audit-log", - "description": "Disruption (session 4): every mutation appends to an audit log, visible by session 5", - "file": "src/**/*.ts", - "sessionNumber": 5, - "matcher": { "kind": "containsAny", "substrings": ["auditLog", "AuditEntry", "audit"] } - }, - { - "id": "s5-minor-units-persist", - "description": "Retention: the session-2 correction is not regressed by the final session", - "file": "src/**/*.ts", - "sessionNumber": 5, - "matcher": { "kind": "containsAny", "substrings": ["minorUnits", "amountMinor", "cents"] } - }, - { - "id": "s5-layering-persists", - "description": "Retention: the store abstraction survives to the final session", - "file": "src/**/*.ts", - "sessionNumber": 5, - "matcher": { "kind": "matchesRegex", "pattern": "interface\\s+\\w*Store" } - }, - { - "id": "s5-decisions-log-persists", - "description": "Retention: the decision log is still maintained at the end", - "file": "DECISIONS.md", - "sessionNumber": 5, - "matcher": { "kind": "fileExists" } - } - ], - "disruptions": [ - { - "type": "correction", - "sessionNumber": 2, - "content": "Monetary amounts must never be floats. Store all amounts as integer minor units (cents). Retrofit the domain type, the service, and any arithmetic, and record this decision in DECISIONS.md.", - "recoveryPatterns": ["minorUnits", "cents"] - }, - { - "type": "new-constraint", - "sessionNumber": 4, - "content": "Every mutating operation (add, update, delete) must append an AuditEntry (timestamp, operation, expense id) to an audit log exposed by the service. Record this decision in DECISIONS.md.", - "recoveryPatterns": ["audit", "AuditEntry"] - } - ], - "expectedJumboMemoryCaptures": [ - { - "kind": "decision", - "match": "layered" - }, - { - "kind": "component", - "match": "ExpenseService" - }, - { - "kind": "invariant", - "match": "minor units", - "sessionNumber": 2 - } - ], - "jumboPlan": { - "preSeededMemory": [ - { - "kind": "invariant", - "planRef": "inv_layering", - "title": "Layered architecture with store abstraction", - "description": "Domain types, service logic, and storage are separate layers; the service depends on a store interface, never a concrete store.", - "rationale": "Restates the layering rule already given in the scenario prompt so both arms hold identical information at session 1." - }, - { - "kind": "guideline", - "planRef": "gl_decisions_log", - "title": "Record decisions in DECISIONS.md", - "description": "Every architectural decision is appended to DECISIONS.md with a date and rationale, as the scenario prompt requires.", - "rationale": "Restates the prompt's decision-log rule; no information beyond the shared prompt." - } - ], - "goals": [ - { - "planRef": "goal_foundation", - "title": "Domain, store, and decision log", - "objective": "Establish the Expense domain type, the store interface with an in-memory implementation, and start DECISIONS.md.", - "criteria": [ - "src/domain/expense.ts exports the Expense type", - "src/storage/memory-store.ts implements a store interface the service depends on", - "DECISIONS.md exists with the initial layering decision" - ], - "scopeIn": ["src", "DECISIONS.md"], - "sessionAvailableFrom": 1 - }, - { - "planRef": "goal_service", - "title": "Expense service operations", - "objective": "Implement add/list/update/delete on ExpenseService against the store interface, absorbing any corrections issued in the session.", - "criteria": [ - "src/services/expense-service.ts exports ExpenseService", - "Service operations go through the store interface only", - "New decisions from this session are appended to DECISIONS.md" - ], - "scopeIn": ["src", "DECISIONS.md"], - "sessionAvailableFrom": 2, - "prerequisitePlanRefs": ["goal_foundation"] - }, - { - "planRef": "goal_reporting", - "title": "Category summaries and audit integration", - "objective": "Add per-category summary reporting and integrate any newly mandated cross-cutting constraints, keeping all prior decisions intact.", - "criteria": [ - "Summary reporting groups expense totals by category", - "All mutating operations satisfy the audit constraint once mandated", - "DECISIONS.md reflects every decision made along the way" - ], - "scopeIn": ["src", "DECISIONS.md"], - "sessionAvailableFrom": 4, - "prerequisitePlanRefs": ["goal_service"] - } - ] - } -} diff --git a/evals/scenarios/null-hypothesis-rate-limiter.json b/evals/scenarios/null-hypothesis-rate-limiter.json deleted file mode 100644 index 83fd1df2..00000000 --- a/evals/scenarios/null-hypothesis-rate-limiter.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "name": "Null Hypothesis: Single-Session Rate Limiter", - "initialPrompt": "Build a rate-limiter library in TypeScript with two interchangeable strategies: a token bucket (capacity, refill rate per second) and a fixed window counter (limit per window). Both implement a common RateLimiter interface with a tryAcquire(key: string): boolean method. Pure TypeScript, no external dependencies, in-memory state. File structure: src/rate-limiter.ts (the RateLimiter interface), src/token-bucket.ts (TokenBucket), src/fixed-window.ts (FixedWindow), src/index.ts (re-exports).", - "sessionCount": 1, - "expectedFiles": [ - "src/rate-limiter.ts", - "src/token-bucket.ts", - "src/fixed-window.ts", - "src/index.ts" - ], - "retentionPatterns": [ - "RateLimiter", - "TokenBucket", - "FixedWindow", - "tryAcquire" - ], - "structuralAssertions": [ - { - "id": "s1-interface-exists", - "description": "The common RateLimiter interface exists", - "file": "src/rate-limiter.ts", - "sessionNumber": 1, - "matcher": { "kind": "exportsSymbol", "symbol": "RateLimiter" } - }, - { - "id": "s1-try-acquire", - "description": "The interface declares the tryAcquire contract", - "file": "src/rate-limiter.ts", - "sessionNumber": 1, - "matcher": { "kind": "matchesRegex", "pattern": "tryAcquire\\s*\\(" } - }, - { - "id": "s1-token-bucket", - "description": "Token bucket strategy is implemented and exported", - "file": "src/token-bucket.ts", - "sessionNumber": 1, - "matcher": { "kind": "exportsSymbol", "symbol": "TokenBucket" } - }, - { - "id": "s1-token-bucket-semantics", - "description": "Token bucket carries capacity and refill semantics", - "file": "src/token-bucket.ts", - "sessionNumber": 1, - "matcher": { "kind": "containsAll", "substrings": ["capacity", "refill"] } - }, - { - "id": "s1-fixed-window", - "description": "Fixed window strategy is implemented and exported", - "file": "src/fixed-window.ts", - "sessionNumber": 1, - "matcher": { "kind": "exportsSymbol", "symbol": "FixedWindow" } - }, - { - "id": "s1-barrel", - "description": "The index re-exports both strategies", - "file": "src/index.ts", - "sessionNumber": 1, - "matcher": { "kind": "containsAll", "substrings": ["TokenBucket", "FixedWindow"] } - } - ], - "expectedJumboMemoryCaptures": [ - { - "kind": "decision", - "match": "token bucket" - }, - { - "kind": "component", - "match": "RateLimiter" - } - ], - "jumboPlan": { - "goals": [ - { - "planRef": "goal_rate_limiter", - "title": "Rate limiter library", - "objective": "Implement the rate-limiter library exactly as specified in the scenario prompt: a RateLimiter interface with tryAcquire(key), a TokenBucket strategy and a FixedWindow strategy, re-exported from src/index.ts.", - "criteria": [ - "src/rate-limiter.ts exports a RateLimiter interface with tryAcquire(key: string): boolean", - "src/token-bucket.ts exports TokenBucket with capacity and per-second refill", - "src/fixed-window.ts exports FixedWindow with a limit per window", - "src/index.ts re-exports the interface and both strategies" - ], - "scopeIn": ["src"], - "sessionAvailableFrom": 1 - } - ] - } -} diff --git a/evals/scenarios/plugin-architecture.json b/evals/scenarios/plugin-architecture.json deleted file mode 100644 index 132da8ff..00000000 --- a/evals/scenarios/plugin-architecture.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "name": "Extensible Plugin Architecture", - "initialPrompt": "Build a plugin system in TypeScript for a document processing pipeline. The core pipeline loads plugins at startup, each plugin implements a DocumentPlugin interface with hooks: beforeProcess, transform, afterProcess, onError. Plugins are discovered from a plugins/ directory. The pipeline executes hooks in registration order. Start with: src/core/pipeline.ts (orchestrator), src/core/plugin-loader.ts (discovery and loading), src/core/types.ts (Plugin interface, DocumentContext, HookResult). Use a strict contract — plugins that throw during loading are skipped with a warning, not a crash.", - "continuationPrompt": "Continue building the plugin system. Review the existing plugin interface, pipeline, and any plugins already created. Add the next capability while maintaining the established patterns and plugin contract.", - "sessionCount": 6, - "expectedFiles": [ - "src/core/pipeline.ts", - "src/core/plugin-loader.ts", - "src/core/types.ts", - "src/plugins/markdown-plugin.ts", - "src/plugins/metadata-plugin.ts" - ], - "retentionPatterns": [ - "DocumentPlugin", - "beforeProcess", - "transform", - "afterProcess", - "onError", - "HookResult", - "DocumentContext", - "registration order" - ], - "expectedJumboMemoryCaptures": [ - { - "kind": "component", - "match": "plugin" - }, - { - "kind": "guideline", - "match": "priority", - "sessionNumber": 3 - }, - { - "kind": "decision", - "match": "topological", - "sessionNumber": 5 - } - ], - "disruptions": [ - { - "type": "correction", - "sessionNumber": 3, - "content": "The plugin interface is missing a priority field. Add 'priority: number' to DocumentPlugin. Plugins execute in priority order (lowest first), not registration order. Update the pipeline to sort by priority before executing each hook phase. Default priority is 100.", - "recoveryPatterns": ["priority", "lowest first", "sort"] - }, - { - "type": "scope-change", - "sessionNumber": 5, - "content": "Add plugin dependency resolution. Plugins can declare 'dependencies: string[]' listing other plugin names they require. The loader must topologically sort plugins so dependencies load first. Circular dependencies are a fatal error with a clear message naming the cycle.", - "recoveryPatterns": ["dependencies", "topological", "circular"] - } - ] -} diff --git a/evals/scenarios/progressive-backlog.json b/evals/scenarios/progressive-backlog.json deleted file mode 100644 index 29636619..00000000 --- a/evals/scenarios/progressive-backlog.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "name": "Progressive Backlog Release", - "initialPrompt": "Build a TypeScript CLI for managing personal reading lists. Session 1 lays foundations; the rest of the backlog will be released as the project matures.", - "continuationPrompt": "Continue work on the reading-list CLI. Pick up where the previous session left off, then take on the next goal the framework has handed you.", - "sessionCount": 3, - "expectedFiles": [ - "src/domain/book.ts", - "src/storage/json-store.ts", - "src/cli/commands.ts" - ], - "retentionPatterns": [ - "Book", - "ReadingList", - "json-store" - ], - "jumboPlan": { - "preSeededMemory": [ - { - "kind": "invariant", - "planRef": "inv_storage", - "title": "Single JSON storage file", - "description": "All reading-list state persists to a single ~/.reading-list.json file. No in-memory or sqlite stores.", - "rationale": "Keeps the scenario portable across sessions and easy to inspect." - }, - { - "kind": "component", - "planRef": "cmp_json_store", - "name": "JsonStore", - "type": "storage", - "description": "Reads and writes the reading-list JSON file with atomic replacement.", - "responsibility": "Durable persistence of reading-list state.", - "path": "src/storage/json-store.ts" - }, - { - "kind": "decision", - "planRef": "dec_commander", - "title": "Use Commander for the CLI surface", - "context": "The reading-list CLI needs subcommands, help text, and option parsing without hand-rolling argv handling.", - "rationale": "Commander matches the project's existing conventions and produces idiomatic --help output.", - "alternatives": ["yargs", "hand-rolled argv parser"] - } - ], - "goals": [ - { - "planRef": "goal_foundation", - "title": "Book domain + JSON store", - "objective": "Establish the Book type and a JsonStore that loads and saves ~/.reading-list.json.", - "criteria": [ - "src/domain/book.ts exports a Book type with id, title, author, status", - "src/storage/json-store.ts exposes load/save with atomic replacement", - "JsonStore round-trips an empty list cleanly" - ], - "scopeIn": ["src/domain", "src/storage"], - "sessionAvailableFrom": 1 - }, - { - "planRef": "goal_add_command", - "title": "`reading add` subcommand", - "objective": "Add a `reading add <title> --author <author>` subcommand that appends a book in the 'to-read' state.", - "criteria": [ - "src/cli/commands.ts registers the `add` subcommand via Commander", - "New books default to status='to-read' with a generated id", - "Adding two books produces a JSON file with two entries" - ], - "scopeIn": ["src/cli", "src/storage"], - "sessionAvailableFrom": 2, - "prerequisitePlanRefs": ["goal_foundation"] - }, - { - "planRef": "goal_status_transitions", - "title": "Reading status transitions", - "objective": "Allow books to move to 'reading' and 'finished' via `reading start <id>` and `reading finish <id>`.", - "criteria": [ - "`reading start` rejects books that are already 'finished'", - "`reading finish` records a finishedAt ISO timestamp", - "List output groups books by status" - ], - "scopeIn": ["src/cli", "src/domain"], - "sessionAvailableFrom": 3, - "prerequisitePlanRefs": ["goal_add_command"] - } - ] - } -} diff --git a/evals/scripts/jumbo-contract-smoke.mjs b/evals/scripts/jumbo-contract-smoke.mjs deleted file mode 100644 index aeda0102..00000000 --- a/evals/scripts/jumbo-contract-smoke.mjs +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env node -/** - * Contract smoke test between the evals and the jumbo CLI. - * - * Drives the real `jumbo` on PATH (point it at source via with-jumbo-source.mjs) - * through the exact surface the evals depend on, and asserts the contract holds: - * - `jumbo init --non-interactive ... --yolo` succeeds (ab-runner setup) - * - `jumbo <kind> add ...` succeeds for every memory kind (memory seeding + agent lifecycle) - * - `jumbo decisions list --format json` returns parseable JSON (memory snapshot) - * - `.jumbo/events/<aggregateId>/<seq>.<EventType>.json` layout holds, and every - * entry in the scorer's addedEventTypeByKind map is actually emitted by jumbo - * (jumbo-event-capture scorer contract) (Outcome 2/4) - * - * If a jumbo primitive refactor breaks any of these, this exits non-zero — so - * the jumbo PR gate catches it instead of the evals silently rotting. - * - * No coding-agent harness is involved, so this runs headless in CI. - */ -import { mkdtempSync, rmSync, existsSync, readdirSync, statSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { spawnSync } from 'node:child_process'; -// The production kind -> Added-event-type map, imported from the built evals so -// this smoke validates the *actual* scorer contract (not a duplicated copy) -// against real jumbo. Requires `npm run build` first (the smoke:jumbo script -// does this); a missing dist means the evals were not built. -import { addedEventTypeByKind } from '../dist/scoring/jumbo-event-capture-scorer.js'; - -const isWin = process.platform === 'win32'; -const failures = []; -function check(cond, message) { - if (cond) { - console.log(` ✓ ${message}`); - } else { - console.error(` ✗ ${message}`); - failures.push(message); - } -} - -// On Windows, `jumbo` resolves to a `.cmd` shim that requires shell:true; but -// shell:true does not quote array args, so values with spaces get split. Quote -// each token ourselves (mirrors LocalExecutor's Windows handling). On POSIX we -// pass the args array directly with shell:false, which needs no quoting. -function winQuote(arg) { - return /[\s"]/.test(arg) ? `"${arg.replace(/"/g, '\\"')}"` : arg; -} -function jumbo(args, cwd) { - const r = isWin - ? spawnSync(['jumbo', ...args].map(winQuote).join(' '), { cwd, encoding: 'utf8', shell: true }) - : spawnSync('jumbo', args, { cwd, encoding: 'utf8', shell: false }); - return { code: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '', error: r.error }; -} - -const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/; -function firstUuid(text) { - return (text.match(UUID_RE) || [])[0]; -} - -function listEventFiles(workDir) { - const eventsDir = join(workDir, '.jumbo', 'events'); - const out = []; - const walk = (dir, rel) => { - for (const name of readdirSync(dir)) { - const full = join(dir, name); - const childRel = rel ? `${rel}/${name}` : name; - if (statSync(full).isDirectory()) walk(full, childRel); - else out.push(childRel); - } - }; - if (existsSync(eventsDir)) walk(eventsDir, ''); - return out; -} - -const workDir = mkdtempSync(join(tmpdir(), 'jumbo-contract-')); -console.log(`[jumbo-contract-smoke] workDir: ${workDir}`); - -try { - // Confirm we are exercising a real jumbo binary at all. - const version = jumbo(['--version'], workDir); - check(version.code === 0, `jumbo --version succeeds (${version.stdout.trim() || version.stderr.trim()})`); - - const init = jumbo( - ['init', '--purpose', 'contract smoke', '--non-interactive', '--name', 'contract-smoke', '--yolo'], - workDir, - ); - check(init.code === 0, 'jumbo init --non-interactive --yolo succeeds'); - check(existsSync(join(workDir, '.jumbo', 'events')), '.jumbo/events/ created by init'); - - const decision = jumbo(['decision', 'add', '--title', 'Use event sourcing', '--context', 'smoke'], workDir); - check(decision.code === 0, 'jumbo decision add succeeds'); - - const component = jumbo( - ['component', 'add', '--name', 'EventStore', '--type', 'lib', '--description', 'd', '--responsibility', 'r', '--path', 'src/x.ts'], - workDir, - ); - check(component.code === 0, 'jumbo component add succeeds'); - - const goal = jumbo( - ['goal', 'add', '--title', 'Smoke goal', '--objective', 'verify contract', '--criteria', 'compiles', 'tests pass'], - workDir, - ); - check(goal.code === 0, 'jumbo goal add succeeds'); - - // Register one of every remaining memory kind so the event log exercises the - // whole addedEventTypeByKind map. relation needs two existing entity ids. - const invariant = jumbo(['invariant', 'add', '--title', 'Append only', '--description', 'log is immutable'], workDir); - check(invariant.code === 0, 'jumbo invariant add succeeds'); - - const guideline = jumbo( - ['guideline', 'add', '--category', 'testing', '--title', 'Coverage', '--description', 'cover new code', '--rationale', 'quality'], - workDir, - ); - check(guideline.code === 0, 'jumbo guideline add succeeds'); - - const dependency = jumbo( - ['dependency', 'add', '--name', 'Commander', '--ecosystem', 'npm', '--package-name', 'commander'], - workDir, - ); - check(dependency.code === 0, 'jumbo dependency add succeeds'); - - const decisionId = firstUuid(decision.stdout); - const componentId = firstUuid(component.stdout); - const relation = decisionId && componentId - ? jumbo( - ['relation', 'add', '--from-type', 'component', '--from-id', componentId, '--to-type', 'decision', '--to-id', decisionId, '--type', 'involves', '--description', 'EventStore realizes the decision'], - workDir, - ) - : { code: 1, stdout: '', stderr: 'could not parse decision/component ids' }; - check(relation.code === 0, 'jumbo relation add succeeds'); - - const decisionsList = jumbo(['decisions', 'list', '--format', 'json'], workDir); - check(decisionsList.code === 0, 'jumbo decisions list --format json succeeds'); - let parsed = null; - try { - parsed = JSON.parse(decisionsList.stdout); - } catch { - /* parsed stays null */ - } - check(parsed !== null, 'decisions list --format json emits parseable JSON'); - - // Event-store layout the Outcome 2 summarizer depends on. - const eventFiles = listEventFiles(workDir); - check(eventFiles.length > 0, `events written under .jumbo/events/ (${eventFiles.length} files)`); - const namingOk = eventFiles.every((p) => /^[^/]+\/\d+\..+\.json$/.test(p)); - check(namingOk, 'every event file matches <aggregateId>/<seq>.<EventType>.json'); - const types = new Set(eventFiles.map((p) => p.split('/').pop().replace(/^\d+\./, '').replace(/\.json$/, ''))); - // Validate the scorer's production map against what real jumbo actually emits. - // If a jumbo refactor renames an event type, the corresponding literal in - // addedEventTypeByKind no longer matches and this fails — instead of the - // scorer silently under-counting captures. - for (const [kind, eventType] of Object.entries(addedEventTypeByKind)) { - check(types.has(eventType), `event log contains ${eventType} for kind '${kind}' (validates addedEventTypeByKind)`); - } -} finally { - try { - rmSync(workDir, { recursive: true, force: true }); - } catch { - /* best-effort cleanup */ - } -} - -if (failures.length > 0) { - console.error(`\n[jumbo-contract-smoke] FAILED (${failures.length} contract violation(s)):`); - for (const f of failures) console.error(` - ${f}`); - process.exit(1); -} -console.log('\n[jumbo-contract-smoke] OK — evals/jumbo CLI contract holds.'); diff --git a/evals/scripts/with-jumbo-source.mjs b/evals/scripts/with-jumbo-source.mjs deleted file mode 100644 index 186eb382..00000000 --- a/evals/scripts/with-jumbo-source.mjs +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env node -/** - * Runs a command with `jumbo` on PATH resolved to the *sibling jumbo source - * build* (../dist/cli.js) instead of any globally-installed jumbo binary. - * - * This only works inside the jumbo.cli monorepo, where `evals/..` is the jumbo - * root — which is exactly the point: it ties the evals to the jumbo source they - * are meant to guard, eliminating the version skew between a globally-installed - * jumbo and the source under test. - * - * Usage: node scripts/with-jumbo-source.mjs <command> [args...] - * e.g. node scripts/with-jumbo-source.mjs npm run test:integration - * - * The eval harness (LocalExecutor) spawns `jumbo` directly on POSIX (shell:false) - * and via cmd.exe on Windows (shell:true), so we install both a `jumbo` shell - * shim and a `jumbo.cmd` shim into a temp dir prepended to PATH. - */ -import { mkdtempSync, writeFileSync, chmodSync, existsSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join, resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { spawnSync } from 'node:child_process'; - -const here = dirname(fileURLToPath(import.meta.url)); -const evalsDir = resolve(here, '..'); -const jumboRoot = resolve(evalsDir, '..'); -const jumboEntry = join(jumboRoot, 'dist', 'cli.js'); - -if (!existsSync(jumboEntry)) { - console.error( - `[with-jumbo-source] jumbo build not found at ${jumboEntry}\n` + - `Build it first:\n` + - ` npm --prefix "${jumboRoot}" ci\n` + - ` npm --prefix "${jumboRoot}" run build`, - ); - process.exit(1); -} - -const args = process.argv.slice(2); -if (args.length === 0) { - console.error('[with-jumbo-source] usage: node scripts/with-jumbo-source.mjs <command> [args...]'); - process.exit(2); -} - -const isWin = process.platform === 'win32'; -const binDir = mkdtempSync(join(tmpdir(), 'jumbo-src-bin-')); - -// POSIX shim (used when LocalExecutor spawns with shell:false) -writeFileSync(join(binDir, 'jumbo'), `#!/bin/sh\nexec node "${jumboEntry}" "$@"\n`); -chmodSync(join(binDir, 'jumbo'), 0o755); - -// Windows shim (used when LocalExecutor spawns with shell:true via cmd.exe) -writeFileSync(join(binDir, 'jumbo.cmd'), `@echo off\r\nnode "${jumboEntry}" %*\r\n`); - -const env = { ...process.env, PATH: binDir + (isWin ? ';' : ':') + process.env.PATH }; -const result = spawnSync(args[0], args.slice(1), { stdio: 'inherit', env, shell: isWin }); - -if (result.error) { - console.error(`[with-jumbo-source] failed to run: ${args.join(' ')}`); - console.error(result.error); - process.exit(1); -} -process.exit(result.status ?? 1); diff --git a/evals/src/ab-runner.ts b/evals/src/ab-runner.ts deleted file mode 100644 index 334eae20..00000000 --- a/evals/src/ab-runner.ts +++ /dev/null @@ -1,1395 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import type { TestScenario, ComparisonResult, SessionRecord, PerSessionScore, DimensionScore, JumboLifecycleAudit, JumboLifecycleCommandResult, JumboLifecycleEvidence, JumboMemoryCommandResult, JumboMemoryEntity, JumboMemoryKind, JumboMemorySnapshot, SessionHeartbeat, SessionPhaseTimings, TimingSpan, TamperEvent, RunControlFile, JumboPlan, JumboPlanEntry, JumboPlanGoal } from './domain/types.js'; -import { createTestResult, createComparisonResult } from './domain/types.js'; -import type { ResultStore } from './storage/result-store.js'; -import { LocalExecutor } from './infrastructure/local-executor.js'; -import type { HeartbeatWriter } from './infrastructure/heartbeat-writer.js'; -import { buildHeartbeatUpdate } from './infrastructure/heartbeat-writer.js'; -import type { ExecResult } from './infrastructure/container-manager.js'; -import type { HarnessAdapter } from './harness/harness-adapter.js'; -import { runSession } from './run-session.js'; -import { scoreFileAccuracy, producedAllExpectedFiles } from './scoring/file-accuracy-scorer.js'; -import { scoreKnowledgeRetention, scoreKnowledgeRetentionTimeline } from './scoring/knowledge-retention-scorer.js'; -import { scoreStructuralAssertions, scoreStructuralAssertionsTimeline } from './scoring/structural-assertion-scorer.js'; -import { scoreDisruptionRecovery, scoreDisruptionRecoveryTimeline } from './scoring/disruption-recovery-scorer.js'; -import { baselineJumboMemoryCaptureScore, scoreJumboMemoryCapture, scoreJumboMemoryCaptureTimeline } from './scoring/jumbo-memory-capture-scorer.js'; -import { baselineJumboEventCaptureScore, scoreJumboEventCapture, scoreJumboEventCaptureTimeline } from './scoring/jumbo-event-capture-scorer.js'; -import { adherenceForSession, baselineProtocolAdherenceScore, scoreProtocolAdherence, scoreProtocolAdherenceTimeline } from './scoring/protocol-adherence-scorer.js'; -import { compareTokenEfficiency, tokenUsageTimeline } from './scoring/token-efficiency-scorer.js'; -import type { JudgeConfig, JudgeFn } from './scoring/llm-judge-scorer.js'; -import { scoreAllJudgeDimensions } from './scoring/llm-judge-scorer.js'; - -export interface ABRunConfig { - readonly scenario: TestScenario; - readonly adapter: HarnessAdapter; - readonly executor: LocalExecutor; - readonly store: ResultStore; - readonly judgeConfig?: JudgeConfig; - readonly judgeFn?: JudgeFn; - readonly runId?: string; - readonly heartbeatWriter?: HeartbeatWriter; -} - -interface RunningSpan { - readonly startedAt: string; - readonly startedHrtime: bigint; -} - -interface HeartbeatContext { - readonly runId: string; - readonly writer: HeartbeatWriter; -} - -export class JumboInitError extends Error { - constructor( - message: string, - readonly result: ExecResult, - ) { - super(message); - this.name = 'JumboInitError'; - } -} - -export class JumboReachabilityError extends Error { - constructor( - message: string, - readonly variant: 'jumbo' | 'baseline', - readonly result: ExecResult, - ) { - super(message); - this.name = 'JumboReachabilityError'; - } -} - -export class JumboBaselineLeakError extends Error { - constructor( - message: string, - readonly result: ExecResult, - ) { - super(message); - this.name = 'JumboBaselineLeakError'; - } -} - -export class JumboPlanSeedError extends Error { - constructor( - message: string, - readonly entry: JumboPlanEntry, - readonly result: ExecResult, - ) { - super(message); - this.name = 'JumboPlanSeedError'; - } -} - -export class JumboPlanGoalRegistrationError extends Error { - constructor( - message: string, - readonly goal: JumboPlanGoal, - readonly result: ExecResult, - ) { - super(message); - this.name = 'JumboPlanGoalRegistrationError'; - } -} - -export class TamperAbortError extends Error { - constructor( - message: string, - readonly variant: 'jumbo' | 'baseline', - readonly tamperedRecords: readonly SessionRecord[], - ) { - super(message); - this.name = 'TamperAbortError'; - } -} - -const PAUSE_POLL_INTERVAL_MS = 1000; - -function delay(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function assertJumboReachable( - workDir: string, - executor: LocalExecutor, -): Promise<void> { - const result = await executor.exec(workDir, ['jumbo', '--version']); - if (result.exitCode !== 0) { - throw new JumboReachabilityError( - `jumbo --version failed in jumbo workdir with exit code ${result.exitCode}. ` + - `The adapter cannot invoke jumbo — check PATH and per-adapter permission seeding. ` + - `stderr: ${result.stderr.trim() || '(empty)'}`, - 'jumbo', - result, - ); - } -} - -/** - * Probes the baseline workdir to assert jumbo is NOT reachable on PATH (after - * the baseline shim is installed). This enforces the parity invariant: the - * treatment under test is the whole Jumbo system, so the baseline agent must - * not have access to the jumbo binary. The probe result is also recorded as - * evidence so any leak is loud and explainable. - */ -async function assertJumboUnreachableFromBaseline( - workDir: string, - executor: LocalExecutor, - baselineEnv: Record<string, string>, -): Promise<ExecResult> { - const result = await executor.exec(workDir, ['jumbo', '--version'], { env: baselineEnv }); - if (result.exitCode === 0) { - throw new JumboBaselineLeakError( - `jumbo --version unexpectedly succeeded in the baseline workdir. ` + - `Baseline parity requires that the agent cannot reach the real jumbo ` + - `binary; the shim setup failed to shadow it. stdout: ${result.stdout.trim() || '(empty)'}`, - result, - ); - } - return result; -} - -function isEventRelevant( - event: TamperEvent, - harness: string, - variant: 'jumbo' | 'baseline', -): boolean { - if (event.action === 'inject-context') { - if (variant !== 'jumbo') return false; - if (event.variant !== undefined && event.variant !== 'jumbo') return false; - if (event.harness !== undefined && event.harness !== harness) return false; - return true; - } - if (event.harness !== undefined && event.harness !== harness) return false; - if (event.variant !== undefined && event.variant !== variant) return false; - return true; -} - -interface ControlPollResult { - readonly abortEvent: TamperEvent | null; - readonly newEvents: readonly TamperEvent[]; - readonly injectedContext?: string; -} - -async function pollAndApplyControl(params: { - store: ResultStore; - runId: string; - harness: string; - variant: 'jumbo' | 'baseline'; -}): Promise<ControlPollResult> { - const { store, runId, harness, variant } = params; - const collected: TamperEvent[] = []; - let injectedContext: string | undefined; - let abortEvent: TamperEvent | null = null; - - if (typeof store.readRunControl !== 'function' || typeof store.writeRunControl !== 'function') { - return { abortEvent: null, newEvents: [], injectedContext: undefined }; - } - - while (true) { - const control = await store.readRunControl(runId); - if (!control) { - return { abortEvent, newEvents: collected, injectedContext }; - } - - if (control.pendingActions.length > 0) { - const drained: TamperEvent[] = []; - for (const ev of control.pendingActions) { - if (!isEventRelevant(ev, harness, variant)) continue; - drained.push(ev); - if (ev.action === 'inject-context' && ev.payload) { - injectedContext = injectedContext === undefined - ? ev.payload - : `${injectedContext}\n\n${ev.payload}`; - } - if (ev.action === 'abort') { - abortEvent = ev; - } - } - collected.push(...drained); - const next: RunControlFile = { - runId: control.runId, - updatedAt: new Date().toISOString(), - pendingActions: [], - pauseRequested: control.pauseRequested, - abortRequested: control.abortRequested, - }; - await store.writeRunControl(runId, next); - } - - if (control.abortRequested) { - if (!abortEvent) { - abortEvent = { - occurredAt: new Date().toISOString(), - action: 'abort', - harness, - variant, - }; - collected.push(abortEvent); - } - return { abortEvent, newEvents: collected, injectedContext }; - } - - if (control.pauseRequested) { - await delay(PAUSE_POLL_INTERVAL_MS); - continue; - } - - return { abortEvent, newEvents: collected, injectedContext }; - } -} - -/** - * Returns the provider-neutral scenario prompt for a given session number. - * Session 1 gets the initial prompt; subsequent sessions get the continuation prompt. - * If a disruption is scheduled for this session, its content is prepended. - * Both variants (Jumbo and baseline) derive from this byte-identical prompt per session. - */ -export function getSessionPrompt(scenario: TestScenario, sessionNumber: number): string { - let prompt: string; - if (sessionNumber === 1) { - prompt = scenario.initialPrompt; - } else { - prompt = scenario.continuationPrompt ?? 'Continue working on the project. Review what has been done so far and proceed with the next steps.'; - } - - // Inject disruptions scheduled for this session - const disruptions = (scenario.disruptions ?? []).filter((d) => d.sessionNumber === sessionNumber); - if (disruptions.length > 0) { - const disruptionText = disruptions.map((d) => - `[${d.type.toUpperCase()}]: ${d.content}`, - ).join('\n\n'); - prompt = `${disruptionText}\n\n${prompt}`; - } - - return prompt; -} - -/** - * Composes the Jumbo arm's session prompt. The framework no longer injects - * `jumbo session start` stdout into the prompt — instead, the agent is - * instructed to drive the lifecycle itself. The composed prompt has three - * parts: (a) the scenario prompt for the session, (b) the goal-id the - * framework assigned for this session, and (c) the explicit lifecycle - * protocol the agent must follow. Pure function — no I/O. - */ -export function buildJumboLifecyclePrompt(params: { - scenarioPrompt: string; - activeGoalId: string; - injectedContext?: string; -}): string { - const protocol = [ - 'Jumbo lifecycle protocol — execute in order:', - '1. Run `jumbo session start` to load project orientation.', - `2. Run \`jumbo goal start --id ${params.activeGoalId}\` to load the active goal context.`, - '3. Capture decisions, components, invariants, dependencies, and relations via the corresponding `jumbo <kind> add` commands as you make them — not as a cleanup step at the end.', - `4. Track progress with \`jumbo goal update-progress --id ${params.activeGoalId} --task-description "<description>"\` as you complete sub-tasks.`, - `5. When implementation is complete and all success criteria are met, run \`jumbo goal submit --id ${params.activeGoalId}\`.`, - '6. Run `jumbo session end --focus "<focus>" --summary "<summary>"` to close the session.', - ].join('\n'); - - const sections: string[] = [ - `Active goal for this session: ${params.activeGoalId}`, - protocol, - ]; - if (params.injectedContext !== undefined) { - sections.push(`[OPERATOR-INJECTED CONTEXT]:\n${params.injectedContext}`); - } - sections.push('Scenario task for this session:', params.scenarioPrompt); - return sections.join('\n\n'); -} - -function startSpan(): RunningSpan { - return { - startedAt: new Date().toISOString(), - startedHrtime: process.hrtime.bigint(), - }; -} - -function completeSpan(span: RunningSpan): TimingSpan { - return { - startedAt: span.startedAt, - completedAt: new Date().toISOString(), - elapsedMs: Math.max(0.001, Number(process.hrtime.bigint() - span.startedHrtime) / 1_000_000), - }; -} - -async function emitHeartbeat(params: { - heartbeat?: HeartbeatContext; - scenario: TestScenario; - harness: string; - variant: 'jumbo' | 'baseline'; - session: SessionHeartbeat; -}): Promise<void> { - if (!params.heartbeat) return; - await params.heartbeat.writer.writeHeartbeat( - params.heartbeat.runId, - buildHeartbeatUpdate({ - runId: params.heartbeat.runId, - scenarioId: params.scenario.id, - harness: params.harness, - variant: params.variant, - session: params.session, - }), - ); -} - -const JUMBO_MEMORY_COMMANDS: ReadonlyArray<{ - readonly kind: JumboMemoryKind; - readonly command: readonly string[]; -}> = [ - { kind: 'decision', command: ['jumbo', 'decisions', 'list', '--format', 'json'] }, - { kind: 'guideline', command: ['jumbo', 'guidelines', 'list', '--format', 'json'] }, - { kind: 'invariant', command: ['jumbo', 'invariants', 'list', '--format', 'json'] }, - { kind: 'component', command: ['jumbo', 'components', 'list', '--format', 'json'] }, - { kind: 'relation', command: ['jumbo', 'relations', 'list', '--format', 'json'] }, - { kind: 'dependency', command: ['jumbo', 'dependencies', 'list', '--format', 'json'] }, -]; - -/** - * Default plan used when a scenario omits jumboPlan. Preserves today's - * behavior: no preSeededMemory, no progressive release, the agent picks - * from the (empty) backlog. Existing fixtures keep working. - */ -const DEFAULT_JUMBO_PLAN: JumboPlan = { goals: [] }; - -function buildPlanEntryCommand(entry: JumboPlanEntry): string[] { - switch (entry.kind) { - case 'decision': { - const cmd = ['jumbo', 'decision', 'add', '--title', entry.title, '--context', entry.context]; - if (entry.rationale !== undefined) cmd.push('--rationale', entry.rationale); - if (entry.consequences !== undefined) cmd.push('--consequences', entry.consequences); - for (const alt of entry.alternatives ?? []) cmd.push('--alternative', alt); - return cmd; - } - case 'component': - return [ - 'jumbo', 'component', 'add', - '--name', entry.name, - '--type', entry.type, - '--description', entry.description, - '--responsibility', entry.responsibility, - '--path', entry.path, - ]; - case 'invariant': { - const cmd = ['jumbo', 'invariant', 'add', '--title', entry.title, '--description', entry.description]; - if (entry.rationale !== undefined) cmd.push('--rationale', entry.rationale); - return cmd; - } - case 'dependency': { - const cmd = [ - 'jumbo', 'dependency', 'add', - '--name', entry.name, - '--ecosystem', entry.ecosystem, - '--package-name', entry.packageName, - ]; - if (entry.versionConstraint !== undefined) cmd.push('--version-constraint', entry.versionConstraint); - if (entry.endpoint !== undefined) cmd.push('--endpoint', entry.endpoint); - if (entry.contract !== undefined) cmd.push('--contract', entry.contract); - return cmd; - } - case 'relation': { - const cmd = [ - 'jumbo', 'relation', 'add', - '--from-type', entry.fromType, - '--from-id', entry.fromId, - '--to-type', entry.toType, - '--to-id', entry.toId, - '--type', entry.type, - '--description', entry.description, - ]; - if (entry.strength !== undefined) cmd.push('--strength', entry.strength); - return cmd; - } - } -} - -function buildGoalAddCommand(goal: JumboPlanGoal, prerequisiteGoalIds: readonly string[]): string[] { - const cmd = [ - 'jumbo', 'goal', 'add', - '--title', goal.title, - '--objective', goal.objective, - ]; - for (const c of goal.criteria) cmd.push('--criteria', c); - for (const s of goal.scopeIn ?? []) cmd.push('--scope-in', s); - for (const s of goal.scopeOut ?? []) cmd.push('--scope-out', s); - for (const p of prerequisiteGoalIds) cmd.push('--prerequisite-goals', p); - return cmd; -} - -const GOAL_ID_KEYS = ['goalId', 'id', 'uuid'] as const; - -function extractGoalId(stdout: string): string | undefined { - const trimmed = stdout.trim(); - if (trimmed.length === 0) return undefined; - // Try JSON object first; fall back to plain UUID-on-stdout. - try { - const parsed = JSON.parse(trimmed) as unknown; - const record = asRecord(parsed); - if (record) { - for (const key of GOAL_ID_KEYS) { - const value = record[key]; - if (typeof value === 'string' && value.length > 0) return value; - } - const nested = asRecord(record.goal); - if (nested) { - for (const key of GOAL_ID_KEYS) { - const value = nested[key]; - if (typeof value === 'string' && value.length > 0) return value; - } - } - } - } catch { - // Not JSON; jumbo may print just the id on stdout. - } - const uuidMatch = trimmed.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i); - return uuidMatch ? uuidMatch[0] : undefined; -} - -async function seedJumboPlanMemory(params: { - plan: JumboPlan; - workDir: string; - executor: LocalExecutor; -}): Promise<void> { - for (const entry of params.plan.preSeededMemory ?? []) { - const command = buildPlanEntryCommand(entry); - const result = await params.executor.exec(params.workDir, command); - if (result.exitCode !== 0) { - throw new JumboPlanSeedError( - `pre-seed ${entry.kind} failed with exit code ${result.exitCode}: ${result.stderr.trim() || '(empty)'}`, - entry, - result, - ); - } - } -} - -async function registerJumboPlanGoal(params: { - goal: JumboPlanGoal; - workDir: string; - executor: LocalExecutor; - refToId: Map<string, string>; -}): Promise<string> { - const prerequisiteIds: string[] = []; - for (const ref of params.goal.prerequisitePlanRefs ?? []) { - const id = params.refToId.get(ref); - if (!id) { - throw new Error( - `JumboPlan goal "${params.goal.title}" references unknown prerequisitePlanRef "${ref}"`, - ); - } - prerequisiteIds.push(id); - } - const command = buildGoalAddCommand(params.goal, prerequisiteIds); - const result = await params.executor.exec(params.workDir, command); - if (result.exitCode !== 0) { - throw new JumboPlanGoalRegistrationError( - `jumbo goal add failed for "${params.goal.title}" with exit code ${result.exitCode}: ${result.stderr.trim() || '(empty)'}`, - params.goal, - result, - ); - } - const goalId = extractGoalId(result.stdout); - if (!goalId) { - throw new JumboPlanGoalRegistrationError( - `jumbo goal add for "${params.goal.title}" succeeded but did not return a parseable goal id`, - params.goal, - result, - ); - } - if (params.goal.planRef) { - params.refToId.set(params.goal.planRef, goalId); - } - return goalId; -} - -function asRecord(value: unknown): Record<string, unknown> | null { - return typeof value === 'object' && value !== null ? value as Record<string, unknown> : null; -} - -function entityId(kind: JumboMemoryKind, value: unknown): string | undefined { - const record = asRecord(value); - if (!record) return undefined; - const candidate = record.id - ?? record[`${kind}Id`] - ?? record.uuid - ?? record.name - ?? record.title; - return typeof candidate === 'string' ? candidate : undefined; -} - -function entityText(value: unknown): string { - if (typeof value === 'string') return value; - return JSON.stringify(value); -} - -function listItems(kind: JumboMemoryKind, parsed: unknown): unknown[] { - if (Array.isArray(parsed)) return parsed; - const record = asRecord(parsed); - if (!record) return []; - - const pluralKey = `${kind}s`; - const candidates = [ - record.items, - record.data, - record.results, - record[pluralKey], - ]; - - for (const candidate of candidates) { - if (Array.isArray(candidate)) return candidate; - } - - return []; -} - -function parseJumboMemoryEntities(kind: JumboMemoryKind, stdout: string): JumboMemoryEntity[] { - if (stdout.trim().length === 0) return []; - - const parsed = JSON.parse(stdout) as unknown; - return listItems(kind, parsed).map((item) => ({ - kind, - id: entityId(kind, item), - text: entityText(item), - raw: item, - })); -} - -async function captureJumboMemorySnapshot(params: { - sessionNumber: number; - workDir: string; - executor: LocalExecutor; -}): Promise<JumboMemorySnapshot> { - const commandResults: JumboMemoryCommandResult[] = []; - const entities: JumboMemoryEntity[] = []; - - for (const spec of JUMBO_MEMORY_COMMANDS) { - const result = await params.executor.exec(params.workDir, [...spec.command]); - commandResults.push({ - kind: spec.kind, - command: spec.command, - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - }); - - if (result.exitCode === 0) { - entities.push(...parseJumboMemoryEntities(spec.kind, result.stdout)); - } - } - - return { - sessionNumber: params.sessionNumber, - capturedAt: new Date().toISOString(), - entities, - commands: commandResults, - }; -} - -function parseGoalStatus(stdout: string): string | undefined { - if (stdout.trim().length === 0) return undefined; - try { - const parsed = JSON.parse(stdout) as unknown; - const root = asRecord(parsed); - const goal = asRecord(root?.goal) ?? root; - const status = goal?.status; - return typeof status === 'string' ? status : undefined; - } catch { - return undefined; - } -} - -function parseGoalVersion(stdout: string): number | undefined { - if (stdout.trim().length === 0) return undefined; - try { - const parsed = JSON.parse(stdout) as unknown; - const root = asRecord(parsed); - const goal = asRecord(root?.goal) ?? root; - const v = goal?.version; - return typeof v === 'number' ? v : undefined; - } catch { - return undefined; - } -} - -function diffSnapshotEntities( - before: JumboMemorySnapshot | undefined, - after: JumboMemorySnapshot | undefined, -): JumboMemoryEntity[] { - if (!after) return []; - const keyOf = (entity: JumboMemoryEntity): string => - entity.id ? `${entity.kind}:id:${entity.id}` : `${entity.kind}:${entity.text.toLowerCase().replace(/\s+/g, ' ').trim()}`; - const beforeKeys = new Set<string>(); - for (const entity of before?.entities ?? []) beforeKeys.add(keyOf(entity)); - return after.entities.filter((entity) => !beforeKeys.has(keyOf(entity))); -} - -function parseSessionsCount(stdout: string): number { - if (stdout.trim().length === 0) return 0; - try { - const parsed = JSON.parse(stdout) as unknown; - if (Array.isArray(parsed)) return parsed.length; - const record = asRecord(parsed); - if (record) { - for (const key of ['sessions', 'items', 'data', 'results']) { - const value = record[key]; - if (Array.isArray(value)) return value.length; - } - } - return 0; - } catch { - return 0; - } -} - -const GOAL_POST_START_STATUSES = new Set([ - 'doing', 'paused', 'blocked', 'submitted', 'in-review', 'reviewed', 'rejected', - 'approved', 'codifying', 'done', 'closed', 'completed', -]); - -const GOAL_POST_SUBMIT_STATUSES = new Set([ - 'submitted', 'in-review', 'reviewed', 'approved', 'codifying', 'done', 'closed', 'completed', -]); - -function toCommandResult(command: readonly string[], result: ExecResult): JumboLifecycleCommandResult { - return { - command, - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - }; -} - -async function captureLifecycleSnapshot(params: { - workDir: string; - executor: LocalExecutor; - activeGoalId?: string; -}): Promise<{ - readonly goalStatus?: string; - readonly goalVersion?: number; - readonly sessionsTotal: number; - readonly sessionsEnded: number; - readonly goalShow?: JumboLifecycleCommandResult; - readonly sessionsList?: JumboLifecycleCommandResult; - readonly sessionsEndedList?: JumboLifecycleCommandResult; -}> { - let goalShow: JumboLifecycleCommandResult | undefined; - let goalStatus: string | undefined; - let goalVersion: number | undefined; - if (params.activeGoalId !== undefined) { - const command = ['jumbo', 'goal', 'show', '--id', params.activeGoalId, '--format', 'json']; - const result = await params.executor.exec(params.workDir, command); - goalShow = toCommandResult(command, result); - if (result.exitCode === 0) { - goalStatus = parseGoalStatus(result.stdout); - goalVersion = parseGoalVersion(result.stdout); - } - } - - const sessionsListCommand = ['jumbo', 'sessions', 'list', '--status', 'all', '--format', 'json']; - const sessionsListResult = await params.executor.exec(params.workDir, sessionsListCommand); - const sessionsList = toCommandResult(sessionsListCommand, sessionsListResult); - const sessionsTotal = sessionsListResult.exitCode === 0 - ? parseSessionsCount(sessionsListResult.stdout) - : 0; - - const sessionsEndedCommand = ['jumbo', 'sessions', 'list', '--status', 'ended', '--format', 'json']; - const sessionsEndedResult = await params.executor.exec(params.workDir, sessionsEndedCommand); - const sessionsEndedList = toCommandResult(sessionsEndedCommand, sessionsEndedResult); - const sessionsEnded = sessionsEndedResult.exitCode === 0 - ? parseSessionsCount(sessionsEndedResult.stdout) - : 0; - - return { goalStatus, goalVersion, sessionsTotal, sessionsEnded, goalShow, sessionsList, sessionsEndedList }; -} - -async function captureDecisionsListResult(params: { - workDir: string; - executor: LocalExecutor; -}): Promise<JumboLifecycleCommandResult> { - const command = ['jumbo', 'decisions', 'list', '--format', 'json']; - const result = await params.executor.exec(params.workDir, command); - return toCommandResult(command, result); -} - -async function performLifecycleAudit(params: { - workDir: string; - executor: LocalExecutor; - activeGoalId?: string; - pre: Awaited<ReturnType<typeof captureLifecycleSnapshot>>; - preMemorySnapshot?: JumboMemorySnapshot; - postMemorySnapshot?: JumboMemorySnapshot; -}): Promise<JumboLifecycleAudit> { - const post = await captureLifecycleSnapshot({ - workDir: params.workDir, - executor: params.executor, - activeGoalId: params.activeGoalId, - }); - const decisionsListAfter = await captureDecisionsListResult({ - workDir: params.workDir, - executor: params.executor, - }); - - const sessionsTotalDelta = post.sessionsTotal - params.pre.sessionsTotal; - const sessionsEndedDelta = post.sessionsEnded - params.pre.sessionsEnded; - const sessionStartExecuted = sessionsTotalDelta > 0 || sessionsEndedDelta > 0; - const sessionEndExecuted = sessionsEndedDelta > 0; - const goalStartExecuted = params.activeGoalId !== undefined - && post.goalStatus !== undefined - && GOAL_POST_START_STATUSES.has(post.goalStatus); - const goalSubmitExecuted = params.activeGoalId !== undefined - && post.goalStatus !== undefined - && GOAL_POST_SUBMIT_STATUSES.has(post.goalStatus); - - // In-session captures: entities present in the post snapshot but not in - // the pre snapshot. Pre-seeded memory is in both, so only entities the - // agent registered during this session window count. - const newEntities = diffSnapshotEntities(params.preMemorySnapshot, params.postMemorySnapshot); - const inSessionCapturesExecuted = newEntities.length > 0; - - // Progress updates: derived from the goal aggregate's version delta. The - // canonical lifecycle accounts for at most one mutation each from goal - // start (refined→doing) and goal submit (doing→submitted). Any version - // delta beyond that implies extra mutations on the goal — the most likely - // source being `jumbo goal update-progress`. This is a snapshot-format - // heuristic; precise progress-entry inspection requires a CLI surface - // that emits progress entries in the `goal show` JSON, which is out of - // scope here (snapshot-format contract guideline). - const versionDelta = - typeof post.goalVersion === 'number' && typeof params.pre.goalVersion === 'number' - ? post.goalVersion - params.pre.goalVersion - : undefined; - const baselineMutations = - (goalStartExecuted ? 1 : 0) + (goalSubmitExecuted ? 1 : 0); - const progressUpdatesExecuted = versionDelta !== undefined && versionDelta > baselineMutations; - - const evidence: JumboLifecycleEvidence = { - goalShowBefore: params.pre.goalShow, - goalShowAfter: post.goalShow, - sessionsListBefore: params.pre.sessionsList, - sessionsListAfter: post.sessionsList, - sessionsEndedListAfter: post.sessionsEndedList, - decisionsListAfter, - }; - - return { - sessionStartExecuted, - goalStartExecuted, - inSessionCapturesExecuted, - progressUpdatesExecuted, - goalSubmitExecuted, - sessionEndExecuted, - activeGoalId: params.activeGoalId, - goalStatusBefore: params.pre.goalStatus, - goalStatusAfter: post.goalStatus, - goalVersionBefore: params.pre.goalVersion, - goalVersionAfter: post.goalVersion, - sessionsTotalDelta, - sessionsEndedDelta, - newEntityCount: newEntities.length, - evidence, - }; -} - -/** - * Runs N sessions inside a single persistent working directory. - * - * For Jumbo runs the framework no longer issues `jumbo session start` or - * `jumbo session end` on the agent's behalf. Each Jumbo session prompt - * contains the scenario task, the framework-picked active goal-id, and an - * explicit lifecycle protocol that the agent executes itself. After the - * harness exec the framework verifies what the agent did via `jumbo goal - * show`, `jumbo sessions list`, and `jumbo decisions list` and records the - * result on the SessionRecord as JumboLifecycleAudit. - * - * When a jumboPlan is supplied (jumbo variant only), the framework registers - * any plan goals whose sessionAvailableFrom matches the current session - * boundary, then picks the active goal-id for the session and threads it - * into the prompt builder. The agent never picks from the backlog. - */ -async function runMultiSession(params: { - scenario: TestScenario; - workDir: string; - executor: LocalExecutor; - adapter: HarnessAdapter; - store: ResultStore; - jumboEnabled: boolean; - plan?: JumboPlan; - runId?: string; - heartbeatWriter?: HeartbeatWriter; - env?: Record<string, string | undefined>; -}): Promise<SessionRecord[]> { - const { scenario, workDir, executor, adapter, store, jumboEnabled } = params; - const variant = jumboEnabled ? 'jumbo' : 'baseline'; - const heartbeat = params.runId && params.heartbeatWriter - ? { runId: params.runId, writer: params.heartbeatWriter } - : undefined; - const records: SessionRecord[] = []; - let tainted = false; - // refToId maps plan-local goal/entity refs to real CLI-minted IDs so that - // later plan goals can declare prerequisites that resolve correctly. - const refToId = new Map<string, string>(); - let lastActiveGoalId: string | undefined; - - for (let sessionNum = 1; sessionNum <= scenario.sessionCount; sessionNum++) { - let pendingEventsForSession: readonly TamperEvent[] = []; - let injectedContextForSession: string | undefined; - if (params.runId) { - const poll = await pollAndApplyControl({ - store, - runId: params.runId, - harness: adapter.name, - variant, - }); - if (poll.newEvents.length > 0) tainted = true; - pendingEventsForSession = poll.newEvents; - injectedContextForSession = poll.injectedContext; - if (poll.abortEvent) { - const tamperedExisting = records.map((r, idx) => ({ - ...r, - tampered: true, - tamperLog: idx === records.length - 1 - ? [...r.tamperLog, ...poll.newEvents] - : r.tamperLog, - })); - await Promise.all(tamperedExisting.map((r) => store.saveSessionRecord(r))); - throw new TamperAbortError( - `Run aborted by operator at session ${sessionNum} boundary (variant=${variant})`, - variant, - tamperedExisting, - ); - } - } - const sessionStartedAt = new Date().toISOString(); - let phaseTimings: Partial<SessionPhaseTimings> = {}; - let activeGoalIdForSession: string | undefined; - - // Register plan goals scheduled for this session boundary, in plan order. - // Only the jumbo arm has a live Jumbo project; the baseline arm runs - // without any backlog (its scope is out for this goal). - if (jumboEnabled && params.plan) { - const dueGoals = params.plan.goals.filter((g) => g.sessionAvailableFrom === sessionNum); - for (const goal of dueGoals) { - const goalId = await registerJumboPlanGoal({ - goal, - workDir, - executor, - refToId, - }); - if (activeGoalIdForSession === undefined) { - activeGoalIdForSession = goalId; - } - } - } - if (activeGoalIdForSession === undefined) { - activeGoalIdForSession = lastActiveGoalId; - } else { - lastActiveGoalId = activeGoalIdForSession; - } - - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: sessionNum, - status: 'running', - startedAt: sessionStartedAt, - phase: 'harness-exec', - }, - }); - - // Pre-harness snapshots: capture Jumbo's own state of the active - // goal, session list, and project memory so the post-session audit - // can prove the agent (not the framework) drove the lifecycle this - // session, and the scorer can credit only entities the agent - // registered during this session window. - const preLifecycleSnapshot = jumboEnabled - ? await captureLifecycleSnapshot({ - workDir, - executor, - activeGoalId: activeGoalIdForSession, - }) - : undefined; - const preMemorySnapshot = jumboEnabled - ? await captureJumboMemorySnapshot({ - sessionNumber: sessionNum, - workDir, - executor, - }) - : undefined; - - const scenarioPrompt = getSessionPrompt(scenario, sessionNum); - const deliveredContextForSession = injectedContextForSession !== undefined - ? `[OPERATOR-INJECTED CONTEXT]:\n${injectedContextForSession}` - : undefined; - const effectivePrompt = jumboEnabled && activeGoalIdForSession !== undefined - ? buildJumboLifecyclePrompt({ - scenarioPrompt, - activeGoalId: activeGoalIdForSession, - injectedContext: injectedContextForSession, - }) - : scenarioPrompt; - - let record: SessionRecord; - try { - const preExecSnapshot = await executor.captureWorkspaceSnapshot(workDir); - const harnessSpan = startSpan(); - record = await runSession({ - scenario, - sessionNumber: sessionNum, - variant, - prompt: effectivePrompt, - scenarioPrompt, - deliveredContext: deliveredContextForSession, - workDir, - executor, - adapter, - store, - env: params.env, - }); - phaseTimings = { - ...phaseTimings, - harnessExec: completeSpan(harnessSpan), - }; - // Workspace-diff fallback: harness CLIs (Claude/Codex/Gemini) do not - // reliably surface a files_modified field in their JSON output. The - // pre/post workspace snapshot diff is ground truth for what the agent - // created or edited during the harness exec. - const diffFilesModified = record.workspaceSnapshot - ? LocalExecutor.diffWorkspaceSnapshots(preExecSnapshot, record.workspaceSnapshot) - : []; - const filesModified = record.filesModified.length > 0 - ? record.filesModified - : diffFilesModified; - record = { - ...record, - filesModified, - phaseTimings: phaseTimings as SessionPhaseTimings, - tampered: tainted, - tamperLog: pendingEventsForSession, - }; - await store.saveSessionRecord(record); - } catch (err: unknown) { - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: sessionNum, - status: 'failed', - startedAt: sessionStartedAt, - completedAt: new Date().toISOString(), - phase: 'harness-exec', - errorMessage: err instanceof Error ? err.message : String(err), - phaseTimings: phaseTimings.harnessExec ? phaseTimings as SessionPhaseTimings : undefined, - }, - }); - throw err; - } - - // For Jumbo runs, verify the agent actually executed the lifecycle - // protocol and capture the resulting memory snapshot. The audit reads - // Jumbo's own state (goal show / sessions list / decisions list) — it - // does not parse the agent transcript. - if (jumboEnabled) { - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: sessionNum, - status: 'running', - startedAt: sessionStartedAt, - phase: 'lifecycle-audit', - phaseTimings: phaseTimings as SessionPhaseTimings, - }, - }); - - const auditSpan = startSpan(); - // Post-harness memory snapshot captured first so the lifecycle audit - // can diff pre/post and derive in-session captures. - const postMemorySnapshot = await captureJumboMemorySnapshot({ - sessionNumber: sessionNum, - workDir, - executor, - }); - const audit = await performLifecycleAudit({ - workDir, - executor, - activeGoalId: activeGoalIdForSession, - pre: preLifecycleSnapshot!, - preMemorySnapshot, - postMemorySnapshot, - }); - const lifecycleAuditSpan = completeSpan(auditSpan); - phaseTimings = { - ...phaseTimings, - lifecycleAudit: lifecycleAuditSpan, - }; - record = { - ...record, - jumboLifecycleAudit: audit, - jumboMemorySnapshotBefore: preMemorySnapshot, - jumboMemorySnapshot: postMemorySnapshot, - phaseTimings: phaseTimings as SessionPhaseTimings, - }; - await store.saveSessionRecord(record); - } - - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: sessionNum, - status: 'completed', - startedAt: sessionStartedAt, - completedAt: record.completedAt, - phaseTimings: record.phaseTimings, - }, - }); - records.push(record); - } - - return records; -} - -/** - * Runs an A/B comparison across N sessions. The treatment under test is the - * whole Jumbo system — orchestration (CLI), memory (decisions/components/ - * invariants/relations/etc.), and the lifecycle protocol — not a memory-only - * A/B. - * - * Variant A (jumbo): `jumbo init` at setup; per session the agent receives a - * scenario prompt + active goal-id + lifecycle protocol and drives session - * start/end and goal start/submit itself. The framework verifies execution - * via a post-session lifecycle audit. - * Variant B (baseline): Jumbo not initialized — clean working directory and a - * PATH-shadowing shim so the real jumbo binary is unreachable. The agent - * receives only the scenario prompt, no Jumbo collaboration block. - * - * Parity invariants enforced here: - * (a) The scenario prompt is byte-identical between the two arms. - * (b) The Jumbo collaboration block (lifecycle protocol + active goal-id + - * operator-context wrapper) exists in exactly one arm — the jumbo arm. - * (c) The baseline arm cannot reach the real jumbo binary; any invocation - * resolves to the fail-loud shim and is recorded. - * - * Each variant gets its own temp working directory on the host. The directory - * persists across sessions — only the agent invocation resets. Both results - * must complete for a valid ComparisonResult. - */ -export async function runABComparison(config: ABRunConfig): Promise<ComparisonResult> { - const { scenario, adapter, executor, store, judgeConfig, judgeFn } = config; - const runId = config.runId ?? randomUUID(); - const heartbeatWriter = config.heartbeatWriter; - - const jumboWorkDir = await executor.createWorkDir(`jumbo-eval-jumbo-`); - const baselineWorkDir = await executor.createWorkDir(`jumbo-eval-baseline-`); - const heartbeat = heartbeatWriter ? { runId, writer: heartbeatWriter } : undefined; - let jumboRecords: SessionRecord[] = []; - let baselineRecords: SessionRecord[] = []; - let scoringStarted = false; - - try { - // Seed per-adapter permission/config artifacts in both arms so the - // agent CLI can shell out without interactive approval. Done before any - // session work so a misconfigured environment fails loudly at setup, - // not mid-run. - await adapter.seedToolPermissions(jumboWorkDir); - await adapter.seedToolPermissions(baselineWorkDir); - - // Baseline parity setup: install a PATH-shadowing shim so `jumbo` - // resolves to a fail-loud script in the baseline arm. The treatment - // under test is the whole Jumbo system, so the baseline agent must not - // be able to call the real jumbo binary. - const { env: baselineEnv } = await executor.installJumboShim(baselineWorkDir); - - // Probe both arms via the same exec channel the adapter will use during - // sessions. The jumbo arm must reach jumbo; the baseline arm must not. - // Fail before any session work begins (lifecycle-setup invariant). - await assertJumboReachable(jumboWorkDir, executor); - await assertJumboUnreachableFromBaseline(baselineWorkDir, executor, baselineEnv); - - // Initialize Jumbo in the Jumbo working directory - const initResult = await executor.exec(jumboWorkDir, [ - 'jumbo', - 'init', - '--purpose', - scenario.name, - '--non-interactive', - '--name', - scenario.name, - '--yolo', - ]); - if (initResult.exitCode !== 0) { - throw new JumboInitError( - `jumbo init failed for scenario "${scenario.name}" with exit code ${initResult.exitCode}`, - initResult, - ); - } - - // Seed plan-supplied preSeededMemory into the jumbo arm only, after - // init and before any session work begins. Failure here aborts before - // sessions run (lifecycle-setup invariant). - const plan = scenario.jumboPlan ?? DEFAULT_JUMBO_PLAN; - await seedJumboPlanMemory({ plan, workDir: jumboWorkDir, executor }); - - // Run N sessions in each working directory - jumboRecords = await runMultiSession({ - scenario, workDir: jumboWorkDir, executor, adapter, store, jumboEnabled: true, plan, runId, heartbeatWriter, - }); - - baselineRecords = await runMultiSession({ - scenario, workDir: baselineWorkDir, executor, adapter, store, jumboEnabled: false, runId, heartbeatWriter, - env: baselineEnv, - }); - - scoringStarted = true; - await Promise.all((['jumbo', 'baseline'] as const).map((variant) => - emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: scenario.sessionCount, - status: 'running', - phase: 'scoring', - phaseTimings: (variant === 'jumbo' ? jumboRecords : baselineRecords).at(-1)?.phaseTimings, - }, - }), - )); - - // Build TestResults - const jumboResult = createTestResult({ - id: randomUUID(), - scenarioId: scenario.id, - harness: adapter.name, - sessionRecords: jumboRecords, - }); - - const baselineResult = createTestResult({ - id: randomUUID(), - scenarioId: scenario.id, - harness: adapter.name, - sessionRecords: baselineRecords, - }); - - await store.saveTestResult(jumboResult); - await store.saveTestResult(baselineResult); - - // Score both runs — aggregate across all sessions - const expectedFiles = scenario.expectedFiles ?? []; - const retentionPatterns = scenario.retentionPatterns ?? []; - const structuralAssertions = scenario.structuralAssertions ?? []; - const disruptions = scenario.disruptions ?? []; - const expectedJumboMemoryCaptures = scenario.expectedJumboMemoryCaptures ?? []; - - const jumboCleanRecords = jumboRecords.filter((r) => !r.tampered); - const baselineCleanRecords = baselineRecords.filter((r) => !r.tampered); - - // Adherence-gating (GOAL.md Outcome 4 + Scoring "Adherence Rate"): the - // Jumbo arm's lifecycle adherence is computed and reported SEPARATELY from - // quality scores, "not averaged into the lift". A per-session record filter - // was rejected: scoring a quality dimension over only the adherent subset of - // Jumbo sessions (while baseline uses all sessions) is asymmetric and - // fabricates lift — e.g. an empty adherent set makes knowledge-retention - // return its trivially-satisfied 1.0, inventing a +1 delta against a - // baseline that did nothing. So quality scores stay symmetric across arms, - // and the adherence rate is surfaced as the diagnostic that explains a low - // lift (non-adherence) vs. a real null result (memory didn't help). - const MIN_SESSION_ADHERENCE = 0.5; - const jumboAdherentSessions = jumboCleanRecords.filter( - (r) => adherenceForSession(r).score >= MIN_SESSION_ADHERENCE, - ).length; - const jumboAdherenceRate = jumboCleanRecords.length === 0 - ? 0 - : jumboAdherentSessions / jumboCleanRecords.length; - - const jumboFileScore = scoreFileAccuracy(jumboCleanRecords, expectedFiles); - const jumboRetentionScore = scoreKnowledgeRetention(jumboCleanRecords, retentionPatterns); - const jumboStructuralScore = scoreStructuralAssertions(jumboCleanRecords, structuralAssertions); - const jumboDisruptionScore = scoreDisruptionRecovery(jumboCleanRecords, disruptions); - const jumboMemoryScore = scoreJumboMemoryCapture(jumboCleanRecords, expectedJumboMemoryCaptures); - const jumboEventCaptureScore = scoreJumboEventCapture(jumboCleanRecords, expectedJumboMemoryCaptures); - const baseProtocolScore = scoreProtocolAdherence(jumboCleanRecords); - const jumboProtocolScore = { - ...baseProtocolScore, - details: `${baseProtocolScore.details}; adherence-rate=${jumboAdherenceRate.toFixed(2)} (adherent-sessions=${jumboAdherentSessions}/${jumboCleanRecords.length}, threshold=${MIN_SESSION_ADHERENCE})`, - }; - - const baselineFileScore = scoreFileAccuracy(baselineCleanRecords, expectedFiles); - const baselineRetentionScore = scoreKnowledgeRetention(baselineCleanRecords, retentionPatterns); - const baselineStructuralScore = scoreStructuralAssertions(baselineCleanRecords, structuralAssertions); - const baselineDisruptionScore = scoreDisruptionRecovery(baselineCleanRecords, disruptions); - const baselineMemoryScore = baselineJumboMemoryCaptureScore(expectedJumboMemoryCaptures); - const baselineEventCaptureScore = baselineJumboEventCaptureScore(expectedJumboMemoryCaptures); - const baselineProtocolScore = baselineProtocolAdherenceScore(); - - // Token efficiency is normalised by the structural-assertion score (GOAL.md - // Scoring) and only reported when both arms produced functionally - // EQUIVALENT outputs — both produced every expected file AND both reached - // >=0.8 structural-assertion score. Otherwise the ratio compares the cost of - // producing different things, so it is reported as N/A. Equivalence uses - // ungated artefacts (independent of adherence). - const MIN_STRUCTURAL_FOR_EQUIVALENCE = 0.8; - const jumboProducedAll = producedAllExpectedFiles(jumboCleanRecords, expectedFiles); - const baselineProducedAll = producedAllExpectedFiles(baselineCleanRecords, expectedFiles); - const outputsEquivalent = - jumboProducedAll && - baselineProducedAll && - jumboStructuralScore.score >= MIN_STRUCTURAL_FOR_EQUIVALENCE && - baselineStructuralScore.score >= MIN_STRUCTURAL_FOR_EQUIVALENCE; - const tokenEfficiencyScore = outputsEquivalent - ? compareTokenEfficiency(jumboCleanRecords, baselineCleanRecords, jumboStructuralScore.score, baselineStructuralScore.score) - : { - dimension: 'token-efficiency', - score: 0, - maxScore: 0, - details: `N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=${jumboStructuralScore.score.toFixed(2)}, baseline=${baselineStructuralScore.score.toFixed(2)}, >=${MIN_STRUCTURAL_FOR_EQUIVALENCE} required; all expected files produced: jumbo=${jumboProducedAll}, baseline=${baselineProducedAll}).`, - }; - - const jumboScores = [jumboFileScore, jumboRetentionScore, jumboStructuralScore, jumboDisruptionScore, jumboMemoryScore, jumboEventCaptureScore, jumboProtocolScore, tokenEfficiencyScore]; - const baselineScores = [baselineFileScore, baselineRetentionScore, baselineStructuralScore, baselineDisruptionScore, baselineMemoryScore, baselineEventCaptureScore, baselineProtocolScore, tokenEfficiencyScore]; - - // LLM-judge scoring (optional — requires both judgeConfig and judgeFn) - if (judgeConfig && judgeFn) { - const jumboJudgeScores = await scoreAllJudgeDimensions(jumboCleanRecords, judgeConfig, judgeFn); - const baselineJudgeScores = await scoreAllJudgeDimensions(baselineCleanRecords, judgeConfig, judgeFn); - jumboScores.push(...jumboJudgeScores); - baselineScores.push(...baselineJudgeScores); - } - - // Compute deltas - const deltas = jumboScores.map((js, i) => ({ - dimension: js.dimension, - score: Math.round((js.score - baselineScores[i].score) * 100) / 100, - maxScore: js.maxScore, - details: `jumbo=${js.score.toFixed(2)} baseline=${baselineScores[i].score.toFixed(2)}`, - })); - - // Build per-session timelines - const jumboRetentionTimeline = scoreKnowledgeRetentionTimeline(jumboRecords, retentionPatterns); - const baselineRetentionTimeline = scoreKnowledgeRetentionTimeline(baselineRecords, retentionPatterns); - // Structural-retention timelines are sparse — one entry per session that - // has assertions due, in ascending session order (the scorer's contract). - // Re-key them by session number so they attach to the right session below. - const dueStructuralSessions = [...new Set(structuralAssertions.map((a) => a.sessionNumber))].sort((a, b) => a - b); - const jumboStructuralTimeline = scoreStructuralAssertionsTimeline(jumboRecords, structuralAssertions); - const baselineStructuralTimeline = scoreStructuralAssertionsTimeline(baselineRecords, structuralAssertions); - const jumboStructuralBySession = new Map<number, DimensionScore>(); - const baselineStructuralBySession = new Map<number, DimensionScore>(); - dueStructuralSessions.forEach((sessionNumber, idx) => { - if (jumboStructuralTimeline[idx]) jumboStructuralBySession.set(sessionNumber, jumboStructuralTimeline[idx]); - if (baselineStructuralTimeline[idx]) baselineStructuralBySession.set(sessionNumber, baselineStructuralTimeline[idx]); - }); - const jumboDisruptionTimeline = scoreDisruptionRecoveryTimeline(jumboRecords, disruptions); - const baselineDisruptionTimeline = scoreDisruptionRecoveryTimeline(baselineRecords, disruptions); - const jumboTokenTimeline = tokenUsageTimeline(jumboRecords); - const baselineTokenTimeline = tokenUsageTimeline(baselineRecords); - const jumboMemoryTimeline = scoreJumboMemoryCaptureTimeline(jumboRecords, expectedJumboMemoryCaptures); - const baselineMemoryTimeline = baselineRecords.map(() => baselineMemoryScore); - const jumboEventCaptureTimeline = scoreJumboEventCaptureTimeline(jumboRecords, expectedJumboMemoryCaptures); - const baselineEventCaptureTimeline = baselineRecords.map(() => baselineEventCaptureScore); - const jumboProtocolTimeline = scoreProtocolAdherenceTimeline(jumboRecords); - const baselineProtocolTimeline = baselineRecords.map(() => baselineProtocolScore); - - const jumboTimeline: PerSessionScore[] = jumboRecords.map((r, i) => ({ - sessionNumber: r.sessionNumber, - scores: [ - scoreFileAccuracy([r], expectedFiles), - ...(jumboRetentionTimeline[i] ? [jumboRetentionTimeline[i]] : []), - ...(jumboStructuralBySession.has(r.sessionNumber) ? [jumboStructuralBySession.get(r.sessionNumber)!] : []), - ...(jumboDisruptionTimeline[i] ? [jumboDisruptionTimeline[i]] : []), - ...(jumboMemoryTimeline[i] ? [jumboMemoryTimeline[i]] : []), - ...(jumboEventCaptureTimeline[i] ? [jumboEventCaptureTimeline[i]] : []), - ...(jumboProtocolTimeline[i] ? [jumboProtocolTimeline[i]] : []), - ...(jumboTokenTimeline[i] ? [jumboTokenTimeline[i]] : []), - ], - })); - - const baselineTimeline: PerSessionScore[] = baselineRecords.map((r, i) => ({ - sessionNumber: r.sessionNumber, - scores: [ - scoreFileAccuracy([r], expectedFiles), - ...(baselineRetentionTimeline[i] ? [baselineRetentionTimeline[i]] : []), - ...(baselineStructuralBySession.has(r.sessionNumber) ? [baselineStructuralBySession.get(r.sessionNumber)!] : []), - ...(baselineDisruptionTimeline[i] ? [baselineDisruptionTimeline[i]] : []), - ...(baselineMemoryTimeline[i] ? [baselineMemoryTimeline[i]] : []), - ...(baselineEventCaptureTimeline[i] ? [baselineEventCaptureTimeline[i]] : []), - ...(baselineProtocolTimeline[i] ? [baselineProtocolTimeline[i]] : []), - ...(baselineTokenTimeline[i] ? [baselineTokenTimeline[i]] : []), - ], - })); - - const comparison = createComparisonResult({ - id: randomUUID(), - scenarioId: scenario.id, - harness: adapter.name, - jumboResult, - baselineResult, - jumboScores, - baselineScores, - deltas, - jumboTimeline, - baselineTimeline, - }); - - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant: 'jumbo', - session: { - sessionNumber: scenario.sessionCount, - status: 'completed', - completedAt: new Date().toISOString(), - phaseTimings: jumboRecords.at(-1)?.phaseTimings, - }, - }); - await emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant: 'baseline', - session: { - sessionNumber: scenario.sessionCount, - status: 'completed', - completedAt: new Date().toISOString(), - phaseTimings: baselineRecords.at(-1)?.phaseTimings, - }, - }); - - return comparison; - } catch (err: unknown) { - if (scoringStarted) { - await Promise.all((['jumbo', 'baseline'] as const).map((variant) => - emitHeartbeat({ - heartbeat, - scenario, - harness: adapter.name, - variant, - session: { - sessionNumber: scenario.sessionCount, - status: 'failed', - completedAt: new Date().toISOString(), - phase: 'scoring', - phaseTimings: (variant === 'jumbo' ? jumboRecords : baselineRecords).at(-1)?.phaseTimings, - errorMessage: err instanceof Error ? err.message : String(err), - }, - }), - )); - } - throw err; - } finally { - await executor.cleanup(jumboWorkDir); - await executor.cleanup(baselineWorkDir); - } -} diff --git a/evals/src/analysis/replication-stats.ts b/evals/src/analysis/replication-stats.ts deleted file mode 100644 index 4eb197a9..00000000 --- a/evals/src/analysis/replication-stats.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Aggregates K replicated A/B comparisons of the same scenario and harness into - * per-dimension lift statistics (GOAL.md Outcome 5). Pure and deterministic — - * no I/O, no clock dependence beyond the report timestamp. - * - * Lift is reported as mean ± sample standard deviation across replications, and - * is only flagged as a signal when the absolute mean lift exceeds one SD. A - * t-statistic is reported alongside so a caller can compare it to the K=5, - * one-tailed α=0.05 critical value (t > 2.13, df=4). - */ -import type { ComparisonResult, DimensionScore } from '../domain/result.js'; -import type { DimensionLiftStat, ReplicationReport, ReplicationSignificance } from '../domain/replication.js'; - -/** One-tailed α=0.05 critical t by degrees of freedom (df = n − 1). */ -const T_CRITICAL_ONE_TAILED_05: Readonly<Record<number, number>> = { - 1: 6.314, - 2: 2.920, - 3: 2.353, - 4: 2.132, - 5: 2.015, - 6: 1.943, - 7: 1.895, - 8: 1.860, - 9: 1.833, - 10: 1.812, - 15: 1.753, - 20: 1.725, - 30: 1.697, -}; - -function mean(xs: readonly number[]): number { - return xs.length === 0 ? 0 : xs.reduce((sum, x) => sum + x, 0) / xs.length; -} - -function sampleStdDev(xs: readonly number[]): number { - if (xs.length < 2) return 0; - const m = mean(xs); - const variance = xs.reduce((sum, x) => sum + (x - m) ** 2, 0) / (xs.length - 1); - return Math.sqrt(variance); -} - -function scoreByDimension(scores: readonly DimensionScore[]): Map<string, DimensionScore> { - const map = new Map<string, DimensionScore>(); - for (const score of scores) map.set(score.dimension, score); - return map; -} - -/** Dimensions present in the jumboScores of every replication, in first-replication order. */ -function dimensionsInEveryReplication(comparisons: readonly ComparisonResult[]): string[] { - if (comparisons.length === 0) return []; - const [first, ...rest] = comparisons; - let common = new Set(first.jumboScores.map((s) => s.dimension)); - for (const comparison of rest) { - const here = new Set(comparison.jumboScores.map((s) => s.dimension)); - common = new Set([...common].filter((d) => here.has(d))); - } - return first.jumboScores.map((s) => s.dimension).filter((d) => common.has(d)); -} - -export function aggregateReplications(comparisons: readonly ComparisonResult[]): ReplicationReport { - const k = comparisons.length; - const createdAt = new Date().toISOString(); - const significance: ReplicationSignificance = { - rule: 'isSignal = |meanLift| > sdLift', - tCriticalOneTailed05: T_CRITICAL_ONE_TAILED_05[k - 1] ?? null, - note: `Lift is a signal only when |meanLift| exceeds one SD. For K=5 (df=4) the one-tailed alpha=0.05 t-threshold is 2.13; current K=${k} (df=${Math.max(k - 1, 0)}).`, - }; - - if (k === 0) { - return { scenarioId: '', harness: '', k: 0, dimensions: [], significance, createdAt }; - } - - const jumboMaps = comparisons.map((c) => scoreByDimension(c.jumboScores)); - const baselineMaps = comparisons.map((c) => scoreByDimension(c.baselineScores)); - - const dimensions: DimensionLiftStat[] = dimensionsInEveryReplication(comparisons).map((dimension) => { - const jumboVals: number[] = []; - const baselineVals: number[] = []; - const lifts: number[] = []; - for (let i = 0; i < k; i++) { - const jumbo = jumboMaps[i].get(dimension); - const baseline = baselineMaps[i].get(dimension); - if (!jumbo || !baseline) continue; - // N/A markers (e.g. token-efficiency without output-equivalence) carry - // maxScore 0 and are excluded from this dimension's statistics. - if (jumbo.maxScore === 0) continue; - jumboVals.push(jumbo.score); - baselineVals.push(baseline.score); - lifts.push(jumbo.score - baseline.score); - } - - const applicable = lifts.length; - const meanLift = mean(lifts); - const sdLift = sampleStdDev(lifts); - const tStatistic = sdLift > 0 && applicable >= 2 ? meanLift / (sdLift / Math.sqrt(applicable)) : 0; - const isSignal = applicable >= 2 && Math.abs(meanLift) > sdLift; - - return { - dimension, - k, - applicableReplications: applicable, - meanJumbo: mean(jumboVals), - meanBaseline: mean(baselineVals), - meanLift, - sdLift, - tStatistic, - isSignal, - }; - }); - - return { - scenarioId: comparisons[0].scenarioId, - harness: comparisons[0].harness, - k, - dimensions, - significance, - createdAt, - }; -} diff --git a/evals/src/cli/commands/control.ts b/evals/src/cli/commands/control.ts deleted file mode 100644 index bd5f4c33..00000000 --- a/evals/src/cli/commands/control.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Command } from 'commander'; -import type { RunControlFile, TamperAction, TamperEvent } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; - -export interface ControlDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -const VALID_ACTIONS = ['pause', 'resume', 'abort', 'inject-context'] as const; - -export function registerControlCommand(program: Command, deps: ControlDeps): void { - program - .command('control <runId> <action> [payload]') - .description('Send a control action (pause, resume, abort, inject-context) to a live run') - .option('--allow-tampering', 'Required acknowledgement that this mutates the run and quarantines its records from aggregate lift calculations') - .option('--operator <name>', 'Operator identifier recorded on the TamperEvent') - .option('--harness <name>', 'Restrict the action to a single harness') - .option('--variant <variant>', 'Restrict the action to a single variant (jumbo|baseline)') - .addHelpText('after', ` -Examples: - eval control <runId> pause --allow-tampering - eval control <runId> resume --allow-tampering - eval control <runId> abort --allow-tampering --operator alice - eval control <runId> inject-context "Remember to test edge case X" --allow-tampering - `) - .action(async ( - runId: string, - action: string, - payload: string | undefined, - opts: { - allowTampering?: boolean; - operator?: string; - harness?: string; - variant?: string; - }, - ) => { - if (!VALID_ACTIONS.includes(action as TamperAction)) { - throw new Error(`Unknown control action: ${action}. Valid: ${VALID_ACTIONS.join(', ')}`); - } - if (!opts.allowTampering) { - throw new Error( - `Refusing to send '${action}' without --allow-tampering. Mutating actions taint the run and exclude it from aggregate lift by default.`, - ); - } - const tamperAction = action as TamperAction; - if (tamperAction === 'inject-context') { - if (!payload || payload.length === 0) { - throw new Error('inject-context requires a payload string'); - } - if (opts.variant && opts.variant !== 'jumbo') { - throw new Error('inject-context is only valid for the jumbo variant'); - } - } - if (opts.variant && opts.variant !== 'jumbo' && opts.variant !== 'baseline') { - throw new Error(`Unknown variant: ${opts.variant}. Must be 'jumbo' or 'baseline'.`); - } - - const store = await deps.storeProvider(); - if (!store.readRunControl || !store.writeRunControl) { - throw new Error('Configured store does not support run control files'); - } - - const event: TamperEvent = { - occurredAt: new Date().toISOString(), - action: tamperAction, - ...(opts.harness !== undefined ? { harness: opts.harness } : {}), - ...(opts.variant !== undefined - ? { variant: opts.variant as 'jumbo' | 'baseline' } - : tamperAction === 'inject-context' ? { variant: 'jumbo' as const } : {}), - ...(payload !== undefined ? { payload } : {}), - ...(opts.operator !== undefined ? { operator: opts.operator } : {}), - }; - - const existing = await store.readRunControl(runId); - const next: RunControlFile = { - runId, - updatedAt: new Date().toISOString(), - pendingActions: [...(existing?.pendingActions ?? []), event], - pauseRequested: tamperAction === 'pause' - ? true - : tamperAction === 'resume' - ? false - : (existing?.pauseRequested ?? false), - abortRequested: tamperAction === 'abort' - ? true - : (existing?.abortRequested ?? false), - }; - await store.writeRunControl(runId, next); - console.log(`Recorded ${tamperAction} for run ${runId}.`); - }); -} diff --git a/evals/src/cli/commands/index.ts b/evals/src/cli/commands/index.ts deleted file mode 100644 index e78cadd7..00000000 --- a/evals/src/cli/commands/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export { registerScenarioCommands, handleScenarioCreate } from './scenario-create.js'; -export { registerScenarioListCommand, formatScenarioList } from './scenario-list.js'; -export { registerRunCommand, validateHarnesses } from './run.js'; -export { registerScoreCommand, formatScoreOutput } from './score.js'; -export { registerReportCommand, filterReportByDimensions, filterComparisonsByHarness } from './report.js'; -export { registerStatusCommand, formatStatusOutput } from './status.js'; diff --git a/evals/src/cli/commands/report.ts b/evals/src/cli/commands/report.ts deleted file mode 100644 index ec3fa7cd..00000000 --- a/evals/src/cli/commands/report.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { Command } from 'commander'; -import type { ComparisonResult, TestResult } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; -import { formatCrossHarnessComparison } from '../../output/cross-harness-display.js'; -import { exportReportAsJson } from '../../output/json-report.js'; -import { formatFullReport, generateFullReport, type FullReport } from '../../output/report-generator.js'; - -const VALID_HARNESSES = ['claude-code', 'codex-cli', 'gemini-cli'] as const; -const VALID_DIMENSIONS = [ - 'file-accuracy', - 'knowledge-retention', - 'disruption-recovery', - 'token-efficiency', - 'jumbo-memory-capture', -] as const; - -export interface ReportDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -interface ReportCommandOptions { - readonly scenario: string; - readonly harness?: readonly string[]; - readonly dimension?: readonly string[]; - readonly json?: boolean; - readonly includeTampered?: boolean; -} - -type TestResultWithComparison = TestResult & { - readonly comparisonResult?: ComparisonResult; -}; - -/** - * Registers the 'report' command. - * Generates lift reports with optional harness and dimension filters. - */ -export function registerReportCommand(program: Command, deps: ReportDeps): void { - program - .command('report') - .description('Generate lift reports from completed comparison runs') - .requiredOption('--scenario <id>', 'Scenario ID to report on') - .option('--harness <harnesses...>', 'Filter by harness(es)') - .option('--dimension <dimensions...>', 'Filter by dimension(s)') - .option('--json', 'Output as JSON for external consumption') - .option('--include-tampered', 'Include tampered ComparisonResults in cross-harness aggregates and lift summaries (excluded by default)') - .addHelpText('after', ` -Examples: - eval report --scenario abc123 - eval report --scenario abc123 --harness claude-code - eval report --scenario abc123 --dimension file-accuracy knowledge-retention - eval report --scenario abc123 --json - eval report --scenario abc123 --include-tampered - `) - .action(async (opts: ReportCommandOptions) => { - const harnesses = validateReportHarnesses(opts.harness ?? []); - const dimensions = validateReportDimensions(opts.dimension ?? []); - const includeTampered = opts.includeTampered ?? false; - - const store = await deps.storeProvider(); - const scenario = await store.getScenario(opts.scenario); - const allComparisons = filterComparisonsByHarness( - extractComparisonResults(await store.listTestResults(opts.scenario)), - harnesses, - ); - - if (allComparisons.length === 0) { - console.log(`No comparisons found for scenario: ${opts.scenario}`); - return; - } - - const tamperedExcluded = includeTampered ? [] : allComparisons.filter((c) => c.tampered); - const aggregateComparisons = includeTampered - ? allComparisons - : allComparisons.filter((c) => !c.tampered); - - const report = filterReportByDimensions( - generateFullReport(aggregateComparisons, scenario?.disruptions ?? [], tamperedExcluded), - dimensions, - ); - - if (opts.json) { - console.log(exportReportAsJson(report)); - return; - } - - console.log(formatTerminalReport(aggregateComparisons, report, dimensions)); - if (tamperedExcluded.length > 0) { - console.log(`\n[NOTICE] ${tamperedExcluded.length} tampered comparison(s) excluded from aggregates. Use --include-tampered to include them.`); - for (const c of tamperedExcluded) { - console.log(` - ${c.harness} (id=${c.id}): ${c.tamperLog.length} tamper event(s)`); - } - } - }); -} - -export function validateReportHarnesses(harnesses: readonly string[]): string[] { - const invalid = harnesses.filter((h) => !VALID_HARNESSES.includes(h as typeof VALID_HARNESSES[number])); - if (invalid.length > 0) { - throw new Error( - `Unknown harness(es): ${invalid.join(', ')}. Valid harnesses: ${VALID_HARNESSES.join(', ')}`, - ); - } - return [...harnesses]; -} - -export function validateReportDimensions(dimensions: readonly string[]): string[] { - const invalid = dimensions.filter((d) => !VALID_DIMENSIONS.includes(d as typeof VALID_DIMENSIONS[number])); - if (invalid.length > 0) { - throw new Error( - `Unknown dimension(s): ${invalid.join(', ')}. Valid dimensions: ${VALID_DIMENSIONS.join(', ')}`, - ); - } - return [...dimensions]; -} - -export function extractComparisonResults(results: readonly TestResult[]): ComparisonResult[] { - const byId = new Map<string, ComparisonResult>(); - for (const result of results as readonly TestResultWithComparison[]) { - if (result.comparisonResult) { - byId.set(result.comparisonResult.id, result.comparisonResult); - } - } - return [...byId.values()]; -} - -function formatTerminalReport( - comparisons: readonly ComparisonResult[], - report: FullReport, - dimensions: readonly string[], -): string { - const displayComparisons = dimensions.length > 0 - ? comparisons.map((comparison) => filterComparisonByDimensions(comparison, dimensions)) - : comparisons; - - if (comparisons.length > 1) { - return `${formatCrossHarnessComparison(displayComparisons)}\n\n${formatFullReport(report)}`; - } - return formatFullReport(report); -} - -function filterComparisonByDimensions( - comparison: ComparisonResult, - dimensions: readonly string[], -): ComparisonResult { - const dimSet = new Set(dimensions); - return { - ...comparison, - jumboScores: comparison.jumboScores.filter((score) => dimSet.has(score.dimension)), - baselineScores: comparison.baselineScores.filter((score) => dimSet.has(score.dimension)), - deltas: comparison.deltas.filter((score) => dimSet.has(score.dimension)), - jumboTimeline: comparison.jumboTimeline?.map((session) => ({ - ...session, - scores: session.scores.filter((score) => dimSet.has(score.dimension)), - })), - baselineTimeline: comparison.baselineTimeline?.map((session) => ({ - ...session, - scores: session.scores.filter((score) => dimSet.has(score.dimension)), - })), - }; -} - -/** - * Filters a report's lift results by dimension names. - * Pure function — no I/O. - */ -export function filterReportByDimensions( - report: FullReport, - dimensions: readonly string[], -): FullReport { - if (dimensions.length === 0) return report; - - const dimSet = new Set(dimensions); - - return { - ...report, - liftResults: report.liftResults.filter((l) => dimSet.has(l.dimension)), - divergenceCurve: report.divergenceCurve.filter((p) => dimSet.has(p.dimension)), - divergenceOnsets: report.divergenceOnsets.filter((o) => dimSet.has(o.dimension)), - disruptionImpacts: report.disruptionImpacts.filter((i) => dimSet.has(i.dimension)), - memoryCaptureEvidence: dimSet.has('jumbo-memory-capture') ? report.memoryCaptureEvidence : [], - harnessAggregation: report.harnessAggregation.map((h) => ({ - ...h, - dimensionLifts: h.dimensionLifts.filter((l) => dimSet.has(l.dimension)), - })), - auditTrails: report.auditTrails.map((t) => ({ - ...t, - scoringEvidence: t.scoringEvidence.filter((e) => dimSet.has(e.dimension)), - })), - }; -} - -/** - * Filters comparisons by harness names. - * Pure function — no I/O. - */ -export function filterComparisonsByHarness( - comparisons: readonly ComparisonResult[], - harnesses: readonly string[], -): ComparisonResult[] { - if (harnesses.length === 0) return [...comparisons]; - const harnessSet = new Set(harnesses); - return comparisons.filter((c) => harnessSet.has(c.harness)); -} diff --git a/evals/src/cli/commands/run.ts b/evals/src/cli/commands/run.ts deleted file mode 100644 index 025b82b2..00000000 --- a/evals/src/cli/commands/run.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { Command } from 'commander'; -import type { ComparisonResult, EvalRunRecord, TestScenario } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; -import type { HarnessAdapter } from '../../harness/harness-adapter.js'; -import { runABComparison } from '../../ab-runner.js'; -import { LocalExecutor } from '../../infrastructure/local-executor.js'; -import type { HeartbeatWriter } from '../../infrastructure/heartbeat-writer.js'; -import { ClaudeCodeAdapter, CodexCliAdapter, GeminiCliAdapter } from '../../harness/index.js'; -import { formatComparisonOutput } from '../../output/comparison-display.js'; -import { formatCrossHarnessComparison } from '../../output/cross-harness-display.js'; -import { generateFullReport, formatFullReport } from '../../output/report-generator.js'; -import { formatReplicationReport } from '../../output/replication-display.js'; -import { aggregateReplications } from '../../analysis/replication-stats.js'; - -const VALID_HARNESSES = ['claude-code', 'codex-cli', 'gemini-cli'] as const; -type HarnessName = typeof VALID_HARNESSES[number]; - -export type ABRunner = typeof runABComparison; -export type HeartbeatWriterProvider = (params: { - readonly store: ResultStore; - readonly runId: string; - readonly scenarioId: string; - readonly harnesses: readonly string[]; - readonly sessionCount: number; -}) => Promise<(HeartbeatWriter & { initialize?: () => Promise<void> }) | undefined>; - -export interface RunDeps { - readonly storeProvider: () => Promise<ResultStore>; - readonly abRunner?: ABRunner; - readonly heartbeatWriterProvider?: HeartbeatWriterProvider; - readonly executorProvider?: () => LocalExecutor; - readonly adapterProvider?: (name: string) => HarnessAdapter; -} - -function buildAdapter(name: HarnessName): HarnessAdapter { - switch (name) { - case 'claude-code': - return new ClaudeCodeAdapter(); - case 'codex-cli': - return new CodexCliAdapter(); - case 'gemini-cli': - return new GeminiCliAdapter(); - } -} - -/** - * Registers the 'run' command. - * Executes a scenario against specified harnesses, producing A/B comparison runs. - */ -export function registerRunCommand(program: Command, deps: RunDeps): void { - program - .command('run') - .description('Execute a scenario against specified harnesses') - .requiredOption('--scenario <id>', 'Scenario ID to run') - .option('--harness <harnesses...>', 'Harness(es) to run against', ['claude-code']) - .option('--sessions <count>', 'Override scenario session count', parseInt) - .option('--replicate <count>', 'Run K replications per arm and report lift as mean +/- SD (Outcome 5)', (v) => parseInt(v, 10), 1) - .addHelpText('after', ` -Examples: - eval run --scenario abc123 - eval run --scenario abc123 --harness claude-code codex-cli - eval run --scenario abc123 --harness gemini-cli --sessions 5 - eval run --scenario abc123 --replicate 5 - `) - .action(async (opts: { scenario: string; harness: string[]; sessions?: number; replicate: number }) => { - const harnesses = validateHarnesses(opts.harness) as HarnessName[]; - - const replicate = opts.replicate ?? 1; - if (!Number.isInteger(replicate) || replicate < 1) { - throw new Error(`--replicate must be an integer >= 1 (got ${String(opts.replicate)})`); - } - if (replicate > 1 && replicate < 5) { - console.warn(`[WARNING] --replicate ${replicate}: K>=5 is the minimum for a credible signal (GOAL.md); below K=5, lift cannot be reliably distinguished from noise.`); - } - - const store = await deps.storeProvider(); - const scenario = await store.getScenario(opts.scenario); - if (!scenario) { - throw new Error(`Scenario not found: ${opts.scenario}`); - } - - const sessionCount = opts.sessions ?? scenario.sessionCount; - const effectiveScenario: TestScenario = sessionCount === scenario.sessionCount - ? scenario - : { ...scenario, sessionCount }; - - const abRunner = deps.abRunner ?? runABComparison; - const executor = deps.executorProvider?.() ?? new LocalExecutor(); - const runId = randomUUID(); - const startedAt = new Date().toISOString(); - const runRecord: EvalRunRecord = { - runId, - scenarioId: effectiveScenario.id, - harnesses, - sessionCount, - startedAt, - status: 'running', - }; - const supportsRunState = hasRunStateMethods(store); - if (supportsRunState) { - await store.saveRunRecord(runRecord); - } - const heartbeatWriter = supportsRunState - ? await deps.heartbeatWriterProvider?.({ - store, - runId, - scenarioId: effectiveScenario.id, - harnesses, - sessionCount, - }) - : undefined; - await heartbeatWriter?.initialize?.(); - console.log(`Run ID: ${runId}`); - - const comparisons: ComparisonResult[] = []; - const replicationsByHarness: ComparisonResult[][] = []; - try { - for (const name of harnesses) { - const adapter = deps.adapterProvider?.(name) ?? buildAdapter(name); - const replications: ComparisonResult[] = []; - for (let rep = 0; rep < replicate; rep++) { - const comparison = await abRunner({ - scenario: effectiveScenario, - adapter, - executor, - store, - runId, - heartbeatWriter, - }); - replications.push(comparison); - comparisons.push(comparison); - console.log(formatComparisonOutput(comparison, scenario.disruptions)); - } - replicationsByHarness.push(replications); - } - - if (supportsRunState) { - await store.saveRunRecord({ - ...runRecord, - completedAt: new Date().toISOString(), - status: 'completed', - }); - } - } catch (err: unknown) { - if (supportsRunState) { - await store.saveRunRecord({ - ...runRecord, - completedAt: new Date().toISOString(), - status: 'failed', - }); - } - throw err; - } - - if (comparisons.length === 0) { - console.log('No comparisons produced; skipping report.'); - return; - } - - // Replication mode (K>1): the statistical artifact (per-dimension lift as - // mean +/- SD with a one-SD significance flag) per harness is produced, - // persisted by runId, and displayed. The single-run cross-harness/full- - // report view below is left exactly as-is for K=1. - if (replicate > 1) { - const canPersistReplication = typeof (store as Partial<ResultStore>).saveReplicationReport === 'function'; - for (const replications of replicationsByHarness) { - const replicationReport = aggregateReplications(replications); - if (canPersistReplication) { - await store.saveReplicationReport(runId, replicationReport); - } - console.log(formatReplicationReport(replicationReport)); - } - return; - } - - if (comparisons.length > 1) { - console.log(formatCrossHarnessComparison(comparisons)); - } - - const tamperedExcluded = comparisons.filter((c) => c.tampered); - const aggregateComparisons = comparisons.filter((c) => !c.tampered); - - if (aggregateComparisons.length === 0) { - console.log(`No comparisons available for report: all ${tamperedExcluded.length} were marked tampered. Use 'eval report --scenario ${effectiveScenario.id} --include-tampered' to view.`); - return; - } - - const report = generateFullReport( - aggregateComparisons, - effectiveScenario.disruptions ?? [], - tamperedExcluded, - ); - console.log(formatFullReport(report)); - if (tamperedExcluded.length > 0) { - console.log(`\n[NOTICE] ${tamperedExcluded.length} tampered comparison(s) excluded from aggregates. Use 'eval report --scenario ${effectiveScenario.id} --include-tampered' to include them.`); - } - }); -} - -function hasRunStateMethods(store: ResultStore): boolean { - const candidate = store as Partial<ResultStore>; - return typeof candidate.saveRunRecord === 'function' - && typeof candidate.writeHeartbeat === 'function'; -} - -/** - * Validates harness names against known adapters. - * Returns validated names or throws with helpful error. - */ -export function validateHarnesses(harnesses: readonly string[]): string[] { - const invalid = harnesses.filter((h) => !VALID_HARNESSES.includes(h as HarnessName)); - if (invalid.length > 0) { - throw new Error( - `Unknown harness(es): ${invalid.join(', ')}. Valid harnesses: ${VALID_HARNESSES.join(', ')}`, - ); - } - return [...harnesses]; -} diff --git a/evals/src/cli/commands/scenario-create.ts b/evals/src/cli/commands/scenario-create.ts deleted file mode 100644 index 21490417..00000000 --- a/evals/src/cli/commands/scenario-create.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { Command } from 'commander'; -import { randomUUID } from 'node:crypto'; -import * as fs from 'node:fs/promises'; -import { createTestScenario } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; -import type { Disruption, ExpectedJumboMemoryCapture, JumboPlan, StructuralAssertion, TestScenario } from '../../domain/types.js'; - -export interface ScenarioCreateDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -interface ScenarioTemplate { - name: string; - initialPrompt: string; - continuationPrompt?: string; - sessionCount: number; - expectedFiles?: string[]; - retentionPatterns?: string[]; - structuralAssertions?: StructuralAssertion[]; - disruptions?: Disruption[]; - expectedJumboMemoryCaptures?: ExpectedJumboMemoryCapture[]; - jumboPlan?: JumboPlan; -} - -export function registerScenarioCommands(parent: Command, deps: ScenarioCreateDeps): void { - parent - .command('create') - .description('Define a new test scenario from a JSON template') - .requiredOption('--from-template <path>', 'Path to JSON template file') - .option('--name <name>', 'Override scenario name from template') - .option('--sessions <count>', 'Override session count', parseInt) - .addHelpText('after', ` -Examples: - eval scenario create --from-template ./scenarios/basic.json - eval scenario create --from-template ./scenarios/basic.json --name "Custom Name" --sessions 5 - `) - .action(async (opts: { fromTemplate: string; name?: string; sessions?: number }) => { - const template = await loadTemplate(opts.fromTemplate); - const scenario = handleScenarioCreate(template, { name: opts.name, sessions: opts.sessions }); - const store = await deps.storeProvider(); - await store.saveScenario(scenario); - console.log(scenario.id); - }); -} - -async function loadTemplate(filePath: string): Promise<ScenarioTemplate> { - let raw: string; - try { - raw = await fs.readFile(filePath, 'utf-8'); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - throw new Error(`Cannot read template file '${filePath}': ${msg}`); - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - throw new Error(`Template file '${filePath}' is not valid JSON: ${msg}`); - } - - return validateTemplate(parsed, filePath); -} - -function validateTemplate(value: unknown, filePath: string): ScenarioTemplate { - if (typeof value !== 'object' || value === null) { - throw new Error(`Template '${filePath}' must be a JSON object`); - } - const t = value as Record<string, unknown>; - if (typeof t.name !== 'string' || t.name.length === 0) { - throw new Error(`Template '${filePath}' is missing required string field 'name'`); - } - if (typeof t.initialPrompt !== 'string' || t.initialPrompt.length === 0) { - throw new Error(`Template '${filePath}' is missing required string field 'initialPrompt'`); - } - if (typeof t.sessionCount !== 'number' || !Number.isInteger(t.sessionCount) || t.sessionCount < 1) { - throw new Error(`Template '${filePath}' must have integer 'sessionCount' >= 1`); - } - return t as unknown as ScenarioTemplate; -} - -export function handleScenarioCreate( - template: ScenarioTemplate, - overrides?: { name?: string; sessions?: number }, -): TestScenario { - return createTestScenario({ - id: randomUUID(), - name: overrides?.name ?? template.name, - initialPrompt: template.initialPrompt, - continuationPrompt: template.continuationPrompt, - sessionCount: overrides?.sessions ?? template.sessionCount, - expectedFiles: template.expectedFiles, - retentionPatterns: template.retentionPatterns, - structuralAssertions: template.structuralAssertions, - disruptions: template.disruptions, - expectedJumboMemoryCaptures: template.expectedJumboMemoryCaptures, - jumboPlan: template.jumboPlan, - }); -} diff --git a/evals/src/cli/commands/scenario-list.ts b/evals/src/cli/commands/scenario-list.ts deleted file mode 100644 index 53c36d6d..00000000 --- a/evals/src/cli/commands/scenario-list.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Command } from 'commander'; -import type { TestScenario } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; - -export interface ScenarioListDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -export function registerScenarioListCommand(parent: Command, deps: ScenarioListDeps): void { - parent - .command('list') - .description('Show available test scenarios') - .option('--json', 'Output as JSON') - .addHelpText('after', ` -Examples: - eval scenario list - eval scenario list --json - `) - .action(async (opts: { json?: boolean }) => { - const store = await deps.storeProvider(); - const scenarios = await store.listScenarios(); - if (opts.json) { - console.log(JSON.stringify(scenarios, null, 2)); - } else { - console.log(formatScenarioList(scenarios)); - } - }); -} - -export function formatScenarioList(scenarios: readonly TestScenario[]): string { - if (scenarios.length === 0) { - return 'No scenarios found. Create one with: eval scenario create --from-template <path>'; - } - - const lines: string[] = []; - const divider = '─'.repeat(60); - - lines.push(` ${scenarios.length} scenario(s) available`); - lines.push(divider); - - for (const s of scenarios) { - lines.push(` ${s.id}`); - lines.push(` Name: ${s.name}`); - lines.push(` Sessions: ${s.sessionCount}`); - lines.push(` Created: ${s.createdAt}`); - if (s.disruptions && s.disruptions.length > 0) { - lines.push(` Disruptions: ${s.disruptions.length}`); - } - lines.push(''); - } - - return lines.join('\n'); -} diff --git a/evals/src/cli/commands/score.ts b/evals/src/cli/commands/score.ts deleted file mode 100644 index 1eb129dc..00000000 --- a/evals/src/cli/commands/score.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { Command } from 'commander'; -import type { TestResult, DimensionScore, TestScenario, ComparisonResult } from '../../domain/types.js'; -import { createComparisonResult } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; -import { scoreFileAccuracy } from '../../scoring/file-accuracy-scorer.js'; -import { scoreKnowledgeRetention } from '../../scoring/knowledge-retention-scorer.js'; -import { scoreDisruptionRecovery } from '../../scoring/disruption-recovery-scorer.js'; -import { baselineJumboMemoryCaptureScore, scoreJumboMemoryCapture } from '../../scoring/jumbo-memory-capture-scorer.js'; -import { compareTokenEfficiency, scoreTokenEfficiency } from '../../scoring/token-efficiency-scorer.js'; - -export interface ScoreDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -interface ScoreCommandOptions { - readonly scenario: string; - readonly result?: string; - readonly includeTampered?: boolean; -} - -type ScoredTestResult = TestResult & { - readonly scores: readonly DimensionScore[]; - readonly comparisonResult?: ComparisonResult; -}; - -/** - * Registers the 'score' command. - * Triggers the scoring engine against completed runs. - */ -export function registerScoreCommand(program: Command, deps: ScoreDeps): void { - program - .command('score') - .description('Run the scoring engine against completed runs') - .requiredOption('--scenario <id>', 'Scenario ID to score') - .option('--result <id>', 'Specific test result ID to score') - .option('--include-tampered', 'Include tampered SessionRecords in aggregate dimension scores (excluded by default)') - .addHelpText('after', ` -Examples: - eval score --scenario abc123 - eval score --scenario abc123 --result result-456 - eval score --scenario abc123 --include-tampered - `) - .action(async (opts: ScoreCommandOptions) => { - const store = await deps.storeProvider(); - const scenario = await store.getScenario(opts.scenario); - if (!scenario) { - throw new Error(`Scenario not found: ${opts.scenario}`); - } - - const results = await loadResults(store, opts); - if (results.length === 0) { - throw new Error(`No completed test results found for scenario: ${opts.scenario}`); - } - - const scored = results.map((result) => ({ - result, - scores: scoreDeterministicDimensions(result, scenario, { includeTampered: opts.includeTampered ?? false }), - })); - - for (const item of scored) { - console.log(formatScoreOutput(item.result, item.scores)); - } - - const comparisons = computeComparisons(scored, { includeTampered: opts.includeTampered ?? false }); - const comparisonsByResultId = new Map<string, ComparisonResult>(); - for (const comparison of comparisons) { - comparisonsByResultId.set(comparison.jumboResult.id, comparison); - comparisonsByResultId.set(comparison.baselineResult.id, comparison); - } - - for (const item of scored) { - const scoredResult: ScoredTestResult = { - ...item.result, - scores: item.scores, - comparisonResult: comparisonsByResultId.get(item.result.id), - }; - await store.saveTestResult(scoredResult); - } - }); -} - -async function loadResults( - store: ResultStore, - opts: ScoreCommandOptions, -): Promise<TestResult[]> { - const results = await store.listTestResults(opts.scenario); - if (!opts.result) return results; - - const result = results.find((candidate) => candidate.id === opts.result); - if (!result) { - throw new Error(`Test result not found for scenario ${opts.scenario}: ${opts.result}`); - } - return [result]; -} - -function isBaselineResult(result: TestResult): boolean { - return result.sessionRecords.some((record) => record.variant === 'baseline') - && !result.sessionRecords.some((record) => record.variant === 'jumbo'); -} - -function isJumboResult(result: TestResult): boolean { - return result.sessionRecords.some((record) => record.variant === 'jumbo'); -} - -function scoreDeterministicDimensions( - result: TestResult, - scenario: TestScenario, - opts: { includeTampered: boolean }, -): DimensionScore[] { - const expectedFiles = scenario.expectedFiles ?? []; - const retentionPatterns = scenario.retentionPatterns ?? []; - const disruptions = scenario.disruptions ?? []; - const expectedJumboMemoryCaptures = scenario.expectedJumboMemoryCaptures ?? []; - const records = opts.includeTampered - ? result.sessionRecords - : result.sessionRecords.filter((r) => !r.tampered); - - const fileScore = scoreFileAccuracy(records, expectedFiles); - const retentionScore = scoreKnowledgeRetention(records, retentionPatterns); - const disruptionScore = scoreDisruptionRecovery(records, disruptions); - const memoryScore = isBaselineResult(result) - ? baselineJumboMemoryCaptureScore(expectedJumboMemoryCaptures) - : scoreJumboMemoryCapture(records, expectedJumboMemoryCaptures); - const averageQuality = (fileScore.score + retentionScore.score) / 2; - const tokenScore = scoreTokenEfficiency(records, averageQuality); - - return [ - fileScore, - retentionScore, - disruptionScore, - tokenScore, - memoryScore, - ]; -} - -function computeComparisons( - scored: readonly { result: TestResult; scores: readonly DimensionScore[] }[], - opts: { includeTampered: boolean }, -): ComparisonResult[] { - const comparisons: ComparisonResult[] = []; - - const byHarness = new Map<string, typeof scored>(); - for (const item of scored) { - byHarness.set(item.result.harness, [...(byHarness.get(item.result.harness) ?? []), item]); - } - - for (const items of byHarness.values()) { - const jumbo = items.find((item) => isJumboResult(item.result)); - const baseline = items.find((item) => isBaselineResult(item.result)); - if (!jumbo || !baseline) continue; - - const jumboQuality = averageQualityScore(jumbo.scores); - const baselineQuality = averageQualityScore(baseline.scores); - const jumboTokenRecords = opts.includeTampered - ? jumbo.result.sessionRecords - : jumbo.result.sessionRecords.filter((r) => !r.tampered); - const baselineTokenRecords = opts.includeTampered - ? baseline.result.sessionRecords - : baseline.result.sessionRecords.filter((r) => !r.tampered); - const tokenDelta = compareTokenEfficiency( - jumboTokenRecords, - baselineTokenRecords, - jumboQuality, - baselineQuality, - ); - const jumboScores = jumbo.scores.map((score) => - score.dimension === tokenDelta.dimension ? tokenDelta : score, - ); - const baselineScores = baseline.scores.map((score) => - score.dimension === tokenDelta.dimension ? tokenDelta : score, - ); - - const deltas = jumboScores.map((jumboScore) => { - const baselineScore = baselineScores.find((score) => score.dimension === jumboScore.dimension); - return { - dimension: jumboScore.dimension, - score: Math.round((jumboScore.score - (baselineScore?.score ?? 0)) * 100) / 100, - maxScore: jumboScore.maxScore, - details: `jumbo=${jumboScore.score.toFixed(2)} baseline=${(baselineScore?.score ?? 0).toFixed(2)}`, - }; - }); - - comparisons.push(createComparisonResult({ - id: randomUUID(), - scenarioId: jumbo.result.scenarioId, - harness: jumbo.result.harness, - jumboResult: jumbo.result, - baselineResult: baseline.result, - jumboScores, - baselineScores, - deltas, - })); - } - - return comparisons; -} - -function averageQualityScore(scores: readonly DimensionScore[]): number { - const qualityScores = scores.filter((score) => - score.dimension === 'file-accuracy' || score.dimension === 'knowledge-retention' - ); - if (qualityScores.length === 0) return 0; - return qualityScores.reduce((sum, score) => sum + score.score, 0) / qualityScores.length; -} - -/** - * Formats scoring output for terminal display. - * Pure function — no I/O. - */ -export function formatScoreOutput( - result: TestResult, - scores: readonly DimensionScore[], -): string { - const lines: string[] = []; - const divider = '─'.repeat(50); - - lines.push(` Scores for result: ${result.id}`); - lines.push(` Scenario: ${result.scenarioId}`); - lines.push(` Harness: ${result.harness}`); - lines.push(` Sessions: ${result.sessionRecords.length}`); - lines.push(divider); - - for (const score of scores) { - const pct = score.maxScore === 0 ? 'N/A' : `${(score.score / score.maxScore * 100).toFixed(0)}%`; - lines.push(` ${padRight(score.dimension, 24)} ${score.score.toFixed(2)}/${score.maxScore.toFixed(2)} (${pct})`); - if (score.details) { - lines.push(` ${score.details}`); - } - } - - return lines.join('\n'); -} - -function padRight(str: string, width: number): string { - return str.length >= width ? str : str + ' '.repeat(width - str.length); -} diff --git a/evals/src/cli/commands/status.ts b/evals/src/cli/commands/status.ts deleted file mode 100644 index 1999065a..00000000 --- a/evals/src/cli/commands/status.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { Command } from 'commander'; -import type { ReactElement } from 'react'; -import type { RunHeartbeat, TestResult, ComparisonResult } from '../../domain/types.js'; -import type { ResultStore } from '../../storage/result-store.js'; -import { - createHeartbeatView, - formatHeartbeatDisplay, - isHeartbeatComplete, -} from '../../output/heartbeat-display.js'; - -export interface StatusDeps { - readonly storeProvider: () => Promise<ResultStore>; -} - -const POLL_INTERVAL_MS = 1000; - -/** - * Registers the 'status' command. - * Shows in-progress and completed runs with summary scores. - */ -export function registerStatusCommand(program: Command, deps: StatusDeps): void { - program - .command('status') - .description('Show in-progress and completed runs with summary scores') - .option('--scenario <id>', 'Filter by scenario ID') - .option('--json', 'Output as JSON') - .option('--watch <runId>', 'Watch a live run heartbeat by run ID') - .addHelpText('after', ` -Examples: - eval status - eval status --scenario abc123 - eval status --json - eval status --watch <runId> - `) - .action(async (opts: { scenario?: string; json?: boolean; watch?: string }) => { - const store = await deps.storeProvider(); - if (opts.watch) { - await watchRunStatus(store, opts.watch); - return; - } - - const results = await store.listTestResults(opts.scenario); - const comparisons: ComparisonResult[] = []; - if (opts.json) { - console.log(JSON.stringify({ results, comparisons }, null, 2)); - } else { - console.log(formatStatusOutput(results, comparisons)); - } - }); -} - -export interface WatchRunStatusOptions { - readonly pollIntervalMs?: number; - readonly isInteractive?: boolean; -} - -export async function watchRunStatus( - store: ResultStore, - runId: string, - options: WatchRunStatusOptions = {}, -): Promise<void> { - const pollIntervalMs = options.pollIntervalMs ?? POLL_INTERVAL_MS; - const isInteractive = options.isInteractive ?? process.stdout.isTTY === true; - - if (!isInteractive) { - await watchRunStatusNonInteractive(store, runId, pollIntervalMs); - return; - } - - await watchRunStatusInteractive(store, runId, pollIntervalMs); -} - -async function watchRunStatusNonInteractive( - store: ResultStore, - runId: string, - pollIntervalMs: number, -): Promise<void> { - while (true) { - const [heartbeat, runRecord] = await Promise.all([ - store.readHeartbeat(runId), - store.getRunRecord(runId), - ]); - if (!heartbeat) { - console.log(`No heartbeat found for run: ${runId}`); - return; - } - const completed = isRunComplete(heartbeat, runRecord?.status); - if (completed) { - console.log(formatHeartbeatDisplay(heartbeat)); - return; - } - await delay(pollIntervalMs); - } -} - -async function watchRunStatusInteractive( - store: ResultStore, - runId: string, - pollIntervalMs: number, -): Promise<void> { - const ink = await import('ink'); - const React = await import('react'); - const element = createWatchAppElement(React, ink, { store, runId, pollIntervalMs }); - const instance = ink.render(element); - - const onSigint = (): void => { - instance.unmount(); - }; - process.once('SIGINT', onSigint); - try { - await instance.waitUntilExit(); - } finally { - process.removeListener('SIGINT', onSigint); - } -} - -interface WatchAppProps { - readonly store: ResultStore; - readonly runId: string; - readonly pollIntervalMs: number; -} - -/** - * Exported for tests. Builds the root ink element for the watch view. - * Accepts dynamically-loaded React and ink modules so the caller controls module - * resolution (the project compiles to CJS while ink ships ESM-only). - */ -export function createWatchAppElement( - React: typeof import('react'), - ink: typeof import('ink', { with: { 'resolution-mode': 'import' } }), - props: WatchAppProps, -): ReactElement { - const { useApp } = ink; - const { useEffect, useState, createElement } = React; - - const WatchApp = (): ReactElement => { - const [heartbeat, setHeartbeat] = useState<RunHeartbeat | null>(null); - const [missing, setMissing] = useState(false); - const [done, setDone] = useState(false); - const { exit } = useApp(); - - useEffect(() => { - if (!done && !missing) return; - const id = setTimeout(() => exit(), 0); - return (): void => clearTimeout(id); - }, [done, missing, exit]); - - useEffect(() => { - let cancelled = false; - let timer: ReturnType<typeof setTimeout> | undefined; - - const tick = async (): Promise<void> => { - try { - const [hb, runRecord] = await Promise.all([ - props.store.readHeartbeat(props.runId), - props.store.getRunRecord(props.runId), - ]); - if (cancelled) return; - if (!hb) { - setMissing(true); - return; - } - setHeartbeat(hb); - if (isRunComplete(hb, runRecord?.status)) { - setDone(true); - return; - } - timer = setTimeout(() => { void tick(); }, props.pollIntervalMs); - } catch (err) { - if (!cancelled) exit(err instanceof Error ? err : new Error(String(err))); - } - }; - - void tick(); - - return (): void => { - cancelled = true; - if (timer) clearTimeout(timer); - }; - }, []); - - if (missing) { - return createElement(ink.Text, null, `No heartbeat found for run: ${props.runId}`); - } - return createHeartbeatView(React, ink, { heartbeat }); - }; - - return React.createElement(WatchApp); -} - -function isRunComplete(heartbeat: RunHeartbeat, runStatus: string | undefined): boolean { - return runStatus === 'completed' || runStatus === 'failed' || isHeartbeatComplete(heartbeat); -} - -function delay(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Formats status output for terminal display. - * Shows test results grouped by scenario with session counts and completion status. - * Pure function — no I/O. - */ -export function formatStatusOutput( - results: readonly TestResult[], - comparisons?: readonly ComparisonResult[], -): string { - if (results.length === 0 && (!comparisons || comparisons.length === 0)) { - return 'No runs found. Start one with: eval run --scenario <id> --image <image>'; - } - - const lines: string[] = []; - const divider = '─'.repeat(60); - - // Group results by scenario - const byScenario = new Map<string, TestResult[]>(); - for (const r of results) { - const existing = byScenario.get(r.scenarioId) ?? []; - existing.push(r); - byScenario.set(r.scenarioId, existing); - } - - lines.push(` ${results.length} test result(s)`); - if (comparisons && comparisons.length > 0) { - lines.push(` ${comparisons.length} comparison(s) completed`); - } - lines.push(divider); - - for (const [scenarioId, scenarioResults] of byScenario) { - lines.push(` Scenario: ${scenarioId}`); - - for (const r of scenarioResults) { - lines.push(` Result: ${r.id}`); - lines.push(` Harness: ${r.harness}`); - lines.push(` Sessions: ${r.sessionRecords.length}`); - lines.push(` Created: ${r.createdAt}`); - } - - // Show comparison summaries for this scenario - const scenarioComparisons = (comparisons ?? []).filter((c) => c.scenarioId === scenarioId); - for (const comp of scenarioComparisons) { - lines.push(` Comparison: ${comp.id}`); - lines.push(` Harness: ${comp.harness}`); - const avgDelta = comp.deltas.length > 0 - ? comp.deltas.reduce((sum, d) => sum + d.score, 0) / comp.deltas.length - : 0; - const sign = avgDelta > 0 ? '+' : ''; - lines.push(` Avg Lift: ${sign}${avgDelta.toFixed(3)}`); - lines.push(` Dimensions: ${comp.jumboScores.length}`); - } - - lines.push(''); - } - - return lines.join('\n'); -} diff --git a/evals/src/cli/index.ts b/evals/src/cli/index.ts deleted file mode 100644 index c3563c11..00000000 --- a/evals/src/cli/index.ts +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env node - -import { Command } from 'commander'; -import * as path from 'node:path'; -import { StoreHeartbeatWriter } from '../infrastructure/heartbeat-writer.js'; -import type { LocalExecutor } from '../infrastructure/local-executor.js'; -import type { HarnessAdapter } from '../harness/harness-adapter.js'; -import { JsonResultStore } from '../storage/json-result-store.js'; -import type { ResultStore } from '../storage/result-store.js'; -import { registerScenarioCommands } from './commands/scenario-create.js'; -import { registerScenarioListCommand } from './commands/scenario-list.js'; -import { registerRunCommand, type ABRunner, type HeartbeatWriterProvider } from './commands/run.js'; -import { registerScoreCommand } from './commands/score.js'; -import { registerReportCommand } from './commands/report.js'; -import { registerStatusCommand } from './commands/status.js'; -import { registerControlCommand } from './commands/control.js'; - -export interface CliDeps { - readonly storeProvider: () => Promise<ResultStore>; - readonly abRunner?: ABRunner; - readonly heartbeatWriterProvider?: HeartbeatWriterProvider; - readonly executorProvider?: () => LocalExecutor; - readonly adapterProvider?: (name: string) => HarnessAdapter; -} - -function defaultStoreProvider(): () => Promise<ResultStore> { - return async () => { - const base = process.env.EVAL_STORE_PATH ?? path.join(process.cwd(), '.eval-store'); - const store = new JsonResultStore(base); - await store.initialize(); - return store; - }; -} - -export function createProgram(deps?: CliDeps): Command { - const storeProvider = deps?.storeProvider ?? defaultStoreProvider(); - const heartbeatWriterProvider = deps?.heartbeatWriterProvider ?? defaultHeartbeatWriterProvider; - - const program = new Command(); - - program - .name('eval') - .description('Jumbo longitudinal evaluation system — measure Jumbo lift across harnesses and sessions') - .version('0.1.0'); - - const scenario = program - .command('scenario') - .description('Manage test scenarios'); - - registerScenarioCommands(scenario, { storeProvider }); - registerScenarioListCommand(scenario, { storeProvider }); - - registerRunCommand(program, { - storeProvider, - abRunner: deps?.abRunner, - heartbeatWriterProvider, - executorProvider: deps?.executorProvider, - adapterProvider: deps?.adapterProvider, - }); - registerScoreCommand(program, { storeProvider }); - registerReportCommand(program, { storeProvider }); - registerStatusCommand(program, { storeProvider }); - registerControlCommand(program, { storeProvider }); - - return program; -} - -const defaultHeartbeatWriterProvider: HeartbeatWriterProvider = async (params: { - readonly store: ResultStore; - readonly runId: string; - readonly scenarioId: string; - readonly harnesses: readonly string[]; - readonly sessionCount: number; -}) => { - if (!hasRunStateMethods(params.store)) return undefined; - return new StoreHeartbeatWriter(params.store, params); -}; - -function hasRunStateMethods(store: ResultStore): boolean { - const candidate = store as Partial<ResultStore>; - return typeof candidate.saveRunRecord === 'function' - && typeof candidate.writeHeartbeat === 'function'; -} - -export function isMainModule(modulePath: string, argv1: string | undefined): boolean { - if (!argv1) return false; - try { - return path.resolve(modulePath) === path.resolve(argv1); - } catch { - return false; - } -} - -if (isMainModule(__filename, process.argv[1])) { - const program = createProgram(); - program.parseAsync(process.argv).catch((err: Error) => { - console.error(`Error: ${err.message}`); - process.exit(1); - }); -} diff --git a/evals/src/domain/clock.ts b/evals/src/domain/clock.ts deleted file mode 100644 index 8e65489d..00000000 --- a/evals/src/domain/clock.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Injectable wall-clock source. Factories accept a Clock so timestamp - * generation is deterministic under test instead of reaching for `new Date()` - * inline. Production code uses `systemClock`. - */ -export type Clock = () => string; - -/** ISO-8601 UTC timestamp from the system clock. */ -export const systemClock: Clock = () => new Date().toISOString(); diff --git a/evals/src/domain/heartbeat.ts b/evals/src/domain/heartbeat.ts deleted file mode 100644 index e73fc9c5..00000000 --- a/evals/src/domain/heartbeat.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Live progress heartbeats written during a run so an external watcher can - * render per-session status without inspecting the agent process directly. - */ - -import type { SessionPhaseTimings } from './timing.js'; - -export type SessionHeartbeatStatus = 'pending' | 'running' | 'completed' | 'failed'; -export type SessionHeartbeatPhase = 'harness-exec' | 'lifecycle-audit' | 'scoring'; - -export interface SessionHeartbeat { - readonly sessionNumber: number; - readonly status: SessionHeartbeatStatus; - readonly phase?: SessionHeartbeatPhase; - readonly startedAt?: string; - readonly completedAt?: string; - readonly phaseTimings?: SessionPhaseTimings; - readonly errorMessage?: string; -} - -export interface HarnessHeartbeat { - readonly harness: string; - readonly variant: 'jumbo' | 'baseline'; - readonly sessions: readonly SessionHeartbeat[]; -} - -export interface RunHeartbeat { - readonly runId: string; - readonly scenarioId: string; - readonly updatedAt: string; - readonly harnesses: readonly HarnessHeartbeat[]; -} diff --git a/evals/src/domain/index.ts b/evals/src/domain/index.ts deleted file mode 100644 index c70d61fe..00000000 --- a/evals/src/domain/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Canonical barrel for the domain model. The types are organised into focused - * modules by concern; import them from here (or, for back-compat, from - * `./types.js`). - */ - -export * from './scenario.js'; -export * from './jumbo-plan.js'; -export * from './jumbo-memory.js'; -export * from './jumbo-lifecycle.js'; -export * from './workspace.js'; -export * from './timing.js'; -export * from './tamper.js'; -export * from './tamper-provenance.js'; -export * from './clock.js'; -export * from './validation.js'; -export * from './session.js'; -export * from './heartbeat.js'; -export * from './result.js'; -export * from './result-factories.js'; -export * from './replication.js'; diff --git a/evals/src/domain/jumbo-lifecycle.ts b/evals/src/domain/jumbo-lifecycle.ts deleted file mode 100644 index 52792aa2..00000000 --- a/evals/src/domain/jumbo-lifecycle.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Post-session verification that the agent actually executed the Jumbo - * lifecycle, derived from Jumbo's own state rather than transcript parsing. - */ - -/** - * Post-session verification that the agent actually executed the Jumbo - * lifecycle (session start, goal start, in-session captures, progress - * updates, goal submit, session end). Each boolean is derived from the - * captured CLI evidence below, not from transcript parsing — the framework - * treats Jumbo's own state as ground truth for what happened during the - * harness exec. - * - * `inSessionCapturesExecuted` is true when the post-session memory snapshot - * contains entities not present in the pre-session snapshot. `progressUpdatesExecuted` - * is derived from the goal entity's version delta: `jumbo goal update-progress` - * mutates the goal aggregate (bumping version) without changing status, so a - * version delta beyond the canonical start+submit mutations indicates - * progress entries were appended. - */ -export interface JumboLifecycleAudit { - readonly sessionStartExecuted: boolean; - readonly goalStartExecuted: boolean; - readonly inSessionCapturesExecuted: boolean; - readonly progressUpdatesExecuted: boolean; - readonly goalSubmitExecuted: boolean; - readonly sessionEndExecuted: boolean; - readonly activeGoalId?: string; - readonly goalStatusBefore?: string; - readonly goalStatusAfter?: string; - readonly goalVersionBefore?: number; - readonly goalVersionAfter?: number; - readonly sessionsTotalDelta: number; - readonly sessionsEndedDelta: number; - readonly newEntityCount: number; - readonly evidence: JumboLifecycleEvidence; -} - -export interface JumboLifecycleEvidence { - readonly goalShowBefore?: JumboLifecycleCommandResult; - readonly goalShowAfter?: JumboLifecycleCommandResult; - readonly sessionsListBefore?: JumboLifecycleCommandResult; - readonly sessionsListAfter?: JumboLifecycleCommandResult; - readonly sessionsEndedListAfter?: JumboLifecycleCommandResult; - readonly decisionsListAfter?: JumboLifecycleCommandResult; -} - -export interface JumboLifecycleCommandResult { - readonly command: readonly string[]; - readonly stdout: string; - readonly stderr: string; - readonly exitCode: number; -} diff --git a/evals/src/domain/jumbo-memory.ts b/evals/src/domain/jumbo-memory.ts deleted file mode 100644 index 698ab357..00000000 --- a/evals/src/domain/jumbo-memory.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Captured Jumbo memory state: the entities Jumbo holds and the CLI commands - * used to read them. Pre/post-session snapshots are the primary evidence for - * in-session captures. - */ - -export type JumboMemoryKind = 'decision' | 'guideline' | 'invariant' | 'component' | 'relation' | 'dependency'; - -export interface ExpectedJumboMemoryCapture { - readonly kind: JumboMemoryKind; - readonly match: string; - readonly sessionNumber?: number; -} - -export interface JumboMemoryEntity { - readonly kind: JumboMemoryKind; - readonly id?: string; - readonly text: string; - readonly raw: unknown; -} - -export interface JumboMemoryCommandResult { - readonly kind: JumboMemoryKind; - readonly command: readonly string[]; - readonly stdout: string; - readonly stderr: string; - readonly exitCode: number; -} - -export interface JumboMemorySnapshot { - readonly sessionNumber: number; - readonly capturedAt: string; - readonly entities: readonly JumboMemoryEntity[]; - readonly commands: readonly JumboMemoryCommandResult[]; -} diff --git a/evals/src/domain/jumbo-plan.ts b/evals/src/domain/jumbo-plan.ts deleted file mode 100644 index 7a10e1ea..00000000 --- a/evals/src/domain/jumbo-plan.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * The pre-seeded Jumbo plan: architectural context the framework registers via - * the jumbo CLI (after `jumbo init`, before session 1) plus the progressively - * released goals. The framework plays the developer role — it stamps starting - * context and picks which goal is active per session; the agent merely executes - * the lifecycle on whichever goal-id it was handed. - */ - -/** - * Pre-seeded memory entry to register via the jumbo CLI after `jumbo init` - * and before session 1. The framework plays the developer role: it stamps - * the project's starting architectural context so the agent inherits it - * rather than discovering it from the prompt alone. - * - * `planRef` is a plan-local identifier used by JumboPlanRelation.fromRef / - * toRef to link entities that don't exist yet (the real IDs are minted by - * the CLI at registration time). It is not sent to Jumbo. - */ -export type JumboPlanEntry = - | JumboPlanDecisionEntry - | JumboPlanComponentEntry - | JumboPlanInvariantEntry - | JumboPlanDependencyEntry - | JumboPlanRelationEntry; - -export interface JumboPlanDecisionEntry { - readonly kind: 'decision'; - readonly planRef?: string; - readonly title: string; - readonly context: string; - readonly rationale?: string; - readonly alternatives?: readonly string[]; - readonly consequences?: string; -} - -export type JumboComponentType = 'service' | 'db' | 'queue' | 'ui' | 'lib' | 'api' | 'worker' | 'cache' | 'storage'; - -export interface JumboPlanComponentEntry { - readonly kind: 'component'; - readonly planRef?: string; - readonly name: string; - readonly type: JumboComponentType; - readonly description: string; - readonly responsibility: string; - readonly path: string; -} - -export interface JumboPlanInvariantEntry { - readonly kind: 'invariant'; - readonly planRef?: string; - readonly title: string; - readonly description: string; - readonly rationale?: string; -} - -export interface JumboPlanDependencyEntry { - readonly kind: 'dependency'; - readonly planRef?: string; - readonly name: string; - readonly ecosystem: string; - readonly packageName: string; - readonly versionConstraint?: string; - readonly endpoint?: string; - readonly contract?: string; -} - -export interface JumboPlanRelationEntry { - readonly kind: 'relation'; - readonly planRef?: string; - readonly fromType: string; - readonly fromId: string; - readonly toType: string; - readonly toId: string; - readonly type: string; - readonly description: string; - readonly strength?: 'strong' | 'medium' | 'weak'; -} - -/** - * A goal in the plan, tagged with the session at which the framework will - * register it via `jumbo goal add`. Progressive release: a goal with - * sessionAvailableFrom: 2 is invisible to the agent during session 1 and - * becomes the active goal for session 2. - * - * The framework picks which goal is active per session (the developer's - * job); the agent merely executes the lifecycle on whichever goal-id it - * was handed. - */ -export interface JumboPlanGoal { - readonly title: string; - readonly objective: string; - readonly criteria: readonly string[]; - readonly scopeIn?: readonly string[]; - readonly scopeOut?: readonly string[]; - readonly sessionAvailableFrom: number; - readonly prerequisitePlanRefs?: readonly string[]; - readonly planRef?: string; -} - -export interface JumboPlan { - readonly preSeededMemory?: readonly JumboPlanEntry[]; - readonly goals: readonly JumboPlanGoal[]; -} diff --git a/evals/src/domain/replication.ts b/evals/src/domain/replication.ts deleted file mode 100644 index 084643a9..00000000 --- a/evals/src/domain/replication.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Statistics over K replicated A/B comparisons of the same scenario and harness - * (GOAL.md Outcome 5). Lift is reported as mean ± standard deviation, never a - * single-point estimate, and is only a "signal" when it exceeds one standard - * deviation of its own distribution across replications. - */ - -export interface DimensionLiftStat { - readonly dimension: string; - /** Total replications in the batch. */ - readonly k: number; - /** Replications where this dimension was applicable (excludes N/A, e.g. token-efficiency with maxScore 0). */ - readonly applicableReplications: number; - readonly meanJumbo: number; - readonly meanBaseline: number; - /** mean over applicable replications of (jumboScore − baselineScore). */ - readonly meanLift: number; - /** Sample (n−1) standard deviation of the per-replication lifts; 0 when fewer than 2 applicable. */ - readonly sdLift: number; - /** meanLift / (sdLift / sqrt(applicable)); 0 when sdLift is 0 or fewer than 2 applicable. */ - readonly tStatistic: number; - /** True only when |meanLift| > sdLift (GOAL.md: a lift is a signal only when it exceeds one SD). */ - readonly isSignal: boolean; -} - -export interface ReplicationSignificance { - /** The rule used for `isSignal`. */ - readonly rule: string; - /** One-tailed α=0.05 critical t for df = k−1, or null when df is outside the lookup table. */ - readonly tCriticalOneTailed05: number | null; - readonly note: string; -} - -export interface ReplicationReport { - readonly scenarioId: string; - readonly harness: string; - /** Number of replications aggregated. */ - readonly k: number; - readonly dimensions: readonly DimensionLiftStat[]; - readonly significance: ReplicationSignificance; - readonly createdAt: string; -} diff --git a/evals/src/domain/result-factories.ts b/evals/src/domain/result-factories.ts deleted file mode 100644 index eaaea9fe..00000000 --- a/evals/src/domain/result-factories.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Factories that build scoring results, composing the shared domain behavior: - * required-field validation, tamper-provenance merging, and createdAt stamping - * via an injectable clock. The shapes themselves live in `result.ts`. - */ -import type { SessionRecord } from './session.js'; -import type { TamperEvent } from './tamper.js'; -import type { ComparisonResult, DimensionScore, PerSessionScore, TestResult } from './result.js'; -import type { Clock } from './clock.js'; -import { systemClock } from './clock.js'; -import { mergeTamperProvenance } from './tamper-provenance.js'; -import { requireField } from './validation.js'; - -export function createComparisonResult( - params: { - id: string; - scenarioId: string; - harness: string; - jumboResult: TestResult; - baselineResult: TestResult; - jumboScores: readonly DimensionScore[]; - baselineScores: readonly DimensionScore[]; - deltas: readonly DimensionScore[]; - jumboTimeline?: readonly PerSessionScore[]; - baselineTimeline?: readonly PerSessionScore[]; - tampered?: boolean; - tamperLog?: readonly TamperEvent[]; - }, - clock: Clock = systemClock, -): ComparisonResult { - requireField(params.id, 'ComparisonResult requires an id'); - requireField(params.jumboResult, 'ComparisonResult requires a jumboResult'); - requireField(params.baselineResult, 'ComparisonResult requires a baselineResult'); - - const { tampered, tamperLog } = mergeTamperProvenance( - { tampered: params.tampered, tamperLog: params.tamperLog }, - [params.jumboResult, params.baselineResult], - ); - - return { - ...params, - createdAt: clock(), - tampered, - tamperLog, - }; -} - -export function createTestResult( - params: { - id: string; - scenarioId: string; - harness: string; - sessionRecords: readonly SessionRecord[]; - tampered?: boolean; - tamperLog?: readonly TamperEvent[]; - }, - clock: Clock = systemClock, -): TestResult { - requireField(params.id, 'TestResult requires an id'); - requireField(params.scenarioId, 'TestResult requires a scenarioId'); - requireField(params.harness, 'TestResult requires a harness'); - - const { tampered, tamperLog } = mergeTamperProvenance( - { tampered: params.tampered, tamperLog: params.tamperLog }, - params.sessionRecords, - ); - - return { - ...params, - createdAt: clock(), - tampered, - tamperLog, - }; -} diff --git a/evals/src/domain/result.ts b/evals/src/domain/result.ts deleted file mode 100644 index b5867e77..00000000 --- a/evals/src/domain/result.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Scoring output shapes: per-dimension scores, a single-arm test result, and - * the side-by-side comparison of the Jumbo and baseline arms. Shape only — the - * behavior that builds these (clock, tamper-provenance, validation) lives in - * `result-factories.ts` and the small helpers it composes. - */ - -import type { SessionRecord } from './session.js'; -import type { TamperEvent } from './tamper.js'; - -export interface TestResult { - readonly id: string; - readonly scenarioId: string; - readonly harness: string; - readonly sessionRecords: readonly SessionRecord[]; - readonly createdAt: string; - readonly tampered: boolean; - readonly tamperLog: readonly TamperEvent[]; -} - -export interface DimensionScore { - readonly dimension: string; - readonly score: number; - readonly maxScore: number; - readonly details?: string; -} - -export interface PerSessionScore { - readonly sessionNumber: number; - readonly scores: readonly DimensionScore[]; -} - -export interface ComparisonResult { - readonly id: string; - readonly scenarioId: string; - readonly harness: string; - readonly jumboResult: TestResult; - readonly baselineResult: TestResult; - readonly jumboScores: readonly DimensionScore[]; - readonly baselineScores: readonly DimensionScore[]; - readonly deltas: readonly DimensionScore[]; - readonly jumboTimeline?: readonly PerSessionScore[]; - readonly baselineTimeline?: readonly PerSessionScore[]; - readonly createdAt: string; - readonly tampered: boolean; - readonly tamperLog: readonly TamperEvent[]; -} diff --git a/evals/src/domain/scenario.ts b/evals/src/domain/scenario.ts deleted file mode 100644 index e9ea3ea4..00000000 --- a/evals/src/domain/scenario.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * The durable scenario specification: the task, its disruptions, and the - * structural assertions used to score it. Shared byte-identically across both - * setups (Jumbo and baseline) before variant-specific prompt wrapping. - */ - -import type { ExpectedJumboMemoryCapture } from './jumbo-memory.js'; -import type { JumboPlan } from './jumbo-plan.js'; - -/** - * Kinds of mid-project disruption a scenario can inject. - * - `correction`: a prior implementation was wrong and must be fixed. - * - `scope-change`: the task's boundaries move (features added/removed). - * - `new-constraint`: a fresh rule the existing work must now satisfy. - * - `design-change`: a design decision reverses or supersedes an earlier one - * (e.g. "switch from REST to events") — the agent must retrofit prior work to - * the new decision, testing whether that decision is retained across sessions. - */ -export type DisruptionType = 'correction' | 'scope-change' | 'new-constraint' | 'design-change'; - -export interface Disruption { - readonly type: DisruptionType; - readonly sessionNumber: number; - readonly content: string; - readonly recoveryPatterns: readonly string[]; -} - -/** - * A declarative, JSON-expressible check on the content of workspace artefacts. - * This is the primary knowledge-retention signal (GOAL.md §"Structural - * assertions vs. keyword matching"): retention is measured on the files the - * agent actually produced, not on keyword presence in transcripts. Matchers - * are interpreted by the structural-assertion scorer, so new scenarios declare - * assertions as data without any scoring-code changes. - * - * Matchers operate over the *concatenated content of every workspace file whose - * path matches `StructuralAssertion.file`* (a glob). `fileExists` only checks - * presence; the rest additionally require at least one matched file. - */ -export type StructuralMatcher = - | { readonly kind: 'fileExists' } - | { readonly kind: 'matchesRegex'; readonly pattern: string; readonly flags?: string } - | { readonly kind: 'containsAll'; readonly substrings: readonly string[] } - | { readonly kind: 'containsAny'; readonly substrings: readonly string[] } - | { readonly kind: 'notContains'; readonly substrings: readonly string[] } - | { readonly kind: 'exportsSymbol'; readonly symbol: string }; - -/** - * One structural check, due in a specific session. `file` is a glob matched - * against workspace-snapshot paths (`*` matches within a path segment, `**` - * matches across segments). The scorer evaluates the assertion against the - * snapshot of session `sessionNumber`. - */ -export interface StructuralAssertion { - readonly id: string; - readonly description?: string; - readonly file: string; - readonly sessionNumber: number; - readonly matcher: StructuralMatcher; -} - -export interface TestScenario { - readonly id: string; - readonly name: string; - readonly initialPrompt: string; - readonly continuationPrompt?: string; - readonly sessionCount: number; - readonly expectedFiles?: readonly string[]; - readonly retentionPatterns?: readonly string[]; - readonly structuralAssertions?: readonly StructuralAssertion[]; - readonly disruptions?: readonly Disruption[]; - readonly expectedJumboMemoryCaptures?: readonly ExpectedJumboMemoryCapture[]; - readonly jumboPlan?: JumboPlan; - readonly createdAt: string; -} - -export function createTestScenario(params: { - id: string; - name: string; - initialPrompt: string; - continuationPrompt?: string; - sessionCount: number; - expectedFiles?: readonly string[]; - retentionPatterns?: readonly string[]; - structuralAssertions?: readonly StructuralAssertion[]; - disruptions?: readonly Disruption[]; - expectedJumboMemoryCaptures?: readonly ExpectedJumboMemoryCapture[]; - jumboPlan?: JumboPlan; -}): TestScenario { - if (!params.id) throw new Error('TestScenario requires an id'); - if (!params.name) throw new Error('TestScenario requires a name'); - if (!params.initialPrompt) throw new Error('TestScenario requires an initialPrompt'); - if (params.sessionCount < 1) throw new Error('TestScenario requires sessionCount >= 1'); - - if (params.structuralAssertions) { - const seen = new Set<string>(); - for (const assertion of params.structuralAssertions) { - if (!assertion.id) { - throw new Error('TestScenario.structuralAssertions entries require an id'); - } - if (seen.has(assertion.id)) { - throw new Error(`TestScenario.structuralAssertions has a duplicate id: ${assertion.id}`); - } - seen.add(assertion.id); - if (!assertion.file) { - throw new Error(`TestScenario.structuralAssertions[${assertion.id}] requires a file glob`); - } - if ( - !Number.isInteger(assertion.sessionNumber) || - assertion.sessionNumber < 1 || - assertion.sessionNumber > params.sessionCount - ) { - throw new Error( - `TestScenario.structuralAssertions[${assertion.id}] sessionNumber must be an integer in 1..${params.sessionCount} (got ${assertion.sessionNumber})`, - ); - } - } - } - - if (params.jumboPlan) { - if (params.jumboPlan.goals.length === 0) { - throw new Error('TestScenario.jumboPlan must declare at least one goal'); - } - for (const goal of params.jumboPlan.goals) { - if ( - goal.sessionAvailableFrom < 1 || - goal.sessionAvailableFrom > params.sessionCount || - !Number.isInteger(goal.sessionAvailableFrom) - ) { - throw new Error( - `TestScenario.jumboPlan goal sessionAvailableFrom must be an integer in 1..${params.sessionCount} (got ${goal.sessionAvailableFrom})`, - ); - } - } - } - - return { - ...params, - createdAt: new Date().toISOString(), - }; -} diff --git a/evals/src/domain/session.ts b/evals/src/domain/session.ts deleted file mode 100644 index e5be8123..00000000 --- a/evals/src/domain/session.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * A single agent interaction period within a run, plus the run-level record. - * A SessionRecord captures everything observed during one harness exec: the - * prompts, the produced files, token usage, Jumbo state snapshots, and timings. - */ - -import type { JumboLifecycleAudit } from './jumbo-lifecycle.js'; -import type { JumboMemorySnapshot } from './jumbo-memory.js'; -import type { SessionPhaseTimings } from './timing.js'; -import type { TamperEvent } from './tamper.js'; -import type { WorkspaceSnapshot } from './workspace.js'; - -export interface SessionRecord { - readonly id: string; - readonly scenarioId: string; - readonly sessionNumber: number; - readonly harness: string; - readonly variant?: 'jumbo' | 'baseline'; - readonly scenarioPrompt?: string; - readonly effectivePrompt?: string; - readonly deliveredContext?: string; - readonly agentOutput: string; - readonly filesModified: readonly string[]; - readonly transcript: string; - readonly jumboLifecycleAudit?: JumboLifecycleAudit; - readonly jumboMemorySnapshotBefore?: JumboMemorySnapshot; - readonly jumboMemorySnapshot?: JumboMemorySnapshot; - readonly workspaceSnapshot?: WorkspaceSnapshot; - readonly inputTokens?: number; - readonly outputTokens?: number; - readonly startedAt: string; - readonly completedAt: string; - readonly phaseTimings?: SessionPhaseTimings; - readonly tampered: boolean; - readonly tamperLog: readonly TamperEvent[]; -} - -export interface EvalRunRecord { - readonly runId: string; - readonly scenarioId: string; - readonly harnesses: readonly string[]; - readonly sessionCount: number; - readonly startedAt: string; - readonly completedAt?: string; - readonly status: 'running' | 'completed' | 'failed'; -} - -export function createSessionRecord(params: { - id: string; - scenarioId: string; - sessionNumber: number; - harness: string; - variant?: 'jumbo' | 'baseline'; - scenarioPrompt?: string; - effectivePrompt?: string; - deliveredContext?: string; - agentOutput: string; - filesModified: readonly string[]; - transcript: string; - jumboLifecycleAudit?: JumboLifecycleAudit; - jumboMemorySnapshotBefore?: JumboMemorySnapshot; - jumboMemorySnapshot?: JumboMemorySnapshot; - workspaceSnapshot?: WorkspaceSnapshot; - inputTokens?: number; - outputTokens?: number; - startedAt: string; - completedAt: string; - phaseTimings?: SessionPhaseTimings; - tampered?: boolean; - tamperLog?: readonly TamperEvent[]; -}): SessionRecord { - if (!params.id) throw new Error('SessionRecord requires an id'); - if (!params.scenarioId) throw new Error('SessionRecord requires a scenarioId'); - if (params.sessionNumber < 1) throw new Error('SessionRecord requires sessionNumber >= 1'); - if (!params.harness) throw new Error('SessionRecord requires a harness'); - - return { - ...params, - tampered: params.tampered ?? false, - tamperLog: params.tamperLog ?? [], - }; -} diff --git a/evals/src/domain/tamper-provenance.ts b/evals/src/domain/tamper-provenance.ts deleted file mode 100644 index 33cf452b..00000000 --- a/evals/src/domain/tamper-provenance.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Shared "fold tamper provenance into a result" behavior. Centralised here so - * each result type (TestResult, ComparisonResult, and any future ones) derives - * its `tampered` flag and merged `tamperLog` the same way instead of - * re-implementing it in every factory. - */ -import type { TamperEvent } from './tamper.js'; - -export interface TamperProvenance { - readonly tampered: boolean; - readonly tamperLog: readonly TamperEvent[]; -} - -/** Anything that carries its own tamper provenance (a session record, an arm result). */ -export interface TamperBearing { - readonly tampered: boolean; - readonly tamperLog: readonly TamperEvent[]; -} - -/** - * Folds an optional seed (caller-supplied `tampered`/`tamperLog`) together with - * any number of tamper-bearing sources into a single provenance. A result is - * tampered if the seed says so or any source is tampered; the log is the seed - * log followed by each source's log, in order. - */ -export function mergeTamperProvenance( - seed: { readonly tampered?: boolean; readonly tamperLog?: readonly TamperEvent[] }, - sources: readonly TamperBearing[], -): TamperProvenance { - const tampered = (seed.tampered ?? false) || sources.some((s) => s.tampered); - const tamperLog: readonly TamperEvent[] = [ - ...(seed.tamperLog ?? []), - ...sources.flatMap((s) => s.tamperLog), - ]; - return { tampered, tamperLog }; -} diff --git a/evals/src/domain/tamper.ts b/evals/src/domain/tamper.ts deleted file mode 100644 index 0f85b64f..00000000 --- a/evals/src/domain/tamper.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Operator tamper events and the run-control file used to drive pause/resume/ - * abort/inject-context during a live run. Tampered records are quarantined from - * aggregate scoring. - */ - -export type TamperAction = 'pause' | 'resume' | 'abort' | 'inject-context'; - -export interface TamperEvent { - readonly occurredAt: string; - readonly action: TamperAction; - readonly sessionNumber?: number; - readonly harness?: string; - readonly variant?: 'jumbo' | 'baseline'; - readonly payload?: string; - readonly operator?: string; -} - -export interface RunControlFile { - readonly runId: string; - readonly updatedAt: string; - readonly pendingActions: readonly TamperEvent[]; - readonly pauseRequested: boolean; - readonly abortRequested: boolean; -} diff --git a/evals/src/domain/timing.ts b/evals/src/domain/timing.ts deleted file mode 100644 index a61613cd..00000000 --- a/evals/src/domain/timing.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Per-phase wall-clock timings captured during a session. - */ - -export interface TimingSpan { - readonly startedAt: string; - readonly completedAt: string; - readonly elapsedMs: number; -} - -export interface SessionPhaseTimings { - readonly harnessExec: TimingSpan; - readonly lifecycleAudit?: TimingSpan; -} diff --git a/evals/src/domain/types.ts b/evals/src/domain/types.ts deleted file mode 100644 index d0c48716..00000000 --- a/evals/src/domain/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Back-compatibility barrel. The domain model was split into focused modules - * (scenario, jumbo-plan, jumbo-memory, jumbo-lifecycle, workspace, timing, - * tamper, session, heartbeat, result); this file preserves the historical - * `domain/types.js` import path. Prefer importing from `./index.js`. - */ - -export * from './index.js'; diff --git a/evals/src/domain/validation.ts b/evals/src/domain/validation.ts deleted file mode 100644 index 883e84ab..00000000 --- a/evals/src/domain/validation.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Tiny shared validation helpers for domain factories, so required-field guards - * read uniformly and the messages stay consistent across result types. - */ - -/** Throws with `message` when `value` is missing/empty/falsy. */ -export function requireField(value: unknown, message: string): void { - if (!value) throw new Error(message); -} diff --git a/evals/src/domain/workspace.ts b/evals/src/domain/workspace.ts deleted file mode 100644 index 16a2f8a4..00000000 --- a/evals/src/domain/workspace.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * A captured snapshot of user-authored workspace files. The structural-assertion - * scorer evaluates retention against these artefacts. - */ - -export interface WorkspaceFileEntry { - readonly path: string; - readonly content: string; -} - -/** - * A content-free summary of Jumbo's own event log (`.jumbo/events/`), captured - * as scoring evidence (GOAL.md §"Workspace evidence: capturing `.jumbo/`"). - * The event log is the ground truth of what happened inside Jumbo; CLI command - * outputs are a derived view. Full event content is not required — file names - * and aggregate counts by event type are sufficient to verify that the agent - * registered entities during the session. - * - * Jumbo stores events as `.jumbo/events/<aggregateId>/<seq>.<EventType>.json`, - * so `countsByType` is keyed by the event type parsed from the file name (the - * stable, reproducible signal); aggregate ids are non-deterministic across runs - * and appear only in `fileNames` as an audit trail. - */ -export interface JumboEventSummary { - readonly capturedAt: string; - readonly aggregateCount: number; - readonly eventCount: number; - readonly countsByType: Readonly<Record<string, number>>; - readonly fileNames: readonly string[]; -} - -export interface WorkspaceSnapshot { - readonly capturedAt: string; - readonly files: readonly WorkspaceFileEntry[]; - readonly jumboEvents?: JumboEventSummary; -} diff --git a/evals/src/harness/claude-code-adapter.ts b/evals/src/harness/claude-code-adapter.ts deleted file mode 100644 index 24ad24b0..00000000 --- a/evals/src/harness/claude-code-adapter.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import type { ExecResult } from '../infrastructure/container-manager.js'; -import type { HarnessAdapter } from './harness-adapter.js'; - -/** - * HarnessAdapter for Claude Code CLI. - * Invokes via 'claude -p' (print mode) which reads the prompt from - * stdin when no positional prompt argument is provided. The eval - * harness pipes the effective prompt through LocalExecutor.exec's - * stdin option so multi-line content survives Windows cmd.exe. - */ -export class ClaudeCodeAdapter implements HarnessAdapter { - readonly name = 'claude-code'; - - buildCommand(): string[] { - // --dangerously-skip-permissions: in headless -p mode no human can approve - // tool use, so without it the agent cannot write files or run commands — - // the first smoke run showed both arms paralyzed ("unable to write files - // without permission approval"). Each eval session runs in an isolated - // throwaway temp workdir, which is the setting this flag exists for. It - // also makes tool access independent of .claude/settings.json, which - // `jumbo init` (>=3.12) overwrites, discarding the seeded allowlist. - return ['claude', '-p', '--output-format', 'json', '--dangerously-skip-permissions']; - } - - /** - * Seeds .claude/settings.json with a permissions.allow allowlist that - * permits Bash invocations starting with `jumbo` to run without - * interactive approval. `Bash(jumbo:*)` is Claude Code's prefix-match - * syntax; the bare `Bash(jumbo)` entry covers the no-arg invocation. - */ - async seedToolPermissions(workDir: string): Promise<void> { - const settingsDir = join(workDir, '.claude'); - await mkdir(settingsDir, { recursive: true }); - const settings = { - permissions: { - allow: [ - 'Bash(jumbo)', - 'Bash(jumbo:*)', - ], - }, - }; - await writeFile(join(settingsDir, 'settings.json'), `${JSON.stringify(settings, null, 2)}\n`, 'utf-8'); - } - - parseOutput(result: ExecResult): { - agentOutput: string; - filesModified: string[]; - transcript: string; - inputTokens?: number; - outputTokens?: number; - } { - const transcript = [result.stdout, result.stderr].filter(Boolean).join('\n---stderr---\n'); - - let agentOutput = result.stdout; - const filesModified: string[] = []; - let inputTokens: number | undefined; - let outputTokens: number | undefined; - - try { - const parsed = JSON.parse(result.stdout); - if (parsed.result) { - agentOutput = parsed.result; - } - if (Array.isArray(parsed.files_modified)) { - filesModified.push(...parsed.files_modified); - } - // Claude Code CLI JSON output includes usage stats - if (typeof parsed.input_tokens === 'number') { - inputTokens = parsed.input_tokens; - } - if (typeof parsed.output_tokens === 'number') { - outputTokens = parsed.output_tokens; - } - // Also check nested usage object - if (parsed.usage) { - if (typeof parsed.usage.input_tokens === 'number') { - inputTokens = parsed.usage.input_tokens; - } - if (typeof parsed.usage.output_tokens === 'number') { - outputTokens = parsed.usage.output_tokens; - } - } - } catch { - // Non-JSON output — use raw stdout as agent output - } - - return { agentOutput, filesModified, transcript, inputTokens, outputTokens }; - } -} diff --git a/evals/src/harness/codex-cli-adapter.ts b/evals/src/harness/codex-cli-adapter.ts deleted file mode 100644 index 348912ce..00000000 --- a/evals/src/harness/codex-cli-adapter.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import type { ExecResult } from '../infrastructure/container-manager.js'; -import type { HarnessAdapter } from './harness-adapter.js'; - -/** - * HarnessAdapter for OpenAI Codex CLI. - * Invokes via 'codex exec --json' and sends the prompt on stdin. - * Current Codex CLI JSON mode emits newline-delimited event objects. - * - * Older fixtures may still use the previous single-object JSON shape: - * { - * "response": "...", - * "files_modified": ["..."], - * "usage": { "prompt_tokens": N, "completion_tokens": N } - * } - */ -export class CodexCliAdapter implements HarnessAdapter { - readonly name = 'codex-cli'; - - buildCommand(): string[] { - return [ - 'codex', - '--ask-for-approval', - 'never', - '--sandbox', - 'workspace-write', - 'exec', - '--json', - '--skip-git-repo-check', - ]; - } - - /** - * Seeds .codex/config.toml so Codex runs shell commands without - * interactive approval (the closest Codex analog to Claude Code's - * `Bash(jumbo:*)` allowlist — Codex OSS config has no per-command - * allowlist, so the entire shell tool is unblocked at the policy - * level). approval_policy="never" suppresses prompts; sandbox_mode - * ="workspace-write" allows writes inside the workdir which is the - * eval scope. - */ - async seedToolPermissions(workDir: string): Promise<void> { - const configDir = join(workDir, '.codex'); - await mkdir(configDir, { recursive: true }); - const toml = [ - 'approval_policy = "never"', - 'sandbox_mode = "workspace-write"', - '', - ].join('\n'); - await writeFile(join(configDir, 'config.toml'), toml, 'utf-8'); - } - - parseOutput(result: ExecResult): { - agentOutput: string; - filesModified: string[]; - transcript: string; - inputTokens?: number; - outputTokens?: number; - } { - const transcript = [result.stdout, result.stderr].filter(Boolean).join('\n---stderr---\n'); - - let agentOutput = result.stdout; - const filesModified: string[] = []; - let inputTokens: number | undefined; - let outputTokens: number | undefined; - - try { - const parsed = JSON.parse(result.stdout); - const single = parseCodexJsonValue(parsed); - agentOutput = single.agentOutput ?? agentOutput; - filesModified.push(...single.filesModified); - inputTokens = single.inputTokens; - outputTokens = single.outputTokens; - } catch { - const jsonl = parseCodexJsonLines(result.stdout); - if (jsonl) { - agentOutput = jsonl.agentOutput ?? agentOutput; - filesModified.push(...jsonl.filesModified); - inputTokens = jsonl.inputTokens; - outputTokens = jsonl.outputTokens; - } - } - - return { agentOutput, filesModified, transcript, inputTokens, outputTokens }; - } -} - -interface ParsedCodexOutput { - readonly agentOutput?: string; - readonly filesModified: readonly string[]; - readonly inputTokens?: number; - readonly outputTokens?: number; -} - -function parseCodexJsonLines(stdout: string): ParsedCodexOutput | undefined { - const values = stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map(parseJsonLine) - .filter((value): value is unknown => value !== undefined); - - if (values.length === 0) return undefined; - - return mergeCodexValues(values); -} - -function parseJsonLine(line: string): unknown | undefined { - try { - return JSON.parse(line); - } catch { - return undefined; - } -} - -function parseCodexJsonValue(value: unknown): ParsedCodexOutput { - if (Array.isArray(value)) { - return mergeCodexValues(value); - } - return mergeCodexValues([value]); -} - -function mergeCodexValues(values: readonly unknown[]): ParsedCodexOutput { - const filesModified: string[] = []; - let agentOutput: string | undefined; - let inputTokens: number | undefined; - let outputTokens: number | undefined; - - for (const value of values) { - const files = extractStringArray(value, 'files_modified'); - if (files) filesModified.push(...files); - - const tokens = extractTokenUsage(value); - inputTokens = tokens.inputTokens ?? inputTokens; - outputTokens = tokens.outputTokens ?? outputTokens; - - const text = extractAgentText(value); - if (text !== undefined && text.length > 0) { - agentOutput = text; - } - } - - return { - agentOutput, - filesModified: [...new Set(filesModified)], - inputTokens, - outputTokens, - }; -} - -function extractAgentText(value: unknown): string | undefined { - if (!isRecord(value)) return undefined; - - const direct = firstString(value, ['response', 'result', 'agent_output', 'final_message']); - if (direct !== undefined) return direct; - - const type = typeof value.type === 'string' ? value.type : ''; - if (type.includes('message') || type.includes('completed')) { - const eventText = firstString(value, ['message', 'text']); - if (eventText !== undefined) return eventText; - - const contentText = extractContentText(value.content); - if (contentText !== undefined) return contentText; - } - - if (isRecord(value.item)) { - const role = typeof value.item.role === 'string' ? value.item.role : ''; - const itemType = typeof value.item.type === 'string' ? value.item.type : ''; - if (role === 'assistant' || itemType.includes('message')) { - const itemText = firstString(value.item, ['message', 'text']); - if (itemText !== undefined) return itemText; - return extractContentText(value.item.content); - } - } - - return undefined; -} - -function extractContentText(content: unknown): string | undefined { - if (typeof content === 'string') return content; - if (!Array.isArray(content)) return undefined; - - const parts = content - .map((part) => { - if (typeof part === 'string') return part; - if (!isRecord(part)) return undefined; - return firstString(part, ['text', 'output_text']); - }) - .filter((part): part is string => part !== undefined && part.length > 0); - - return parts.length > 0 ? parts.join('') : undefined; -} - -function extractTokenUsage(value: unknown): { inputTokens?: number; outputTokens?: number } { - if (!isRecord(value)) return {}; - - const usage = isRecord(value.usage) ? value.usage : value; - return { - inputTokens: firstNumber(usage, ['prompt_tokens', 'input_tokens']), - outputTokens: firstNumber(usage, ['completion_tokens', 'output_tokens']), - }; -} - -function extractStringArray(value: unknown, key: string): string[] | undefined { - if (!isRecord(value)) return undefined; - const candidate = value[key]; - if (!Array.isArray(candidate)) return undefined; - return candidate.filter((item): item is string => typeof item === 'string'); -} - -function firstString(record: Record<string, unknown>, keys: readonly string[]): string | undefined { - for (const key of keys) { - const value = record[key]; - if (typeof value === 'string') return value; - } - return undefined; -} - -function firstNumber(record: Record<string, unknown>, keys: readonly string[]): number | undefined { - for (const key of keys) { - const value = record[key]; - if (typeof value === 'number') return value; - } - return undefined; -} - -function isRecord(value: unknown): value is Record<string, unknown> { - return typeof value === 'object' && value !== null; -} diff --git a/evals/src/harness/gemini-cli-adapter.ts b/evals/src/harness/gemini-cli-adapter.ts deleted file mode 100644 index d879c38e..00000000 --- a/evals/src/harness/gemini-cli-adapter.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import type { ExecResult } from '../infrastructure/container-manager.js'; -import type { HarnessAdapter } from './harness-adapter.js'; - -/** - * HarnessAdapter for Google Gemini CLI. - * Invokes via 'gemini --json' which accepts a prompt - * and returns JSON output with the response and metadata. - * - * Gemini CLI output format (JSON mode): - * { - * "text": "...", - * "files_modified": ["..."], - * "usage_metadata": { "prompt_token_count": N, "candidates_token_count": N } - * } - */ -export class GeminiCliAdapter implements HarnessAdapter { - readonly name = 'gemini-cli'; - - buildCommand(): string[] { - return ['gemini', '--json']; - } - - /** - * Seeds .gemini/settings.json with a tools.allowed entry that bypasses - * confirmation for shell commands beginning with `jumbo`. Gemini's - * `run_shell_command(<prefix>)` is the per-command-prefix allowlist - * syntax — equivalent in spirit to Claude Code's `Bash(jumbo:*)`. - * defaultApprovalMode="auto_edit" is the file-level policy that does - * not require interactive confirmation for tool invocations covered - * by tools.allowed. tools.core is intentionally NOT set: doing so - * would disable every other built-in tool. - */ - async seedToolPermissions(workDir: string): Promise<void> { - const settingsDir = join(workDir, '.gemini'); - await mkdir(settingsDir, { recursive: true }); - const settings = { - general: { - defaultApprovalMode: 'auto_edit', - }, - tools: { - allowed: [ - 'run_shell_command(jumbo)', - ], - }, - }; - await writeFile(join(settingsDir, 'settings.json'), `${JSON.stringify(settings, null, 2)}\n`, 'utf-8'); - } - - parseOutput(result: ExecResult): { - agentOutput: string; - filesModified: string[]; - transcript: string; - inputTokens?: number; - outputTokens?: number; - } { - const transcript = [result.stdout, result.stderr].filter(Boolean).join('\n---stderr---\n'); - - let agentOutput = result.stdout; - const filesModified: string[] = []; - let inputTokens: number | undefined; - let outputTokens: number | undefined; - - try { - const parsed = JSON.parse(result.stdout); - - if (parsed.text) { - agentOutput = parsed.text; - } - - if (Array.isArray(parsed.files_modified)) { - filesModified.push(...parsed.files_modified); - } - - // Gemini uses usage_metadata with prompt_token_count / candidates_token_count - if (parsed.usage_metadata) { - if (typeof parsed.usage_metadata.prompt_token_count === 'number') { - inputTokens = parsed.usage_metadata.prompt_token_count; - } - if (typeof parsed.usage_metadata.candidates_token_count === 'number') { - outputTokens = parsed.usage_metadata.candidates_token_count; - } - } - - // Also check top-level fields - if (typeof parsed.prompt_token_count === 'number') { - inputTokens = parsed.prompt_token_count; - } - if (typeof parsed.candidates_token_count === 'number') { - outputTokens = parsed.candidates_token_count; - } - } catch { - // Non-JSON output — use raw stdout as agent output - } - - return { agentOutput, filesModified, transcript, inputTokens, outputTokens }; - } -} diff --git a/evals/src/harness/harness-adapter.ts b/evals/src/harness/harness-adapter.ts deleted file mode 100644 index 3d990399..00000000 --- a/evals/src/harness/harness-adapter.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { ExecResult } from '../infrastructure/container-manager.js'; - -/** - * Interface for agent harness adapters. - * Each adapter knows how to invoke its CLI, parse the output, and - * return argv that does NOT include the prompt — the prompt is delivered - * to the child process via stdin (see LocalExecutor.exec). This avoids - * Windows cmd.exe truncating multi-line argv at the first newline. - */ -export interface HarnessAdapter { - readonly name: string; - - buildCommand(): string[]; - - parseOutput(result: ExecResult): { - agentOutput: string; - filesModified: string[]; - transcript: string; - inputTokens?: number; - outputTokens?: number; - }; - - /** - * Writes the per-adapter permission/config artifact into a workdir so - * the agent CLI can invoke the `jumbo` binary during a session without - * an interactive approval prompt. Each adapter owns the knowledge of - * its native config path and shape (e.g., Claude Code's - * .claude/settings.json, Gemini's .gemini/settings.json, - * Codex's .codex/config.toml). Idempotent: safe to call on a workdir - * that already contains the artifact. - * - * PATH inheritance is the runner's responsibility (LocalExecutor - * forwards process.env). This method only configures the CLI's - * permission layer. - */ - seedToolPermissions(workDir: string): Promise<void>; -} diff --git a/evals/src/harness/index.ts b/evals/src/harness/index.ts deleted file mode 100644 index 344e64f8..00000000 --- a/evals/src/harness/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { HarnessAdapter } from './harness-adapter.js'; -export { ClaudeCodeAdapter } from './claude-code-adapter.js'; -export { CodexCliAdapter } from './codex-cli-adapter.js'; -export { GeminiCliAdapter } from './gemini-cli-adapter.js'; diff --git a/evals/src/index.ts b/evals/src/index.ts deleted file mode 100644 index 5e3e7103..00000000 --- a/evals/src/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -export { createTestScenario, createSessionRecord, createTestResult, createComparisonResult } from './domain/index.js'; -export type { DisruptionType, Disruption, JumboMemoryKind, ExpectedJumboMemoryCapture, JumboMemoryEntity, JumboMemoryCommandResult, JumboMemorySnapshot, TestScenario, SessionRecord, TestResult, DimensionScore, PerSessionScore, ComparisonResult } from './domain/index.js'; - -export { JsonResultStore } from './storage/index.js'; -export type { ResultStore } from './storage/index.js'; - -export { LocalExecutor } from './infrastructure/index.js'; -export type { ExecResult } from './infrastructure/index.js'; - -/** @deprecated Use LocalExecutor instead */ -export { ContainerManager } from './infrastructure/index.js'; -export type { ContainerConfig } from './infrastructure/index.js'; - -export { ClaudeCodeAdapter, CodexCliAdapter, GeminiCliAdapter } from './harness/index.js'; -export type { HarnessAdapter } from './harness/index.js'; - -export { scoreFileAccuracy, scoreKnowledgeRetention, scoreKnowledgeRetentionTimeline, scoreDisruptionRecovery, scoreDisruptionRecoveryTimeline, scoreJumboMemoryCapture, scoreJumboMemoryCaptureTimeline, baselineJumboMemoryCaptureScore, scoreTokenEfficiency, compareTokenEfficiency, tokenUsageTimeline, scoreWithJudge, scoreAllJudgeDimensions, validateJudgeConfig, parseJudgeResponse, ALL_RUBRICS, CONSISTENCY_RUBRIC, ERROR_CORRECTION_RUBRIC, ARCHITECTURAL_QUALITY_RUBRIC, buildJudgePrompt } from './scoring/index.js'; -export type { JudgeConfig, JudgeFn, JudgeResponse, JudgeQuestionScore, Rubric, RubricQuestion, ScalePoint } from './scoring/index.js'; - -export { runSession } from './run-session.js'; -export { runABComparison } from './ab-runner.js'; -export type { ABRunConfig } from './ab-runner.js'; - -export { formatComparisonOutput, formatCrossHarnessComparison, computeDivergenceCurve, computeLiftPercentages, detectDivergenceOnset, analyzeDisruptionImpact, aggregateHarnessLifts, extractMemoryCaptureEvidence, generateFullReport, formatFullReport, exportReportAsJson, parseJsonReport } from './output/index.js'; -export type { DivergencePoint, LiftResult, DivergenceOnset, DisruptionImpact, HarnessLiftSummary, MemoryCaptureEvidence, FullReport } from './output/index.js'; - -export { createProgram } from './cli/index.js'; -export { handleScenarioCreate, formatScenarioList, validateHarnesses, formatScoreOutput, filterReportByDimensions, filterComparisonsByHarness, formatStatusOutput } from './cli/commands/index.js'; diff --git a/evals/src/infrastructure/container-manager.ts b/evals/src/infrastructure/container-manager.ts deleted file mode 100644 index ef98414d..00000000 --- a/evals/src/infrastructure/container-manager.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); - -export interface ContainerConfig { - readonly image: string; - readonly name: string; - readonly envVars?: Readonly<Record<string, string>>; - readonly workspaceVolume?: string; -} - -export interface ExecResult { - readonly stdout: string; - readonly stderr: string; - readonly exitCode: number; -} - -/** - * @deprecated Use LocalExecutor instead. Docker containers cannot authenticate - * agent CLIs (claude, codex, gemini) which require host-level OAuth/config files. - * - * Manages Docker container lifecycle for eval workspaces. - * Each test run gets a persistent container; sessions are - * fresh harness invocations inside the same container. - */ -export class ContainerManager { - async create(config: ContainerConfig): Promise<string> { - const args = [ - 'create', - '--name', config.name, - '--interactive', - ]; - - if (config.envVars) { - for (const [key, value] of Object.entries(config.envVars)) { - args.push('--env', `${key}=${value}`); - } - } - - if (config.workspaceVolume) { - args.push('--volume', `${config.workspaceVolume}:/workspace`); - args.push('--workdir', '/workspace'); - } - - args.push(config.image); - - const { stdout } = await execFileAsync('docker', args); - return stdout.trim(); - } - - async start(containerName: string): Promise<void> { - await execFileAsync('docker', ['start', containerName]); - } - - async exec(containerName: string, command: string[]): Promise<ExecResult> { - try { - const { stdout, stderr } = await execFileAsync('docker', [ - 'exec', containerName, ...command, - ], { maxBuffer: 10 * 1024 * 1024 }); - - return { stdout, stderr, exitCode: 0 }; - } catch (err: unknown) { - if (err && typeof err === 'object' && 'stdout' in err && 'stderr' in err && 'code' in err) { - const execErr = err as { stdout: string; stderr: string; code: number }; - return { - stdout: execErr.stdout ?? '', - stderr: execErr.stderr ?? '', - exitCode: execErr.code, - }; - } - throw err; - } - } - - async stop(containerName: string): Promise<void> { - await execFileAsync('docker', ['stop', containerName]); - } - - async destroy(containerName: string): Promise<void> { - try { - await execFileAsync('docker', ['rm', '--force', containerName]); - } catch { - // Container may already be removed - } - } - - async isRunning(containerName: string): Promise<boolean> { - try { - const { stdout } = await execFileAsync('docker', [ - 'inspect', '--format', '{{.State.Running}}', containerName, - ]); - return stdout.trim() === 'true'; - } catch { - return false; - } - } -} diff --git a/evals/src/infrastructure/heartbeat-writer.ts b/evals/src/infrastructure/heartbeat-writer.ts deleted file mode 100644 index 60370f5b..00000000 --- a/evals/src/infrastructure/heartbeat-writer.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { HarnessHeartbeat, RunHeartbeat, SessionHeartbeat } from '../domain/types.js'; -import type { ResultStore } from '../storage/result-store.js'; - -export interface HeartbeatWriter { - writeHeartbeat(runId: string, heartbeat: RunHeartbeat): Promise<void>; -} - -export class StoreHeartbeatWriter implements HeartbeatWriter { - private heartbeat: RunHeartbeat; - - constructor( - private readonly store: Pick<ResultStore, 'writeHeartbeat'>, - params: { - readonly runId: string; - readonly scenarioId: string; - readonly harnesses: readonly string[]; - readonly sessionCount: number; - }, - ) { - this.heartbeat = { - runId: params.runId, - scenarioId: params.scenarioId, - updatedAt: new Date().toISOString(), - harnesses: params.harnesses.flatMap((harness) => [ - createHarnessHeartbeat(harness, 'jumbo', params.sessionCount), - createHarnessHeartbeat(harness, 'baseline', params.sessionCount), - ]), - }; - } - - async writeHeartbeat(runId: string, heartbeat: RunHeartbeat): Promise<void> { - this.heartbeat = mergeHeartbeat(this.heartbeat, heartbeat); - await this.store.writeHeartbeat(runId, this.heartbeat); - } - - async initialize(): Promise<void> { - await this.store.writeHeartbeat(this.heartbeat.runId, this.heartbeat); - } -} - -export function createHarnessHeartbeat( - harness: string, - variant: 'jumbo' | 'baseline', - sessionCount: number, -): HarnessHeartbeat { - return { - harness, - variant, - sessions: Array.from({ length: sessionCount }, (_, index) => ({ - sessionNumber: index + 1, - status: 'pending', - })), - }; -} - -export function mergeHeartbeat(current: RunHeartbeat, update: RunHeartbeat): RunHeartbeat { - const byKey = new Map(current.harnesses.map((heartbeat) => [harnessKey(heartbeat), heartbeat])); - for (const harness of update.harnesses) { - byKey.set(harnessKey(harness), mergeHarnessHeartbeat(byKey.get(harnessKey(harness)), harness)); - } - - return { - runId: current.runId, - scenarioId: current.scenarioId, - updatedAt: update.updatedAt, - harnesses: [...byKey.values()], - }; -} - -function mergeHarnessHeartbeat(current: HarnessHeartbeat | undefined, update: HarnessHeartbeat): HarnessHeartbeat { - if (!current) return update; - const sessions = new Map(current.sessions.map((session) => [session.sessionNumber, session])); - for (const session of update.sessions) { - sessions.set(session.sessionNumber, session); - } - return { - harness: update.harness, - variant: update.variant, - sessions: [...sessions.values()].sort((a, b) => a.sessionNumber - b.sessionNumber), - }; -} - -function harnessKey(heartbeat: HarnessHeartbeat): string { - return `${heartbeat.harness}:${heartbeat.variant}`; -} - -export function buildHeartbeatUpdate(params: { - readonly runId: string; - readonly scenarioId: string; - readonly harness: string; - readonly variant: 'jumbo' | 'baseline'; - readonly session: SessionHeartbeat; -}): RunHeartbeat { - return { - runId: params.runId, - scenarioId: params.scenarioId, - updatedAt: new Date().toISOString(), - harnesses: [{ - harness: params.harness, - variant: params.variant, - sessions: [params.session], - }], - }; -} diff --git a/evals/src/infrastructure/index.ts b/evals/src/infrastructure/index.ts deleted file mode 100644 index b9a7f062..00000000 --- a/evals/src/infrastructure/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { LocalExecutor } from './local-executor.js'; -export { StoreHeartbeatWriter } from './heartbeat-writer.js'; -export type { HeartbeatWriter } from './heartbeat-writer.js'; -export type { ExecResult } from './container-manager.js'; - -/** @deprecated Use LocalExecutor instead — Docker containers cannot authenticate agent CLIs */ -export { ContainerManager } from './container-manager.js'; -export type { ContainerConfig } from './container-manager.js'; diff --git a/evals/src/infrastructure/local-executor.ts b/evals/src/infrastructure/local-executor.ts deleted file mode 100644 index e50f9573..00000000 --- a/evals/src/infrastructure/local-executor.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { spawn } from 'node:child_process'; -import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { delimiter, join, relative } from 'node:path'; -import type { ExecResult } from './container-manager.js'; -import type { JumboEventSummary, WorkspaceSnapshot, WorkspaceFileEntry } from '../domain/types.js'; - -/** - * Parses the event type from a Jumbo event file name of the form - * `<seq>.<EventType>.json` (e.g. `000001.DecisionAddedEvent.json`). Names that - * don't match are bucketed as 'unknown' rather than dropped. - */ -function parseJumboEventType(fileName: string): string { - const match = /^\d+\.(.+)\.json$/i.exec(fileName); - return match ? match[1] : 'unknown'; -} - -/** - * Pure: builds a JumboEventSummary from event file paths relative to - * `.jumbo/events/` (e.g. `agg-1/000001.DecisionAddedEvent.json`). The first - * path segment is the aggregate id; the event type is parsed from the file - * name. Counts by type are the stable signal; aggregate ids are audit-only. - */ -export function summarizeJumboEvents( - relativeEventPaths: readonly string[], - capturedAt: string, -): JumboEventSummary { - const aggregates = new Set<string>(); - const countsByType: Record<string, number> = {}; - for (const rawPath of relativeEventPaths) { - const normalized = rawPath.replace(/\\/g, '/'); - const segments = normalized.split('/'); - aggregates.add(segments[0]); - const fileName = segments[segments.length - 1]; - const type = parseJumboEventType(fileName); - countsByType[type] = (countsByType[type] ?? 0) + 1; - } - return { - capturedAt, - aggregateCount: aggregates.size, - eventCount: relativeEventPaths.length, - countsByType, - fileNames: relativeEventPaths.map((p) => p.replace(/\\/g, '/')).sort(), - }; -} - -// MSVCRT-style quoting for a single argv element passed via cmd.exe. -// Wraps in double quotes when needed and escapes embedded quotes and the -// backslashes that precede them per the standard Windows argv parser rules. -function quoteForWindowsShell(arg: string): string { - if (arg.length === 0) return '""'; - if (!/[\s"&|<>^()!]/.test(arg)) return arg; - let escaped = ''; - let backslashes = 0; - for (const ch of arg) { - if (ch === '\\') { - backslashes++; - continue; - } - if (ch === '"') { - escaped += '\\'.repeat(backslashes * 2 + 1) + '"'; - } else { - escaped += '\\'.repeat(backslashes) + ch; - } - backslashes = 0; - } - escaped += '\\'.repeat(backslashes * 2); - return `"${escaped}"`; -} - -/** - * Executes harness CLI commands locally using child_process.spawn - * with temp directory isolation for each run variant. - * - * Replaces ContainerManager for environments where agent CLIs - * require host-level authentication (OAuth, config files in ~/). - */ -export class LocalExecutor { - /** - * Creates a temp working directory for an eval run. - * Each variant (Jumbo, baseline) gets its own directory. - */ - async createWorkDir(prefix: string = 'jumbo-eval-'): Promise<string> { - return mkdtemp(join(tmpdir(), prefix)); - } - - /** - * Executes a command in the given working directory. - * Returns stdout, stderr, and exit code. - * - * When `options.stdin` is provided, the bytes are written to the child's - * stdin and the stream is closed. This is the only safe channel for - * multi-line payloads on Windows: argv strings on Windows are routed - * through `cmd.exe /d /s /c "<joined>"`, and cmd.exe truncates the command - * string at the first embedded newline. Anything multi-line (Jumbo context - * wrappers, scenario continuation prompts, disruption injections) must go - * through stdin, not argv. - */ - async exec( - workDir: string, - command: string[], - options?: { stdin?: string; env?: Record<string, string | undefined> }, - ): Promise<ExecResult> { - const isWindows = process.platform === 'win32'; - const [cmd, ...args] = command; - const stdinPayload = options?.stdin; - const env = options?.env - ? { ...process.env, ...options.env } - : { ...process.env }; - - return new Promise<ExecResult>((resolve) => { - // On Windows, jumbo/claude/npm are .cmd shims that require shell: true to - // execute (Node CVE-2024-27980). When shell is enabled, spawn concatenates - // argv with spaces and offers no escaping — passing summaries with spaces, - // quotes, or JSON would word-split. So we pre-quote into a single command - // string. On POSIX, real argv works fine without a shell. - const child = isWindows - ? spawn([cmd, ...args].map(quoteForWindowsShell).join(' '), { - cwd: workDir, - env, - shell: true, - stdio: ['pipe', 'pipe', 'pipe'], - }) - : spawn(cmd, args, { - cwd: workDir, - env, - shell: false, - stdio: ['pipe', 'pipe', 'pipe'], - }); - - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; - - child.stdout.on('data', (chunk: Buffer) => stdoutChunks.push(chunk)); - child.stderr.on('data', (chunk: Buffer) => stderrChunks.push(chunk)); - - child.on('close', (code) => { - resolve({ - stdout: Buffer.concat(stdoutChunks).toString('utf-8'), - stderr: Buffer.concat(stderrChunks).toString('utf-8'), - exitCode: code ?? 1, - }); - }); - - child.on('error', (err) => { - resolve({ - stdout: '', - stderr: err.message, - exitCode: 1, - }); - }); - - if (stdinPayload !== undefined) { - child.stdin.on('error', () => { - // Child may close stdin before we finish writing; ignore EPIPE. - }); - child.stdin.end(stdinPayload, 'utf-8'); - } else { - child.stdin.end(); - } - }); - } - - /** - * Installs a fail-loud `jumbo` shim inside the given working directory and - * returns an env override whose PATH puts the shim ahead of the real - * binary. Used to enforce baseline-arm parity: the baseline agent must not - * be able to reach the real jumbo CLI, but any attempt to invoke it must - * fail loudly with a recognizable marker rather than silently succeeding - * against an unrelated binary. - */ - async installJumboShim(workDir: string): Promise<{ env: Record<string, string> }> { - const shimDir = join(workDir, '.eval-bin'); - await mkdir(shimDir, { recursive: true }); - const marker = 'ERROR: jumbo is not available in the baseline arm (eval shim)'; - if (process.platform === 'win32') { - const batch = `@echo off\r\necho ${marker} 1>&2\r\nexit /b 127\r\n`; - await writeFile(join(shimDir, 'jumbo.cmd'), batch); - await writeFile(join(shimDir, 'jumbo.bat'), batch); - } else { - const script = `#!/bin/sh\necho "${marker}" >&2\nexit 127\n`; - const target = join(shimDir, 'jumbo'); - await writeFile(target, script); - await chmod(target, 0o755); - } - const PATH = `${shimDir}${delimiter}${process.env.PATH ?? ''}`; - return { env: { PATH } }; - } - - /** - * Captures a snapshot of all readable text files in the working directory. - * Workspace content is the primary evidence source for scoring. - */ - async captureWorkspaceSnapshot(workDir: string): Promise<WorkspaceSnapshot> { - const files: WorkspaceFileEntry[] = []; - await this.walkDir(workDir, workDir, files); - const jumboEvents = await this.captureJumboEventSummary(workDir); - return { - capturedAt: new Date().toISOString(), - files, - ...(jumboEvents ? { jumboEvents } : {}), - }; - } - - /** - * Summarizes Jumbo's own event log (`.jumbo/events/`) as scoring evidence — - * file names and aggregate counts by event type, never event content. The - * `.jumbo` directory is deliberately excluded from the file-content walk - * (see SKIP_DIRS); this is the only place it is read. - * - * Returns undefined when there is no `.jumbo/events/` directory (the baseline - * arm), distinguishing "no Jumbo" from "Jumbo present but no events yet" - * (which yields a zero-count summary). - */ - private async captureJumboEventSummary(workDir: string): Promise<JumboEventSummary | undefined> { - const eventsDir = join(workDir, '.jumbo', 'events'); - const relativePaths: string[] = []; - try { - await this.collectEventFiles(eventsDir, eventsDir, relativePaths); - } catch { - return undefined; - } - return summarizeJumboEvents(relativePaths, new Date().toISOString()); - } - - private async collectEventFiles(rootDir: string, currentDir: string, out: string[]): Promise<void> { - const entries = await readdir(currentDir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(currentDir, entry.name); - if (entry.isDirectory()) { - await this.collectEventFiles(rootDir, fullPath, out); - } else if (entry.isFile()) { - out.push(relative(rootDir, fullPath).replace(/\\/g, '/')); - } - } - } - - /** - * Returns the sorted set of file paths added or whose content changed - * between two workspace snapshots. Files removed in `after` are ignored — - * filesModified semantics target what the agent created or edited. - * Pure: deterministic given inputs, no I/O. - */ - static diffWorkspaceSnapshots( - before: WorkspaceSnapshot | undefined, - after: WorkspaceSnapshot, - ): string[] { - const beforeMap = new Map<string, string>(); - for (const f of before?.files ?? []) { - beforeMap.set(f.path, f.content); - } - const changed: string[] = []; - for (const f of after.files) { - const prior = beforeMap.get(f.path); - if (prior === undefined || prior !== f.content) { - changed.push(f.path); - } - } - return changed.sort(); - } - - // Filenames and patterns that may contain secrets — never captured in snapshots. - private static readonly SECRET_PRONE_PATTERNS: RegExp[] = [ - /^\.env(\.[^/]*)?$/, - /\.(key|pem|p12|pfx|crt|cert|jks|keystore)$/i, - /^(credentials?|secrets?|api[_-]?keys?)(\.(json|yaml|yml|toml|ini|txt))?$/i, - /^(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$/, - ]; - - private isSecretProneFilename(filename: string): boolean { - return LocalExecutor.SECRET_PRONE_PATTERNS.some((re) => re.test(filename)); - } - - private async walkDir(rootDir: string, currentDir: string, files: WorkspaceFileEntry[]): Promise<void> { - const SKIP_DIRS = new Set(['.git', 'node_modules', '.jumbo', 'dist', '.cache']); - const MAX_FILE_BYTES = 100_000; - - let entries; - try { - entries = await readdir(currentDir, { withFileTypes: true }); - } catch { - return; - } - - for (const entry of entries) { - const fullPath = join(currentDir, entry.name); - const relativePath = relative(rootDir, fullPath).replace(/\\/g, '/'); - - if (entry.isDirectory()) { - if (!SKIP_DIRS.has(entry.name)) { - await this.walkDir(rootDir, fullPath, files); - } - } else if (entry.isFile()) { - if (this.isSecretProneFilename(entry.name)) { - continue; - } - try { - const info = await stat(fullPath); - if (info.size <= MAX_FILE_BYTES) { - const content = await readFile(fullPath, 'utf-8'); - files.push({ path: relativePath, content }); - } - } catch { - // skip unreadable or binary files - } - } - } - } - - /** - * Removes a temp working directory and all its contents. - */ - async cleanup(workDir: string): Promise<void> { - try { - await rm(workDir, { recursive: true, force: true }); - } catch { - // Directory may already be removed - } - } -} diff --git a/evals/src/output/comparison-display.ts b/evals/src/output/comparison-display.ts deleted file mode 100644 index b5b5dc7b..00000000 --- a/evals/src/output/comparison-display.ts +++ /dev/null @@ -1,195 +0,0 @@ -import type { ComparisonResult, DimensionScore, Disruption } from '../domain/types.js'; - -/** - * Pure function: takes a ComparisonResult and returns formatted terminal output. - * No I/O — caller is responsible for printing. - */ -export function formatComparisonOutput(comparison: ComparisonResult, disruptions?: readonly Disruption[]): string { - const lines: string[] = []; - const divider = '═'.repeat(60); - const thinDivider = '─'.repeat(60); - - lines.push(divider); - lines.push(` A/B Comparison: ${comparison.scenarioId}`); - lines.push(` Harness: ${comparison.harness}`); - lines.push(divider); - lines.push(''); - - // Session summaries - lines.push(' JUMBO RUN (JUMBO_ENABLED=true)'); - lines.push(thinDivider); - for (const record of comparison.jumboResult.sessionRecords) { - lines.push(` Session ${record.sessionNumber}: ${record.filesModified.length} files modified`); - if (record.filesModified.length > 0) { - lines.push(` Files: ${record.filesModified.join(', ')}`); - } - } - lines.push(''); - - lines.push(' BASELINE RUN (JUMBO_ENABLED=false)'); - lines.push(thinDivider); - for (const record of comparison.baselineResult.sessionRecords) { - lines.push(` Session ${record.sessionNumber}: ${record.filesModified.length} files modified`); - if (record.filesModified.length > 0) { - lines.push(` Files: ${record.filesModified.join(', ')}`); - } - } - lines.push(''); - - // Scores side-by-side - lines.push(' SCORES'); - lines.push(divider); - lines.push( - padRight(' Dimension', 24) + - padRight('Jumbo', 12) + - padRight('Baseline', 12) + - 'Delta', - ); - lines.push(thinDivider); - - // Rendering convention for 'token-efficiency': Jumbo column holds the comparative - // efficiency ratio (positive = Jumbo used fewer tokens per quality point than - // baseline), Baseline column reads '(ref)' to mark it as the reference run, Delta - // is omitted. Raw totals are emitted on the following line for context. - for (let i = 0; i < comparison.jumboScores.length; i++) { - const js = comparison.jumboScores[i]; - const bs = comparison.baselineScores[i]; - const delta = comparison.deltas[i]; - - if (js.dimension === 'token-efficiency') { - lines.push( - padRight(` ${js.dimension}`, 24) + - padRight(formatSignedRatio(js.score), 12) + - padRight('(ref)', 12) + - '—', - ); - const totals = js.details ?? bs.details; - if (totals) { - lines.push(` └─ ${totals}`); - } - continue; - } - - lines.push( - padRight(` ${js.dimension}`, 24) + - padRight(formatScore(js), 12) + - padRight(formatScore(bs), 12) + - formatDelta(delta), - ); - } - - const jumboMemoryScore = comparison.jumboScores.find((score) => score.dimension === 'jumbo-memory-capture'); - const baselineMemoryScore = comparison.baselineScores.find((score) => score.dimension === 'jumbo-memory-capture'); - if (jumboMemoryScore || baselineMemoryScore) { - lines.push(''); - lines.push(' JUMBO MEMORY CAPTURE EVIDENCE'); - lines.push(thinDivider); - if (jumboMemoryScore?.details) { - lines.push(` Jumbo: ${jumboMemoryScore.details}`); - } - if (baselineMemoryScore?.details) { - lines.push(` Baseline: ${baselineMemoryScore.details}`); - } - } - - const jumboProtocolScore = comparison.jumboScores.find((score) => score.dimension === 'protocol-adherence'); - const baselineProtocolScore = comparison.baselineScores.find((score) => score.dimension === 'protocol-adherence'); - if (jumboProtocolScore || baselineProtocolScore) { - lines.push(''); - lines.push(' PROTOCOL ADHERENCE EVIDENCE'); - lines.push(thinDivider); - if (jumboProtocolScore?.details) { - lines.push(` Jumbo: ${jumboProtocolScore.details}`); - } - if (baselineProtocolScore?.details) { - lines.push(` Baseline: ${baselineProtocolScore.details}`); - } - } - - // Per-session timeline (if multi-session) - if (comparison.jumboTimeline && comparison.baselineTimeline && comparison.jumboTimeline.length > 1) { - lines.push(''); - lines.push(' TIMELINE (per-session scores)'); - lines.push(divider); - - // Find all dimensions present in timeline - const dimensions = new Set<string>(); - for (const ps of comparison.jumboTimeline) { - for (const s of ps.scores) dimensions.add(s.dimension); - } - - for (const dim of dimensions) { - lines.push(` ${dim}`); - lines.push( - padRight(' Session', 12) + - padRight('Jumbo', 12) + - padRight('Baseline', 12) + - 'Delta', - ); - lines.push(thinDivider); - - for (let i = 0; i < comparison.jumboTimeline.length; i++) { - const jt = comparison.jumboTimeline[i]; - const bt = comparison.baselineTimeline[i]; - - // Mark disruption injection points - const sessionDisruptions = (disruptions ?? []).filter((d) => d.sessionNumber === jt.sessionNumber); - if (sessionDisruptions.length > 0) { - for (const d of sessionDisruptions) { - lines.push(` >>> [${d.type.toUpperCase()}] ${d.content.slice(0, 50)}${d.content.length > 50 ? '...' : ''}`); - } - } - - const jScore = jt?.scores.find((s) => s.dimension === dim); - const bScore = bt?.scores.find((s) => s.dimension === dim); - - if (jScore && bScore) { - if (dim === 'token-usage') { - // Raw totals: 'token-usage' carries totalTokens in `score`. Render absolute - // counts (jumbo / baseline / signed difference) — no fake 0–1 score. - const diff = jScore.score - bScore.score; - const sign = diff > 0 ? '+' : ''; - lines.push( - padRight(` ${jt.sessionNumber}`, 12) + - padRight(`${jScore.score}`, 12) + - padRight(`${bScore.score}`, 12) + - `${sign}${diff}`, - ); - } else { - const delta = Math.round((jScore.score - bScore.score) * 100) / 100; - const sign = delta > 0 ? '+' : ''; - lines.push( - padRight(` ${jt.sessionNumber}`, 12) + - padRight(formatScore(jScore), 12) + - padRight(formatScore(bScore), 12) + - `${sign}${delta.toFixed(2)}`, - ); - } - } - } - lines.push(''); - } - } - - lines.push(divider); - - return lines.join('\n'); -} - -function formatSignedRatio(value: number): string { - const sign = value > 0 ? '+' : ''; - return `${sign}${value.toFixed(2)}`; -} - -function formatScore(score: DimensionScore): string { - return `${score.score.toFixed(2)}/${score.maxScore.toFixed(2)}`; -} - -function formatDelta(delta: DimensionScore): string { - const sign = delta.score > 0 ? '+' : ''; - return `${sign}${delta.score.toFixed(2)}`; -} - -function padRight(str: string, width: number): string { - return str.length >= width ? str : str + ' '.repeat(width - str.length); -} diff --git a/evals/src/output/cross-harness-display.ts b/evals/src/output/cross-harness-display.ts deleted file mode 100644 index b5bd2f24..00000000 --- a/evals/src/output/cross-harness-display.ts +++ /dev/null @@ -1,137 +0,0 @@ -import type { ComparisonResult, DimensionScore } from '../domain/types.js'; - -/** - * Pure function: takes ComparisonResults from multiple harnesses for the same - * scenario and returns a formatted cross-harness comparison report. - * - * Shows side-by-side: for each dimension, the Jumbo lift (delta) per harness, - * revealing which harnesses benefit most from Jumbo. - */ -export function formatCrossHarnessComparison( - comparisons: readonly ComparisonResult[], -): string { - if (comparisons.length === 0) { - return 'No comparison results to display.'; - } - - if (comparisons.length === 1) { - return 'Cross-harness comparison requires results from at least 2 harnesses.'; - } - - // Verify all comparisons are for the same scenario - const scenarioIds = new Set(comparisons.map((c) => c.scenarioId)); - if (scenarioIds.size > 1) { - return 'Error: comparisons span multiple scenarios. Cross-harness comparison requires a single scenario.'; - } - - const lines: string[] = []; - const divider = '═'.repeat(72); - const thinDivider = '─'.repeat(72); - - lines.push(divider); - lines.push(` Cross-Harness Comparison: ${comparisons[0].scenarioId}`); - lines.push(` Harnesses: ${comparisons.map((c) => c.harness).join(', ')}`); - lines.push(divider); - lines.push(''); - - // Collect all dimensions across all comparisons - const allDimensions = new Set<string>(); - for (const comp of comparisons) { - for (const s of comp.jumboScores) allDimensions.add(s.dimension); - } - - // Header row: Dimension | harness1 Jumbo | harness1 Base | harness1 Δ | harness2 ... - const harnessNames = comparisons.map((c) => c.harness); - - // Jumbo lift table (delta per harness per dimension) - lines.push(' JUMBO LIFT BY HARNESS (delta = jumbo - baseline)'); - lines.push(thinDivider); - - // Header - const headerParts = [padRight(' Dimension', 24)]; - for (const name of harnessNames) { - headerParts.push(padRight(name, 16)); - } - lines.push(headerParts.join('')); - lines.push(thinDivider); - - // Rows by dimension - for (const dim of allDimensions) { - const parts = [padRight(` ${dim}`, 24)]; - - for (const comp of comparisons) { - const delta = comp.deltas.find((d) => d.dimension === dim); - if (delta) { - const sign = delta.score > 0 ? '+' : ''; - parts.push(padRight(`${sign}${delta.score.toFixed(2)}`, 16)); - } else { - parts.push(padRight(' n/a', 16)); - } - } - - lines.push(parts.join('')); - } - - lines.push(''); - - // Absolute scores table - lines.push(' ABSOLUTE SCORES (Jumbo run)'); - lines.push(thinDivider); - - const absHeader = [padRight(' Dimension', 24)]; - for (const name of harnessNames) { - absHeader.push(padRight(name, 16)); - } - lines.push(absHeader.join('')); - lines.push(thinDivider); - - for (const dim of allDimensions) { - const parts = [padRight(` ${dim}`, 24)]; - - for (const comp of comparisons) { - const score = comp.jumboScores.find((s) => s.dimension === dim); - if (score) { - parts.push(padRight(`${score.score.toFixed(2)}/${score.maxScore.toFixed(2)}`, 16)); - } else { - parts.push(padRight(' n/a', 16)); - } - } - - lines.push(parts.join('')); - } - - lines.push(''); - - // Summary: which harness benefits most from Jumbo - lines.push(' SUMMARY'); - lines.push(thinDivider); - - for (const comp of comparisons) { - const avgLift = comp.deltas.length > 0 - ? comp.deltas.reduce((sum, d) => sum + d.score, 0) / comp.deltas.length - : 0; - const sign = avgLift > 0 ? '+' : ''; - lines.push(` ${padRight(comp.harness, 20)} avg lift: ${sign}${avgLift.toFixed(3)}`); - } - - // Find best harness - const ranked = comparisons - .map((c) => ({ - harness: c.harness, - avgLift: c.deltas.length > 0 - ? c.deltas.reduce((sum, d) => sum + d.score, 0) / c.deltas.length - : 0, - })) - .sort((a, b) => b.avgLift - a.avgLift); - - lines.push(''); - lines.push(` Highest Jumbo lift: ${ranked[0].harness} (${ranked[0].avgLift > 0 ? '+' : ''}${ranked[0].avgLift.toFixed(3)})`); - - lines.push(divider); - - return lines.join('\n'); -} - -function padRight(str: string, width: number): string { - return str.length >= width ? str : str + ' '.repeat(width - str.length); -} diff --git a/evals/src/output/heartbeat-display.ts b/evals/src/output/heartbeat-display.ts deleted file mode 100644 index cf998d2a..00000000 --- a/evals/src/output/heartbeat-display.ts +++ /dev/null @@ -1,130 +0,0 @@ -import type { ReactElement } from 'react'; -import type { HarnessHeartbeat, RunHeartbeat, SessionHeartbeat } from '../domain/types.js'; - -export function formatHeartbeatDisplay(heartbeat: RunHeartbeat): string { - const lines: string[] = []; - lines.push(`Run: ${heartbeat.runId}`); - lines.push(`Scenario: ${heartbeat.scenarioId}`); - lines.push(`Updated: ${heartbeat.updatedAt}`); - lines.push('─'.repeat(80)); - lines.push('Summary'); - lines.push(`${pad('Harness', 16)} ${pad('Variant', 8)} ${pad('Done', 6)} ${pad('Failed', 6)} ${pad('Running', 8)} Current`); - - for (const harness of heartbeat.harnesses) { - const summary = summarizeHarness(harness); - lines.push([ - pad(harness.harness, 16), - pad(harness.variant, 8), - pad(String(summary.completed), 6), - pad(String(summary.failed), 6), - pad(String(summary.running), 8), - summary.current, - ].join(' ')); - } - - lines.push(''); - lines.push('Sessions'); - lines.push(`${pad('Harness', 16)} ${pad('Variant', 8)} ${pad('Session', 7)} ${pad('Status', 9)} ${pad('Phase', 12)} ${pad('Elapsed', 9)} Details`); - for (const harness of heartbeat.harnesses) { - for (const session of harness.sessions) { - lines.push(formatSessionRow(harness, session)); - } - } - - return lines.join('\n'); -} - -export function isHeartbeatComplete(heartbeat: RunHeartbeat): boolean { - return heartbeat.harnesses.every((harness) => - harness.sessions.every((session) => session.status === 'completed' || session.status === 'failed'), - ); -} - -export interface HeartbeatViewProps { - readonly heartbeat: RunHeartbeat | null; - readonly missingMessage?: string; -} - -/** - * Builds an ink view element for a heartbeat snapshot. - * Accepts the React and ink modules as dependencies so callers can use the - * dynamically-imported (ESM) instances without re-importing them here. - */ -export function createHeartbeatView( - React: typeof import('react'), - ink: typeof import('ink', { with: { 'resolution-mode': 'import' } }), - props: HeartbeatViewProps, -): ReactElement { - const { Box, Text } = ink; - if (!props.heartbeat) { - return React.createElement(Text, null, props.missingMessage ?? 'No heartbeat available.'); - } - return React.createElement( - Box, - { flexDirection: 'column' }, - React.createElement(Text, null, formatHeartbeatDisplay(props.heartbeat)), - ); -} - -function summarizeHarness(harness: HarnessHeartbeat): { - readonly completed: number; - readonly failed: number; - readonly running: number; - readonly current: string; -} { - const completed = harness.sessions.filter((session) => session.status === 'completed').length; - const failed = harness.sessions.filter((session) => session.status === 'failed').length; - const runningSessions = harness.sessions.filter((session) => session.status === 'running'); - const current = runningSessions[0] ?? harness.sessions.find((session) => session.status === 'failed'); - - return { - completed, - failed, - running: runningSessions.length, - current: current ? formatCurrentSession(current) : '-', - }; -} - -function formatCurrentSession(session: SessionHeartbeat): string { - const phase = session.phase ?? session.status; - const elapsed = activeElapsedMs(session); - const elapsedText = elapsed === undefined ? '' : ` ${Math.round(elapsed)}ms`; - const error = session.errorMessage ? ` ${session.errorMessage}` : ''; - return `s${session.sessionNumber} ${phase}${elapsedText}${error}`; -} - -function formatSessionRow(harness: HarnessHeartbeat, session: SessionHeartbeat): string { - const phase = session.phase ?? '-'; - const elapsed = activeElapsedMs(session) ?? completedElapsedMs(session); - const elapsedText = elapsed === undefined ? '-' : `${Math.round(elapsed)}ms`; - const details = session.errorMessage ?? ''; - return [ - pad(harness.harness, 16), - pad(harness.variant, 8), - pad(String(session.sessionNumber), 7), - pad(session.status, 9), - pad(phase, 12), - pad(elapsedText, 9), - details, - ].join(' '); -} - -function activeElapsedMs(session: SessionHeartbeat): number | undefined { - if (session.status !== 'running' || !session.startedAt) return undefined; - const start = Date.parse(session.startedAt); - if (Number.isNaN(start)) return undefined; - return Date.now() - start; -} - -function completedElapsedMs(session: SessionHeartbeat): number | undefined { - if (!session.startedAt || !session.completedAt) return undefined; - const start = Date.parse(session.startedAt); - const completed = Date.parse(session.completedAt); - if (Number.isNaN(start) || Number.isNaN(completed)) return undefined; - return Math.max(0, completed - start); -} - -function pad(value: string, width: number): string { - if (value.length >= width) return value; - return `${value}${' '.repeat(width - value.length)}`; -} diff --git a/evals/src/output/index.ts b/evals/src/output/index.ts deleted file mode 100644 index 150d02ca..00000000 --- a/evals/src/output/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { formatComparisonOutput } from './comparison-display.js'; -export { formatCrossHarnessComparison } from './cross-harness-display.js'; -export { formatHeartbeatDisplay } from './heartbeat-display.js'; -export { computeDivergenceCurve, computeLiftPercentages, detectDivergenceOnset, analyzeDisruptionImpact, aggregateHarnessLifts, extractMemoryCaptureEvidence, generateFullReport, formatFullReport } from './report-generator.js'; -export type { DivergencePoint, LiftResult, DivergenceOnset, DisruptionImpact, HarnessLiftSummary, MemoryCaptureEvidence, FullReport } from './report-generator.js'; -export { exportReportAsJson, parseJsonReport } from './json-report.js'; -export { formatReplicationReport } from './replication-display.js'; diff --git a/evals/src/output/json-report.ts b/evals/src/output/json-report.ts deleted file mode 100644 index 8908acc5..00000000 --- a/evals/src/output/json-report.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { FullReport } from './report-generator.js'; - -/** - * Converts a FullReport to a JSON string for external consumption. - * The JSON format is designed for documentation and marketing use. - * - * Pure function — no I/O. Caller is responsible for writing to disk. - */ -export function exportReportAsJson(report: FullReport): string { - const exportData = { - meta: { - format: 'jumbo-eval-report', - version: 1, - generatedAt: report.generatedAt, - scenarioId: report.scenarioId, - harnesses: report.harnesses, - }, - lift: { - byDimension: report.liftResults.map((l) => ({ - dimension: l.dimension, - jumboScore: l.jumboScore, - baselineScore: l.baselineScore, - absoluteLift: l.absoluteLift, - percentageLift: l.percentageLift, - })), - }, - divergence: { - curve: report.divergenceCurve.map((p) => ({ - session: p.sessionNumber, - dimension: p.dimension, - jumbo: p.jumboScore, - baseline: p.baselineScore, - delta: p.delta, - })), - onsets: report.divergenceOnsets.map((o) => ({ - dimension: o.dimension, - onsetSession: o.onsetSession, - threshold: o.threshold, - deltaAtOnset: o.deltaAtOnset, - })), - }, - disruptions: { - impacts: report.disruptionImpacts.map((i) => ({ - type: i.disruption.type, - session: i.disruption.sessionNumber, - content: i.disruption.content, - dimension: i.dimension, - deltaBefore: i.deltaBeforeDisruption, - deltaAfter: i.deltaAfterDisruption, - magnitude: i.impactMagnitude, - })), - }, - memoryCapture: { - evidence: report.memoryCaptureEvidence.map((e) => ({ - harness: e.harness, - jumboScore: e.jumboScore, - jumboDetails: e.jumboDetails, - baselineDetails: e.baselineDetails, - })), - }, - audit: { - trails: report.auditTrails.map((t) => ({ - harness: t.harness, - jumboContext: t.jumboContext, - baselineContext: t.baselineContext, - jumboFinalSnapshot: t.jumboFinalSnapshot, - baselineFinalSnapshot: t.baselineFinalSnapshot, - scoringEvidence: t.scoringEvidence, - })), - }, - harnessComparison: report.harnessAggregation.map((h) => ({ - harness: h.harness, - avgLift: h.avgLift, - dimensions: h.dimensionLifts.map((l) => ({ - dimension: l.dimension, - lift: l.absoluteLift, - liftPercent: l.percentageLift, - })), - })), - tamperedComparisons: report.tamperedComparisons.map((t) => ({ - comparisonId: t.comparisonId, - harness: t.harness, - tamperLog: t.tamperLog, - })), - }; - - return JSON.stringify(exportData, null, 2); -} - -/** - * Parses a JSON report string back into its structured form. - * Returns the raw parsed object for validation or re-processing. - */ -export function parseJsonReport(json: string): ReturnType<typeof JSON.parse> { - const parsed = JSON.parse(json); - - if (parsed.meta?.format !== 'jumbo-eval-report') { - throw new Error('Invalid report format: missing or incorrect meta.format'); - } - - if (typeof parsed.meta?.version !== 'number') { - throw new Error('Invalid report format: missing meta.version'); - } - - return parsed; -} diff --git a/evals/src/output/replication-display.ts b/evals/src/output/replication-display.ts deleted file mode 100644 index a53a0173..00000000 --- a/evals/src/output/replication-display.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { ReplicationReport } from '../domain/types.js'; - -/** - * Renders an Outcome 5 replication report as a fixed-width table: per dimension, - * the mean lift ± SD across replications, applicable replication count, - * t-statistic, and whether the lift clears the one-SD signal bar. - */ -function pad(value: string, width: number): string { - return value.length >= width ? value : value + ' '.repeat(width - value.length); -} - -function signed(value: number): string { - const fixed = value.toFixed(2); - return value >= 0 ? `+${fixed}` : fixed; -} - -export function formatReplicationReport(report: ReplicationReport): string { - const header = `REPLICATION REPORT — scenario ${report.scenarioId} / ${report.harness} — K=${report.k}`; - const tCritical = report.significance.tCriticalOneTailed05 === null - ? 'n/a' - : report.significance.tCriticalOneTailed05.toFixed(3); - const lines: string[] = [ - header, - '='.repeat(header.length), - `significance: ${report.significance.rule}; one-tailed alpha=0.05 t-critical (df=${Math.max(report.k - 1, 0)}) = ${tCritical}`, - report.significance.note, - '', - `${pad('dimension', 26)}${pad('mean lift', 12)}${pad('SD', 10)}${pad('applic.', 9)}${pad('t-stat', 10)}signal`, - '-'.repeat(73), - ]; - - if (report.dimensions.length === 0) { - lines.push('(no dimensions common to every replication)'); - } - - for (const d of report.dimensions) { - lines.push( - pad(d.dimension, 26) + - pad(signed(d.meanLift), 12) + - pad(d.sdLift.toFixed(2), 10) + - pad(`${d.applicableReplications}/${d.k}`, 9) + - pad(d.tStatistic.toFixed(2), 10) + - (d.isSignal ? 'SIGNAL' : 'none'), - ); - } - - return lines.join('\n'); -} diff --git a/evals/src/output/report-generator.ts b/evals/src/output/report-generator.ts deleted file mode 100644 index 94e9c373..00000000 --- a/evals/src/output/report-generator.ts +++ /dev/null @@ -1,601 +0,0 @@ -import type { ComparisonResult, DimensionScore, Disruption, SessionRecord, TamperEvent, WorkspaceSnapshot } from '../domain/types.js'; - -/** - * A single point on the divergence curve: per-session delta between Jumbo and baseline. - */ -export interface DivergencePoint { - readonly sessionNumber: number; - readonly dimension: string; - readonly jumboScore: number; - readonly baselineScore: number; - readonly delta: number; -} - -/** - * Lift percentage for a single dimension. - */ -export interface LiftResult { - readonly dimension: string; - readonly jumboScore: number; - readonly baselineScore: number; - readonly absoluteLift: number; - readonly percentageLift: number | null; // null when baseline is zero -} - -/** - * Identifies when amnesia impact becomes significant. - */ -export interface DivergenceOnset { - readonly dimension: string; - readonly onsetSession: number | null; // null if no significant divergence detected - readonly threshold: number; - readonly deltaAtOnset: number | null; -} - -/** - * Ranks disruptions by their impact on the delta. - */ -export interface DisruptionImpact { - readonly disruption: Disruption; - readonly dimension: string; - readonly deltaBeforeDisruption: number; - readonly deltaAfterDisruption: number; - readonly impactMagnitude: number; -} - -/** - * Aggregated lift per harness. - */ -export interface HarnessLiftSummary { - readonly harness: string; - readonly avgLift: number; - readonly dimensionLifts: readonly LiftResult[]; -} - -export interface MemoryCaptureEvidence { - readonly harness: string; - readonly jumboScore: number; - readonly jumboDetails?: string; - readonly baselineDetails?: string; -} - -export interface ProtocolAdherenceEvidence { - readonly harness: string; - readonly jumboScore: number; - readonly jumboDetails?: string; - readonly baselineDetails?: string; - readonly perSession: readonly { - readonly sessionNumber: number; - readonly score: number; - readonly details?: string; - }[]; -} - -/** - * Audit data tying a measured delta back to its inputs and outputs. - * - * For each variant the trail records the effective context delivered per - * session, the final workspace snapshot evidence, and the per-dimension - * scoring evidence. This is what makes a measured delta explainable — - * a reader can see why scores diverged, not just that they did. - * - * Methodology note: deltas attribute to the whole Jumbo system armed - * against a parity-matched baseline agent under framework-as-developer - * orchestration with agent-driven lifecycle (the framework hands the - * agent an active goal-id and lifecycle protocol; the agent itself runs - * every jumbo lifecycle call). Headline lift is not a memory-delivery - * number in isolation. - */ -export interface SessionContextAudit { - readonly sessionNumber: number; - readonly scenarioPrompt?: string; - readonly effectivePrompt?: string; - readonly deliveredContext?: string; -} - -export interface DimensionEvidence { - readonly dimension: string; - readonly jumboScore: number; - readonly baselineScore: number; - readonly delta: number; - readonly jumboDetails?: string; - readonly baselineDetails?: string; -} - -export interface AuditTrail { - readonly harness: string; - readonly jumboContext: readonly SessionContextAudit[]; - readonly baselineContext: readonly SessionContextAudit[]; - readonly jumboFinalSnapshot?: WorkspaceSnapshot; - readonly baselineFinalSnapshot?: WorkspaceSnapshot; - readonly scoringEvidence: readonly DimensionEvidence[]; -} - -/** - * Complete structured report. - */ -export interface TamperedComparisonSummary { - readonly comparisonId: string; - readonly harness: string; - readonly tamperLog: readonly TamperEvent[]; -} - -export interface FullReport { - readonly scenarioId: string; - readonly harnesses: readonly string[]; - readonly divergenceCurve: readonly DivergencePoint[]; - readonly liftResults: readonly LiftResult[]; - readonly divergenceOnsets: readonly DivergenceOnset[]; - readonly disruptionImpacts: readonly DisruptionImpact[]; - readonly memoryCaptureEvidence: readonly MemoryCaptureEvidence[]; - readonly protocolAdherenceEvidence: readonly ProtocolAdherenceEvidence[]; - readonly harnessAggregation: readonly HarnessLiftSummary[]; - readonly auditTrails: readonly AuditTrail[]; - readonly tamperedComparisons: readonly TamperedComparisonSummary[]; - readonly generatedAt: string; -} - -/** - * Computes the session-by-session divergence curve from timelines. - * Returns per-session, per-dimension deltas showing how Jumbo and baseline diverge. - */ -export function computeDivergenceCurve(comparison: ComparisonResult): DivergencePoint[] { - const points: DivergencePoint[] = []; - - if (!comparison.jumboTimeline || !comparison.baselineTimeline) return points; - - for (let i = 0; i < comparison.jumboTimeline.length; i++) { - const jSession = comparison.jumboTimeline[i]; - const bSession = comparison.baselineTimeline[i]; - if (!jSession || !bSession) continue; - - for (const jScore of jSession.scores) { - const bScore = bSession.scores.find((s) => s.dimension === jScore.dimension); - if (!bScore) continue; - - points.push({ - sessionNumber: jSession.sessionNumber, - dimension: jScore.dimension, - jumboScore: jScore.score, - baselineScore: bScore.score, - delta: Math.round((jScore.score - bScore.score) * 1000) / 1000, - }); - } - } - - return points; -} - -/** - * Computes per-dimension lift percentages from aggregate scores. - */ -export function computeLiftPercentages(comparison: ComparisonResult): LiftResult[] { - return comparison.jumboScores.map((js, i) => { - const bs = comparison.baselineScores[i]; - const absoluteLift = Math.round((js.score - bs.score) * 1000) / 1000; - const percentageLift = bs.score !== 0 - ? Math.round(((js.score - bs.score) / bs.score) * 10000) / 100 - : null; - - return { - dimension: js.dimension, - jumboScore: js.score, - baselineScore: bs.score, - absoluteLift, - percentageLift, - }; - }); -} - -/** - * Detects the session number where divergence becomes significant. - * A dimension diverges when the delta exceeds the threshold for the first time. - */ -export function detectDivergenceOnset( - divergenceCurve: readonly DivergencePoint[], - threshold: number = 0.1, -): DivergenceOnset[] { - const dimensions = [...new Set(divergenceCurve.map((p) => p.dimension))]; - - return dimensions.map((dim) => { - const points = divergenceCurve - .filter((p) => p.dimension === dim) - .sort((a, b) => a.sessionNumber - b.sessionNumber); - - const onsetPoint = points.find((p) => Math.abs(p.delta) >= threshold); - - return { - dimension: dim, - onsetSession: onsetPoint ? onsetPoint.sessionNumber : null, - threshold, - deltaAtOnset: onsetPoint ? onsetPoint.delta : null, - }; - }); -} - -/** - * Analyzes the impact of disruptions on Jumbo vs baseline deltas. - * For each disruption, computes the delta change before and after injection. - */ -export function analyzeDisruptionImpact( - divergenceCurve: readonly DivergencePoint[], - disruptions: readonly Disruption[], -): DisruptionImpact[] { - const impacts: DisruptionImpact[] = []; - - const dimensions = [...new Set(divergenceCurve.map((p) => p.dimension))]; - - for (const disruption of disruptions) { - for (const dim of dimensions) { - const points = divergenceCurve - .filter((p) => p.dimension === dim) - .sort((a, b) => a.sessionNumber - b.sessionNumber); - - const beforePoint = points.find((p) => p.sessionNumber === disruption.sessionNumber - 1); - const afterPoint = points.find((p) => p.sessionNumber === disruption.sessionNumber) - ?? points.find((p) => p.sessionNumber === disruption.sessionNumber + 1); - - const deltaBeforeDisruption = beforePoint?.delta ?? 0; - const deltaAfterDisruption = afterPoint?.delta ?? 0; - - impacts.push({ - disruption, - dimension: dim, - deltaBeforeDisruption, - deltaAfterDisruption, - impactMagnitude: Math.round( - Math.abs(deltaAfterDisruption - deltaBeforeDisruption) * 1000, - ) / 1000, - }); - } - } - - // Sort by impact magnitude descending - return impacts.sort((a, b) => b.impactMagnitude - a.impactMagnitude); -} - -/** - * Aggregates lift across multiple harnesses for the same scenario. - */ -export function aggregateHarnessLifts( - comparisons: readonly ComparisonResult[], -): HarnessLiftSummary[] { - return comparisons.map((comp) => { - const lifts = computeLiftPercentages(comp); - const avgLift = lifts.length > 0 - ? Math.round( - (lifts.reduce((sum, l) => sum + l.absoluteLift, 0) / lifts.length) * 1000, - ) / 1000 - : 0; - - return { - harness: comp.harness, - avgLift, - dimensionLifts: lifts, - }; - }); -} - -export function extractMemoryCaptureEvidence( - comparisons: readonly ComparisonResult[], -): MemoryCaptureEvidence[] { - return comparisons.flatMap((comparison) => { - const jumboScore = comparison.jumboScores.find((score) => score.dimension === 'jumbo-memory-capture'); - if (!jumboScore) return []; - const baselineScore = comparison.baselineScores.find((score) => score.dimension === 'jumbo-memory-capture'); - return [{ - harness: comparison.harness, - jumboScore: jumboScore.score, - jumboDetails: jumboScore.details, - baselineDetails: baselineScore?.details, - }]; - }); -} - -export function extractProtocolAdherenceEvidence( - comparisons: readonly ComparisonResult[], -): ProtocolAdherenceEvidence[] { - return comparisons.flatMap((comparison) => { - const jumboScore = comparison.jumboScores.find((score) => score.dimension === 'protocol-adherence'); - if (!jumboScore) return []; - const baselineScore = comparison.baselineScores.find((score) => score.dimension === 'protocol-adherence'); - const perSession = (comparison.jumboTimeline ?? []).map((ps) => { - const protocolScore = ps.scores.find((s) => s.dimension === 'protocol-adherence'); - return { - sessionNumber: ps.sessionNumber, - score: protocolScore?.score ?? 0, - details: protocolScore?.details, - }; - }); - return [{ - harness: comparison.harness, - jumboScore: jumboScore.score, - jumboDetails: jumboScore.details, - baselineDetails: baselineScore?.details, - perSession, - }]; - }); -} - -function summarizeRecordContext(record: SessionRecord): SessionContextAudit { - return { - sessionNumber: record.sessionNumber, - scenarioPrompt: record.scenarioPrompt, - effectivePrompt: record.effectivePrompt, - deliveredContext: record.deliveredContext, - }; -} - -function lastRecord(records: readonly SessionRecord[]): SessionRecord | undefined { - if (records.length === 0) return undefined; - return records.reduce((latest, r) => (r.sessionNumber > latest.sessionNumber ? r : latest)); -} - -export function buildAuditTrail(comparison: ComparisonResult): AuditTrail { - const jumboRecords = comparison.jumboResult.sessionRecords; - const baselineRecords = comparison.baselineResult.sessionRecords; - - const scoringEvidence: DimensionEvidence[] = comparison.jumboScores.map((js, i) => { - const bs = comparison.baselineScores[i]; - return { - dimension: js.dimension, - jumboScore: js.score, - baselineScore: bs?.score ?? 0, - delta: Math.round((js.score - (bs?.score ?? 0)) * 1000) / 1000, - jumboDetails: js.details, - baselineDetails: bs?.details, - }; - }); - - return { - harness: comparison.harness, - jumboContext: jumboRecords.map(summarizeRecordContext), - baselineContext: baselineRecords.map(summarizeRecordContext), - jumboFinalSnapshot: lastRecord(jumboRecords)?.workspaceSnapshot, - baselineFinalSnapshot: lastRecord(baselineRecords)?.workspaceSnapshot, - scoringEvidence, - }; -} - -/** - * Generates a complete structured report from one or more ComparisonResults. - * Pure function — no I/O. - */ -export function generateFullReport( - comparisons: readonly ComparisonResult[], - disruptions: readonly Disruption[] = [], - tamperedExcluded: readonly ComparisonResult[] = [], -): FullReport { - const primary = comparisons[0]; - const divergenceCurves = comparisons.flatMap(computeDivergenceCurve); - const liftResults = primary ? computeLiftPercentages(primary) : []; - const divergenceOnsets = detectDivergenceOnset(divergenceCurves); - const disruptionImpacts = analyzeDisruptionImpact(divergenceCurves, disruptions); - const memoryCaptureEvidence = extractMemoryCaptureEvidence(comparisons); - const protocolAdherenceEvidence = extractProtocolAdherenceEvidence(comparisons); - const harnessAggregation = aggregateHarnessLifts(comparisons); - const auditTrails = comparisons.map(buildAuditTrail); - - return { - scenarioId: primary?.scenarioId ?? '', - harnesses: comparisons.map((c) => c.harness), - divergenceCurve: divergenceCurves, - liftResults, - divergenceOnsets, - disruptionImpacts, - memoryCaptureEvidence, - protocolAdherenceEvidence, - harnessAggregation, - auditTrails, - tamperedComparisons: tamperedExcluded.map((c) => ({ - comparisonId: c.id, - harness: c.harness, - tamperLog: c.tamperLog, - })), - generatedAt: new Date().toISOString(), - }; -} - -/** - * Formats the full report as polished terminal output. - */ -export function formatFullReport(report: FullReport): string { - const lines: string[] = []; - const divider = '═'.repeat(72); - const thinDivider = '─'.repeat(72); - - lines.push(divider); - lines.push(` JUMBO EVALUATION REPORT`); - lines.push(` Scenario: ${report.scenarioId}`); - lines.push(` Harnesses: ${report.harnesses.join(', ')}`); - lines.push(` Treatment: whole Jumbo system vs Jumbo-unreachable baseline`); - lines.push(` Methodology: framework-as-developer orchestration, agent-driven lifecycle`); - lines.push(` Lift attribution: init + pre-seeded memory + lifecycle protocol + agent capture + audit`); - lines.push(divider); - lines.push(''); - - // Lift percentages - lines.push(' LIFT BY DIMENSION'); - lines.push(thinDivider); - lines.push( - padRight(' Dimension', 24) + - padRight('Jumbo', 10) + - padRight('Base', 10) + - padRight('Lift', 10) + - 'Lift %', - ); - lines.push(thinDivider); - - for (const lift of report.liftResults) { - const pctStr = lift.percentageLift !== null ? `${lift.percentageLift > 0 ? '+' : ''}${lift.percentageLift.toFixed(1)}%` : 'n/a'; - const sign = lift.absoluteLift > 0 ? '+' : ''; - lines.push( - padRight(` ${lift.dimension}`, 24) + - padRight(lift.jumboScore.toFixed(2), 10) + - padRight(lift.baselineScore.toFixed(2), 10) + - padRight(`${sign}${lift.absoluteLift.toFixed(3)}`, 10) + - pctStr, - ); - } - lines.push(''); - - // Divergence onset - if (report.divergenceOnsets.length > 0) { - lines.push(' DIVERGENCE ONSET (session where amnesia impact exceeds threshold)'); - lines.push(thinDivider); - - for (const onset of report.divergenceOnsets) { - if (onset.onsetSession !== null) { - const sign = onset.deltaAtOnset! > 0 ? '+' : ''; - lines.push(` ${padRight(onset.dimension, 22)} session ${onset.onsetSession} (delta: ${sign}${onset.deltaAtOnset!.toFixed(3)}, threshold: ${onset.threshold})`); - } else { - lines.push(` ${padRight(onset.dimension, 22)} no significant divergence detected`); - } - } - lines.push(''); - } - - // Divergence curve - if (report.divergenceCurve.length > 0) { - lines.push(' DIVERGENCE CURVE (session-by-session delta)'); - lines.push(thinDivider); - - const dimensions = [...new Set(report.divergenceCurve.map((p) => p.dimension))]; - - for (const dim of dimensions) { - const points = report.divergenceCurve - .filter((p) => p.dimension === dim) - .sort((a, b) => a.sessionNumber - b.sessionNumber); - - const bars = points.map((p) => { - const sign = p.delta > 0 ? '+' : ''; - return `S${p.sessionNumber}:${sign}${p.delta.toFixed(2)}`; - }).join(' '); - - lines.push(` ${padRight(dim, 22)} ${bars}`); - } - lines.push(''); - } - - if (report.memoryCaptureEvidence.length > 0) { - lines.push(' JUMBO MEMORY CAPTURE EVIDENCE'); - lines.push(thinDivider); - - for (const evidence of report.memoryCaptureEvidence) { - lines.push(` ${padRight(evidence.harness, 20)} score: ${evidence.jumboScore.toFixed(2)}`); - if (evidence.jumboDetails) { - lines.push(` Jumbo: ${evidence.jumboDetails}`); - } - if (evidence.baselineDetails) { - lines.push(` Baseline: ${evidence.baselineDetails}`); - } - } - lines.push(''); - } - - // Protocol adherence — first-class signal for lifecycle non-adherence. - if (report.protocolAdherenceEvidence.length > 0) { - lines.push(' PROTOCOL ADHERENCE EVIDENCE (per-step lifecycle execution)'); - lines.push(thinDivider); - - for (const evidence of report.protocolAdherenceEvidence) { - lines.push(` ${padRight(evidence.harness, 20)} score: ${evidence.jumboScore.toFixed(2)}`); - if (evidence.jumboDetails) { - lines.push(` Jumbo: ${evidence.jumboDetails}`); - } - if (evidence.baselineDetails) { - lines.push(` Baseline: ${evidence.baselineDetails}`); - } - for (const session of evidence.perSession) { - const sessionLine = ` S${session.sessionNumber}: ${session.score.toFixed(2)}` + - (session.details ? ` (${session.details})` : ''); - lines.push(sessionLine); - } - } - lines.push(''); - } - - // Disruption impact - if (report.disruptionImpacts.length > 0) { - lines.push(' DISRUPTION IMPACT (ranked by magnitude)'); - lines.push(thinDivider); - - const top = report.disruptionImpacts.slice(0, 10); - for (const impact of top) { - lines.push( - ` [${impact.disruption.type}] S${impact.disruption.sessionNumber} ` + - `${padRight(impact.dimension, 20)} ` + - `before:${impact.deltaBeforeDisruption >= 0 ? '+' : ''}${impact.deltaBeforeDisruption.toFixed(3)} ` + - `after:${impact.deltaAfterDisruption >= 0 ? '+' : ''}${impact.deltaAfterDisruption.toFixed(3)} ` + - `magnitude:${impact.impactMagnitude.toFixed(3)}`, - ); - } - lines.push(''); - } - - // Harness aggregation - if (report.harnessAggregation.length > 1) { - lines.push(' CROSS-HARNESS AGGREGATION'); - lines.push(thinDivider); - - const sorted = [...report.harnessAggregation].sort((a, b) => b.avgLift - a.avgLift); - for (const h of sorted) { - const sign = h.avgLift > 0 ? '+' : ''; - lines.push(` ${padRight(h.harness, 20)} avg lift: ${sign}${h.avgLift.toFixed(3)}`); - } - lines.push(''); - } - - // Audit trail — explains why a measured delta occurred - if (report.auditTrails.length > 0) { - lines.push(' AUDIT TRAIL (effective context, final snapshot, scoring evidence)'); - lines.push(thinDivider); - - for (const trail of report.auditTrails) { - lines.push(` ${trail.harness}`); - - const lastJumbo = trail.jumboContext.at(-1); - const lastBaseline = trail.baselineContext.at(-1); - if (lastJumbo) { - const delivered = compactAuditText(lastJumbo.deliveredContext, 80) ?? 'none'; - lines.push(` Jumbo final session ${lastJumbo.sessionNumber} delivered context: ${delivered}`); - } - if (lastBaseline) { - const effective = compactAuditText(lastBaseline.effectivePrompt, 80) ?? 'n/a'; - lines.push(` Baseline final session ${lastBaseline.sessionNumber} effective prompt: ${effective}`); - } - - const jumboFiles = trail.jumboFinalSnapshot?.files.map((f) => f.path) ?? []; - const baselineFiles = trail.baselineFinalSnapshot?.files.map((f) => f.path) ?? []; - lines.push(` Jumbo final snapshot files: ${jumboFiles.length > 0 ? jumboFiles.join(', ') : '(none)'}`); - lines.push(` Baseline final snapshot files: ${baselineFiles.length > 0 ? baselineFiles.join(', ') : '(none)'}`); - - for (const evidence of trail.scoringEvidence) { - const sign = evidence.delta > 0 ? '+' : ''; - lines.push( - ` ${padRight(evidence.dimension, 22)} jumbo=${evidence.jumboScore.toFixed(2)} ` + - `baseline=${evidence.baselineScore.toFixed(2)} delta=${sign}${evidence.delta.toFixed(3)}`, - ); - if (evidence.jumboDetails) lines.push(` jumbo: ${evidence.jumboDetails}`); - if (evidence.baselineDetails) lines.push(` baseline: ${evidence.baselineDetails}`); - } - } - lines.push(''); - } - - lines.push(divider); - - return lines.join('\n'); -} - -function compactAuditText(value: string | undefined, maxLength: number): string | undefined { - if (value === undefined) return undefined; - const compacted = value.replace(/\s+/g, ' ').trim(); - if (compacted.length === 0) return undefined; - if (compacted.length <= maxLength) return compacted; - return `${compacted.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...`; -} - -function padRight(str: string, width: number): string { - return str.length >= width ? str : str + ' '.repeat(width - str.length); -} diff --git a/evals/src/run-session.ts b/evals/src/run-session.ts deleted file mode 100644 index c280f913..00000000 --- a/evals/src/run-session.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import type { TestScenario, SessionRecord } from './domain/types.js'; -import { createSessionRecord } from './domain/types.js'; -import type { ResultStore } from './storage/result-store.js'; -import type { LocalExecutor } from './infrastructure/local-executor.js'; -import type { ExecResult } from './infrastructure/container-manager.js'; -import type { HarnessAdapter } from './harness/harness-adapter.js'; - -export class HarnessExecutionError extends Error { - constructor( - message: string, - readonly result: ExecResult, - readonly sessionRecord: SessionRecord, - ) { - super(message); - this.name = 'HarnessExecutionError'; - } -} - -/** - * Runs a single session of a test scenario in a local working directory. - * This is the minimal end-to-end path: one scenario, one session, one harness. - * - * Pure orchestration — I/O is delegated to LocalExecutor, HarnessAdapter, - * and ResultStore which are injected as parameters. - */ -export async function runSession(params: { - scenario: TestScenario; - sessionNumber: number; - variant?: 'jumbo' | 'baseline'; - prompt?: string; - scenarioPrompt?: string; - deliveredContext?: string; - workDir: string; - executor: LocalExecutor; - adapter: HarnessAdapter; - store: ResultStore; - env?: Record<string, string | undefined>; -}): Promise<SessionRecord> { - const { - scenario, - sessionNumber, - variant, - workDir, - executor, - adapter, - store, - deliveredContext, - } = params; - const scenarioPrompt = params.scenarioPrompt ?? scenario.initialPrompt; - const effectivePrompt = params.prompt ?? scenarioPrompt; - - const startedAt = new Date().toISOString(); - - const command = adapter.buildCommand(); - const execResult = await executor.exec(workDir, command, { stdin: effectivePrompt, env: params.env }); - const parsed = adapter.parseOutput(execResult); - - const completedAt = new Date().toISOString(); - const workspaceSnapshot = await executor.captureWorkspaceSnapshot(workDir); - - const record = createSessionRecord({ - id: randomUUID(), - scenarioId: scenario.id, - sessionNumber, - harness: adapter.name, - variant, - scenarioPrompt, - effectivePrompt, - deliveredContext, - agentOutput: parsed.agentOutput, - filesModified: parsed.filesModified, - transcript: parsed.transcript, - workspaceSnapshot, - inputTokens: parsed.inputTokens, - outputTokens: parsed.outputTokens, - startedAt, - completedAt, - }); - - await store.saveSessionRecord(record); - - if (execResult.exitCode !== 0) { - throw new HarnessExecutionError( - `harness ${adapter.name} session ${sessionNumber} failed with exit code ${execResult.exitCode}`, - execResult, - record, - ); - } - - return record; -} diff --git a/evals/src/scoring/disruption-recovery-scorer.ts b/evals/src/scoring/disruption-recovery-scorer.ts deleted file mode 100644 index 5d861290..00000000 --- a/evals/src/scoring/disruption-recovery-scorer.ts +++ /dev/null @@ -1,150 +0,0 @@ -import type { SessionRecord, DimensionScore } from '../domain/types.js'; -import type { Disruption } from '../domain/types.js'; - -/** - * Builds the search corpus for a session record. - * If a workspace snapshot is available its file contents are the primary evidence, - * catching cases where recovery patterns are actually implemented in code rather - * than merely mentioned in transcripts. - * Falls back to agentOutput + transcript + filesModified when no snapshot exists. - */ -function buildSearchText(record: SessionRecord): string { - if (record.workspaceSnapshot && record.workspaceSnapshot.files.length > 0) { - return [ - ...record.workspaceSnapshot.files.map((f) => f.content), - ...record.workspaceSnapshot.files.map((f) => f.path), - ].join('\n').toLowerCase(); - } - - return [ - record.agentOutput, - record.transcript, - ...record.filesModified, - ].join('\n').toLowerCase(); -} - -/** - * Pure deterministic scorer: checks whether disruption corrections - * persist in sessions after the disruption was injected. - * - * For each disruption, checks its recoveryPatterns in all sessions - * AFTER the injection session. A correction that persists means the - * agent retained the correction. A correction that disappears after - * a session boundary signals amnesia. - * - * When a workspace snapshot is available, post-disruption workspace - * file contents are the primary evidence; transcript is the fallback. - * - * Score = fraction of (disruption, post-session) pairs where recovery - * patterns are found. - */ -export function scoreDisruptionRecovery( - sessionRecords: readonly SessionRecord[], - disruptions: readonly Disruption[], -): DimensionScore { - if (disruptions.length === 0) { - return { - dimension: 'disruption-recovery', - score: 1, - maxScore: 1, - details: 'No disruptions defined — trivially satisfied', - }; - } - - const sorted = [...sessionRecords].sort((a, b) => a.sessionNumber - b.sessionNumber); - let totalChecks = 0; - let recoveredChecks = 0; - const failures: string[] = []; - - for (const disruption of disruptions) { - if (disruption.recoveryPatterns.length === 0) continue; - - const postSessions = sorted.filter((r) => r.sessionNumber > disruption.sessionNumber); - if (postSessions.length === 0) continue; - - for (const session of postSessions) { - const searchText = buildSearchText(session); - - for (const pattern of disruption.recoveryPatterns) { - totalChecks++; - if (searchText.includes(pattern.toLowerCase())) { - recoveredChecks++; - } else { - failures.push(`session ${session.sessionNumber}: lost "${pattern}" from ${disruption.type} @ session ${disruption.sessionNumber}`); - } - } - } - } - - if (totalChecks === 0) { - return { - dimension: 'disruption-recovery', - score: 1, - maxScore: 1, - details: 'No recovery checks applicable (no post-disruption sessions or no recovery patterns)', - }; - } - - const score = recoveredChecks / totalChecks; - - const details = [ - `${recoveredChecks}/${totalChecks} recovery checks passed`, - failures.length > 0 ? `failures: ${failures.slice(0, 5).join('; ')}${failures.length > 5 ? ` (+${failures.length - 5} more)` : ''}` : null, - ].filter(Boolean).join('; '); - - return { - dimension: 'disruption-recovery', - score: Math.round(score * 100) / 100, - maxScore: 1, - details, - }; -} - -/** - * Per-session disruption recovery timeline. - * Returns one score per session showing recovery status at that point. - */ -export function scoreDisruptionRecoveryTimeline( - sessionRecords: readonly SessionRecord[], - disruptions: readonly Disruption[], -): DimensionScore[] { - if (disruptions.length === 0) return []; - - const sorted = [...sessionRecords].sort((a, b) => a.sessionNumber - b.sessionNumber); - - return sorted.map((session) => { - const activeDisruptions = disruptions.filter((d) => d.sessionNumber < session.sessionNumber); - - if (activeDisruptions.length === 0) { - return { - dimension: 'disruption-recovery', - score: 1, - maxScore: 1, - details: `session ${session.sessionNumber}: no active disruptions yet`, - }; - } - - const searchText = buildSearchText(session); - - let totalPatterns = 0; - let found = 0; - - for (const d of activeDisruptions) { - for (const pattern of d.recoveryPatterns) { - totalPatterns++; - if (searchText.includes(pattern.toLowerCase())) { - found++; - } - } - } - - const score = totalPatterns > 0 ? found / totalPatterns : 1; - - return { - dimension: 'disruption-recovery', - score: Math.round(score * 100) / 100, - maxScore: 1, - details: `session ${session.sessionNumber}: ${found}/${totalPatterns} recovery patterns present`, - }; - }); -} diff --git a/evals/src/scoring/file-accuracy-scorer.ts b/evals/src/scoring/file-accuracy-scorer.ts deleted file mode 100644 index 46f136a7..00000000 --- a/evals/src/scoring/file-accuracy-scorer.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { SessionRecord, DimensionScore } from '../domain/types.js'; - -/** - * Returns the set of modified files for a session record. - * Uses harness-reported filesModified as the primary source. - * Falls back to workspace snapshot file paths when the harness - * did not report any modified files (e.g. adapter metadata missing). - */ -function getModifiedFiles(record: SessionRecord): readonly string[] { - if (record.filesModified.length > 0) { - return record.filesModified; - } - if (record.workspaceSnapshot && record.workspaceSnapshot.files.length > 0) { - return record.workspaceSnapshot.files.map((f) => f.path); - } - return []; -} - -/** - * Whether every expected file was produced (recall === 1) across the given - * sessions — one half of the token-efficiency output-equivalence gate - * (GOAL.md "Comparing token usage fairly"). Vacuously true when no files are - * expected. Independent of precision: extra files do not fail this check. - */ -export function producedAllExpectedFiles( - sessionRecords: readonly SessionRecord[], - expectedFiles: readonly string[], -): boolean { - if (expectedFiles.length === 0) return true; - const allModified = new Set(sessionRecords.flatMap((r) => getModifiedFiles(r))); - return expectedFiles.every((expected) => allModified.has(expected)); -} - -/** - * Pure deterministic scorer: compares files modified in a session - * against the expected file list from the scenario. - * - * Score = F1(precision, recall) over expected vs. actual modified files. - * Penalizes both missed files (low recall) and unexpected files (low precision). - * - * When a session's harness metadata is missing (filesModified empty), actual - * workspace file paths from the snapshot are used as the fallback. - */ -export function scoreFileAccuracy( - sessionRecords: readonly SessionRecord[], - expectedFiles: readonly string[], -): DimensionScore { - if (expectedFiles.length === 0) { - return { - dimension: 'file-accuracy', - score: 1, - maxScore: 1, - details: 'No expected files defined — trivially satisfied', - }; - } - - const allModified = new Set( - sessionRecords.flatMap((r) => getModifiedFiles(r)), - ); - - const expectedSet = new Set(expectedFiles); - - let hits = 0; - const missed: string[] = []; - for (const expected of expectedSet) { - if (allModified.has(expected)) { - hits++; - } else { - missed.push(expected); - } - } - - const unexpected: string[] = []; - for (const modified of allModified) { - if (!expectedSet.has(modified)) { - unexpected.push(modified); - } - } - - const precision = allModified.size > 0 ? hits / allModified.size : 0; - const recall = hits / expectedSet.size; - const f1 = precision + recall > 0 - ? (2 * precision * recall) / (precision + recall) - : 0; - - const details = [ - `${hits}/${expectedSet.size} expected files modified`, - missed.length > 0 ? `missed: ${missed.join(', ')}` : null, - unexpected.length > 0 ? `unexpected: ${unexpected.join(', ')}` : null, - `precision=${precision.toFixed(2)} recall=${recall.toFixed(2)} f1=${f1.toFixed(2)}`, - ].filter(Boolean).join('; '); - - return { - dimension: 'file-accuracy', - score: Math.round(f1 * 100) / 100, - maxScore: 1, - details, - }; -} diff --git a/evals/src/scoring/index.ts b/evals/src/scoring/index.ts deleted file mode 100644 index 3f7e9bd2..00000000 --- a/evals/src/scoring/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -export { scoreFileAccuracy, producedAllExpectedFiles } from './file-accuracy-scorer.js'; -export { scoreKnowledgeRetention, scoreKnowledgeRetentionTimeline } from './knowledge-retention-scorer.js'; -export { scoreStructuralAssertions, scoreStructuralAssertionsTimeline } from './structural-assertion-scorer.js'; -export { scoreDisruptionRecovery, scoreDisruptionRecoveryTimeline } from './disruption-recovery-scorer.js'; -export { scoreJumboMemoryCapture, scoreJumboMemoryCaptureTimeline, baselineJumboMemoryCaptureScore } from './jumbo-memory-capture-scorer.js'; -export { scoreJumboEventCapture, baselineJumboEventCaptureScore, scoreJumboEventCaptureTimeline, resolveAddedEventTypeForKind, addedEventTypeByKind } from './jumbo-event-capture-scorer.js'; -export { scoreProtocolAdherence, scoreProtocolAdherenceTimeline, baselineProtocolAdherenceScore, adherenceForSession, PROTOCOL_STEPS } from './protocol-adherence-scorer.js'; -export type { ProtocolStep, ProtocolStepResult, SessionAdherence } from './protocol-adherence-scorer.js'; -export { scoreTokenEfficiency, compareTokenEfficiency, tokenUsageTimeline } from './token-efficiency-scorer.js'; -export { scoreWithJudge, scoreAllJudgeDimensions, validateJudgeConfig, parseJudgeResponse } from './llm-judge-scorer.js'; -export type { JudgeConfig, JudgeFn, JudgeResponse, JudgeQuestionScore } from './llm-judge-scorer.js'; -export { ALL_RUBRICS, CONSISTENCY_RUBRIC, ERROR_CORRECTION_RUBRIC, ARCHITECTURAL_QUALITY_RUBRIC, buildJudgePrompt } from './rubrics.js'; -export type { Rubric, RubricQuestion, ScalePoint } from './rubrics.js'; - -import type { SessionRecord } from '../domain/types.js'; - -/** - * Filters out tampered SessionRecords for aggregate scoring. - * Aggregate dimension scores must remain attributable to Jumbo-delivered context, - * not operator-injected tamper events. Per-record audit trails retain the full log. - */ -export function nonTamperedSessions( - records: readonly SessionRecord[], -): readonly SessionRecord[] { - return records.filter((r) => !r.tampered); -} diff --git a/evals/src/scoring/jumbo-event-capture-scorer.ts b/evals/src/scoring/jumbo-event-capture-scorer.ts deleted file mode 100644 index 0decc409..00000000 --- a/evals/src/scoring/jumbo-event-capture-scorer.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { - DimensionScore, - ExpectedJumboMemoryCapture, - JumboMemoryKind, - SessionRecord, -} from '../domain/types.js'; - -/** - * Secondary, non-CLI-mediated confirmation that the Jumbo arm registered the - * expected entities (GOAL.md "Workspace evidence: capturing .jumbo/"). Where - * the primary jumbo-memory-capture scorer reads the CLI-derived memory - * snapshot, this scorer reads only the captured `.jumbo/events/` summary on the - * workspace snapshot — the event log is Jumbo's own ground truth, independent - * of any CLI command output. - */ -const DIMENSION = 'jumbo-event-capture'; - -/** - * Explicit, exhaustive registry from a memory kind to the Jumbo domain event - * its registration emits. `satisfies Record<JumboMemoryKind, string>` makes the - * compiler reject a new JumboMemoryKind until it is mapped here, and pins the - * exact event-type spelling rather than deriving it from a naming convention. - * - * These literals mirror Jumbo's domain event-type constants - * (DecisionEventType.ADDED, ComponentEventType.ADDED, …). evals is a separate - * package and cannot import those constants at compile time, so the - * jumbo-contract-smoke validates this map against the real CLI at CI time — if - * Jumbo renames an event type, the smoke fails loudly instead of the scorer - * silently under-counting. - */ -export const addedEventTypeByKind = { - decision: 'DecisionAddedEvent', - guideline: 'GuidelineAddedEvent', - invariant: 'InvariantAddedEvent', - component: 'ComponentAddedEvent', - relation: 'RelationAddedEvent', - dependency: 'DependencyAddedEvent', -} as const satisfies Record<JumboMemoryKind, string>; - -/** - * Resolves the *Added*-event type for a memory kind, e.g. 'decision' -> - * 'DecisionAddedEvent'. This scorer confirms initial registration (capture), - * so it maps only to the Added event — not Updated/Removed/Reversed/etc. - */ -export function resolveAddedEventTypeForKind(kind: JumboMemoryKind): string { - return addedEventTypeByKind[kind]; -} - -/** The event-type counts from the latest session that carries an event-log summary. */ -function latestEventCounts(records: readonly SessionRecord[]): Readonly<Record<string, number>> | undefined { - let latest: SessionRecord | undefined; - for (const record of records) { - if (!record.workspaceSnapshot?.jumboEvents) continue; - if (!latest || record.sessionNumber > latest.sessionNumber) latest = record; - } - return latest?.workspaceSnapshot?.jumboEvents?.countsByType; -} - -/** - * Fraction of the distinct expected entity kinds whose corresponding Jumbo - * event type appears (count >= 1) in the latest session's event-log summary. - * Trivially satisfied when no captures are expected. - */ -export function scoreJumboEventCapture( - records: readonly SessionRecord[], - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore { - if (expectedCaptures.length === 0) { - return { - dimension: DIMENSION, - score: 1, - maxScore: 1, - details: 'No expected Jumbo captures; trivially satisfied.', - }; - } - - const expectedKinds = [...new Set(expectedCaptures.map((c) => c.kind))]; - const counts = latestEventCounts(records) ?? {}; - - const present: string[] = []; - const missing: string[] = []; - for (const kind of expectedKinds) { - const eventType = resolveAddedEventTypeForKind(kind); - if ((counts[eventType] ?? 0) >= 1) present.push(eventType); - else missing.push(eventType); - } - - const score = present.length / expectedKinds.length; - return { - dimension: DIMENSION, - score: Math.round(score * 100) / 100, - maxScore: 1, - details: [ - `${present.length}/${expectedKinds.length} expected entity kinds evidenced in .jumbo/events`, - missing.length > 0 ? `missing: ${missing.join(', ')}` : 'missing: none', - ].join('; '), - }; -} - -/** - * Baseline arm has no `.jumbo/events/` log — there is nothing to confirm. - * Reported as a trivial zero (maxScore 0), mirroring baselineJumboMemoryCaptureScore. - */ -export function baselineJumboEventCaptureScore( - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore { - return { - dimension: DIMENSION, - score: 0, - maxScore: 0, - details: expectedCaptures.length > 0 - ? 'Not applicable: baseline runs have no .jumbo/events log.' - : 'Not applicable: no expected Jumbo captures.', - }; -} - -/** Per-session trajectory: the event-capture score cumulative to each session. */ -export function scoreJumboEventCaptureTimeline( - records: readonly SessionRecord[], - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore[] { - return records.map((record) => - scoreJumboEventCapture( - records.filter((candidate) => candidate.sessionNumber <= record.sessionNumber), - expectedCaptures.filter((capture) => !capture.sessionNumber || capture.sessionNumber <= record.sessionNumber), - ), - ); -} diff --git a/evals/src/scoring/jumbo-memory-capture-scorer.ts b/evals/src/scoring/jumbo-memory-capture-scorer.ts deleted file mode 100644 index 5b46cc23..00000000 --- a/evals/src/scoring/jumbo-memory-capture-scorer.ts +++ /dev/null @@ -1,181 +0,0 @@ -import type { - DimensionScore, - ExpectedJumboMemoryCapture, - JumboMemoryEntity, - JumboMemorySnapshot, - SessionRecord, -} from '../domain/types.js'; - -const DIMENSION = 'jumbo-memory-capture'; - -export interface JumboMemoryCaptureEvidence { - readonly expected: ExpectedJumboMemoryCapture; - readonly matched: boolean; - readonly observed?: JumboMemoryEntity; - readonly observedSessionNumber?: number; -} - -interface NewEntity { - readonly entity: JumboMemoryEntity; - readonly sessionNumber: number; -} - -function normalize(text: string): string { - return text.toLowerCase().replace(/\s+/g, ' ').trim(); -} - -function entityKey(entity: JumboMemoryEntity): string { - return entity.id - ? `${entity.kind}:id:${entity.id}` - : `${entity.kind}:${normalize(entity.text)}`; -} - -function snapshotKeys(snapshot: JumboMemorySnapshot | undefined): Set<string> { - const keys = new Set<string>(); - if (!snapshot) return keys; - for (const entity of snapshot.entities) { - keys.add(entityKey(entity)); - } - return keys; -} - -/** - * Computes the entities the agent registered during a single session window: - * the diff between the pre-session and post-session Jumbo memory snapshots - * taken inside the run. Entities present in `jumboMemorySnapshot` but absent - * from `jumboMemorySnapshotBefore` are credited to that session. - * - * The pre-snapshot may be undefined for legacy records — in that case the - * full post-snapshot is treated as new. New records always carry both. - */ -function newEntitiesForRecord(record: SessionRecord): JumboMemoryEntity[] { - const post = record.jumboMemorySnapshot; - if (!post) return []; - const beforeKeys = snapshotKeys(record.jumboMemorySnapshotBefore); - const seen = new Set<string>(); - const result: JumboMemoryEntity[] = []; - for (const entity of post.entities) { - const key = entityKey(entity); - if (beforeKeys.has(key)) continue; - if (seen.has(key)) continue; - seen.add(key); - result.push(entity); - } - return result; -} - -function collectNewEntities(records: readonly SessionRecord[]): NewEntity[] { - const all: NewEntity[] = []; - const seen = new Set<string>(); - for (const record of records) { - for (const entity of newEntitiesForRecord(record)) { - const key = entityKey(entity); - if (seen.has(key)) continue; - seen.add(key); - all.push({ entity, sessionNumber: record.sessionNumber }); - } - } - return all; -} - -function matchesExpectation( - entity: JumboMemoryEntity, - expected: ExpectedJumboMemoryCapture, -): boolean { - return entity.kind === expected.kind && normalize(entity.text).includes(normalize(expected.match)); -} - -/** - * Counts entities the agent actually registered during the session windows - * — measured as the diff between pre and post snapshots inside the run, not - * as harness-side mirroring. Recall measures expected captures that appear - * in the diff. Precision measures observed new entities that correspond to - * expectations among the relevant kinds. - */ -export function scoreJumboMemoryCapture( - records: readonly SessionRecord[], - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore { - if (expectedCaptures.length === 0) { - return { - dimension: DIMENSION, - score: 1, - maxScore: 1, - details: 'No expected Jumbo memory captures; trivially satisfied.', - }; - } - - const newEntities = collectNewEntities(records); - const relevantNewEntities = newEntities.filter((ne) => - expectedCaptures.some((expected) => expected.kind === ne.entity.kind), - ); - - const matchedEntityKeys = new Set<string>(); - const evidence: JumboMemoryCaptureEvidence[] = expectedCaptures.map((expected) => { - const eligible = expected.sessionNumber !== undefined - ? newEntities.filter((ne) => ne.sessionNumber >= expected.sessionNumber!) - : newEntities; - const observed = eligible.find((ne) => matchesExpectation(ne.entity, expected)); - if (observed) matchedEntityKeys.add(entityKey(observed.entity)); - return { - expected, - matched: observed !== undefined, - observed: observed?.entity, - observedSessionNumber: observed?.sessionNumber, - }; - }); - - const matched = evidence.filter((item) => item.matched); - const recall = matched.length / expectedCaptures.length; - const precision = relevantNewEntities.length === 0 - ? (matched.length === 0 ? 0 : 1) - : relevantNewEntities.filter((ne) => matchedEntityKeys.has(entityKey(ne.entity))).length / - relevantNewEntities.length; - const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall); - - const missing = evidence - .filter((item) => !item.matched) - .map((item) => `${item.expected.kind}:${item.expected.match}`); - const spurious = relevantNewEntities - .filter((ne) => !matchedEntityKeys.has(entityKey(ne.entity))) - .map((ne) => `${ne.entity.kind}:${ne.entity.text}`); - - return { - dimension: DIMENSION, - score: Math.round(f1 * 100) / 100, - maxScore: 1, - details: [ - `precision=${precision.toFixed(2)}`, - `recall=${recall.toFixed(2)}`, - `matched=${matched.length}/${expectedCaptures.length}`, - `new-entities=${newEntities.length}`, - missing.length > 0 ? `missing: ${missing.join('; ')}` : 'missing: none', - spurious.length > 0 ? `spurious: ${spurious.join('; ')}` : 'spurious: none', - ].join('; '), - }; -} - -export function baselineJumboMemoryCaptureScore( - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore { - return { - dimension: DIMENSION, - score: 0, - maxScore: 0, - details: expectedCaptures.length > 0 - ? 'Not applicable: baseline runs do not use Jumbo project memory.' - : 'Not applicable: no expected Jumbo memory captures.', - }; -} - -export function scoreJumboMemoryCaptureTimeline( - records: readonly SessionRecord[], - expectedCaptures: readonly ExpectedJumboMemoryCapture[], -): DimensionScore[] { - return records.map((record) => - scoreJumboMemoryCapture( - records.filter((candidate) => candidate.sessionNumber <= record.sessionNumber), - expectedCaptures.filter((capture) => !capture.sessionNumber || capture.sessionNumber <= record.sessionNumber), - ), - ); -} diff --git a/evals/src/scoring/knowledge-retention-scorer.ts b/evals/src/scoring/knowledge-retention-scorer.ts deleted file mode 100644 index 6c7147d0..00000000 --- a/evals/src/scoring/knowledge-retention-scorer.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { SessionRecord, DimensionScore } from '../domain/types.js'; - -/** - * Builds the search corpus for a session record. - * If a workspace snapshot is available its file contents are used as the - * primary (and sole) evidence — this catches correct code with terse transcripts - * and rejects keyword-only transcripts that don't reflect real implementation. - * Falls back to agentOutput + transcript + filesModified when no snapshot exists. - */ -function buildSearchText(record: SessionRecord): string { - if (record.workspaceSnapshot && record.workspaceSnapshot.files.length > 0) { - return [ - ...record.workspaceSnapshot.files.map((f) => f.content), - ...record.workspaceSnapshot.files.map((f) => f.path), - ].join('\n').toLowerCase(); - } - - return [ - record.agentOutput, - record.transcript, - ...record.filesModified, - ].join('\n').toLowerCase(); -} - -/** - * Pure deterministic scorer: checks whether patterns established in early - * sessions persist through later sessions. - * - * retentionPatterns are strings (file names, code patterns, identifiers) - * that should appear in the workspace across all sessions. When a workspace - * snapshot is available, file contents are the primary evidence; otherwise - * falls back to transcript and agent output. - * - * Score = fraction of patterns still evident in the latest session. - */ -export function scoreKnowledgeRetention( - sessionRecords: readonly SessionRecord[], - retentionPatterns: readonly string[], -): DimensionScore { - if (retentionPatterns.length === 0 || sessionRecords.length === 0) { - return { - dimension: 'knowledge-retention', - score: 1, - maxScore: 1, - details: retentionPatterns.length === 0 - ? 'No retention patterns defined — trivially satisfied' - : 'No session records to evaluate', - }; - } - - const latestSession = sessionRecords.reduce((latest, r) => - r.sessionNumber > latest.sessionNumber ? r : latest, - ); - - const searchText = buildSearchText(latestSession); - - let retained = 0; - const lost: string[] = []; - - for (const pattern of retentionPatterns) { - if (searchText.includes(pattern.toLowerCase())) { - retained++; - } else { - lost.push(pattern); - } - } - - const score = retained / retentionPatterns.length; - - const details = [ - `${retained}/${retentionPatterns.length} patterns retained in session ${latestSession.sessionNumber}`, - lost.length > 0 ? `lost: ${lost.join(', ')}` : null, - ].filter(Boolean).join('; '); - - return { - dimension: 'knowledge-retention', - score: Math.round(score * 100) / 100, - maxScore: 1, - details, - }; -} - -/** - * Scores knowledge retention per-session to produce a trajectory. - * Returns one DimensionScore per session, showing how retention - * degrades (or holds) over time. - */ -export function scoreKnowledgeRetentionTimeline( - sessionRecords: readonly SessionRecord[], - retentionPatterns: readonly string[], -): DimensionScore[] { - if (retentionPatterns.length === 0) return []; - - const sorted = [...sessionRecords].sort((a, b) => a.sessionNumber - b.sessionNumber); - - return sorted.map((record) => { - const searchText = buildSearchText(record); - - let retained = 0; - for (const pattern of retentionPatterns) { - if (searchText.includes(pattern.toLowerCase())) { - retained++; - } - } - - const score = retained / retentionPatterns.length; - - return { - dimension: 'knowledge-retention', - score: Math.round(score * 100) / 100, - maxScore: 1, - details: `session ${record.sessionNumber}: ${retained}/${retentionPatterns.length} retained`, - }; - }); -} diff --git a/evals/src/scoring/llm-judge-scorer.ts b/evals/src/scoring/llm-judge-scorer.ts deleted file mode 100644 index 84b6e66e..00000000 --- a/evals/src/scoring/llm-judge-scorer.ts +++ /dev/null @@ -1,182 +0,0 @@ -import type { SessionRecord, DimensionScore } from '../domain/types.js'; -import type { Rubric } from './rubrics.js'; -import { ALL_RUBRICS, buildJudgePrompt } from './rubrics.js'; - -/** - * Response from a single rubric evaluation by the judge LLM. - */ -export interface JudgeQuestionScore { - readonly questionId: string; - readonly score: number; - readonly evidence: string; -} - -export interface JudgeResponse { - readonly scores: readonly JudgeQuestionScore[]; -} - -/** - * Configuration for the LLM judge. - * The judgeModel MUST differ from runnerModel to avoid self-model bias. - */ -export interface JudgeConfig { - readonly judgeModel: string; - readonly runnerModel: string; -} - -/** - * The judge function type — injected dependency for I/O boundary. - * Takes a system prompt, user prompt, and model identifier. - * Returns the raw string response from the LLM. - */ -export type JudgeFn = ( - systemPrompt: string, - userPrompt: string, - model: string, -) => Promise<string>; - -/** - * Validates that the judge model differs from the runner model. - * Throws if they match — self-model bias is not permitted. - */ -export function validateJudgeConfig(config: JudgeConfig): void { - if (config.judgeModel === config.runnerModel) { - throw new Error( - `Judge model "${config.judgeModel}" must differ from runner model "${config.runnerModel}" to avoid self-model bias`, - ); - } - if (!config.judgeModel) { - throw new Error('Judge model must be specified'); - } - if (!config.runnerModel) { - throw new Error('Runner model must be specified'); - } -} - -/** - * Parses the judge LLM response into structured scores. - * Extracts JSON from the response, handling optional markdown code fences. - */ -export function parseJudgeResponse(raw: string, rubric: Rubric): JudgeResponse { - // Strip markdown code fences if present - let jsonStr = raw.trim(); - const fenceMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/); - if (fenceMatch) { - jsonStr = fenceMatch[1].trim(); - } - - let parsed: { scores?: unknown[] }; - try { - parsed = JSON.parse(jsonStr); - } catch { - throw new Error(`Judge response is not valid JSON: ${raw.slice(0, 200)}`); - } - - if (!Array.isArray(parsed.scores)) { - throw new Error('Judge response missing "scores" array'); - } - - const expectedIds = new Set(rubric.questions.map((q) => q.id)); - const scores: JudgeQuestionScore[] = []; - - for (const entry of parsed.scores) { - const e = entry as Record<string, unknown>; - - if (typeof e.questionId !== 'string' || !expectedIds.has(e.questionId)) { - throw new Error(`Unexpected or missing questionId: ${String(e.questionId)}`); - } - - const score = Number(e.score); - if (!Number.isInteger(score) || score < 1 || score > 5) { - throw new Error(`Invalid score for ${e.questionId}: ${String(e.score)} (must be 1-5)`); - } - - if (typeof e.evidence !== 'string' || e.evidence.trim().length === 0) { - throw new Error(`Missing evidence for ${e.questionId}`); - } - - scores.push({ - questionId: e.questionId as string, - score, - evidence: e.evidence as string, - }); - } - - if (scores.length !== rubric.questions.length) { - throw new Error( - `Expected ${rubric.questions.length} scores, got ${scores.length}`, - ); - } - - return { scores }; -} - -/** - * Scores a single rubric dimension against session transcripts. - * Pure function — the judge LLM call is injected via judgeFn. - * - * Returns a DimensionScore with: - * - score: average across all rubric questions (normalized to 0-1) - * - maxScore: 1 - * - details: per-question scores and evidence - */ -export async function scoreWithJudge( - sessionRecords: readonly SessionRecord[], - rubric: Rubric, - config: JudgeConfig, - judgeFn: JudgeFn, -): Promise<DimensionScore> { - validateJudgeConfig(config); - - if (sessionRecords.length === 0) { - return { - dimension: rubric.dimension, - score: 0, - maxScore: 1, - details: 'No session records to evaluate', - }; - } - - const transcripts = sessionRecords.map((r) => r.transcript); - const userPrompt = buildJudgePrompt(rubric, transcripts); - - const rawResponse = await judgeFn(rubric.systemPrompt, userPrompt, config.judgeModel); - const judgeResponse = parseJudgeResponse(rawResponse, rubric); - - // Average score across questions, normalized to 0-1 - const totalScore = judgeResponse.scores.reduce((sum, s) => sum + s.score, 0); - const maxPossible = rubric.questions.length * 5; - const normalizedScore = Math.round((totalScore / maxPossible) * 100) / 100; - - const evidenceDetails = judgeResponse.scores - .map((s) => `${s.questionId}=${s.score}/5 [${s.evidence}]`) - .join('; '); - - return { - dimension: rubric.dimension, - score: normalizedScore, - maxScore: 1, - details: evidenceDetails, - }; -} - -/** - * Scores all three qualitative dimensions using the LLM judge. - * Returns one DimensionScore per rubric (consistency, error-correction, architectural-quality). - */ -export async function scoreAllJudgeDimensions( - sessionRecords: readonly SessionRecord[], - config: JudgeConfig, - judgeFn: JudgeFn, - rubrics: readonly Rubric[] = ALL_RUBRICS, -): Promise<DimensionScore[]> { - validateJudgeConfig(config); - - const scores: DimensionScore[] = []; - for (const rubric of rubrics) { - const score = await scoreWithJudge(sessionRecords, rubric, config, judgeFn); - scores.push(score); - } - - return scores; -} diff --git a/evals/src/scoring/protocol-adherence-scorer.ts b/evals/src/scoring/protocol-adherence-scorer.ts deleted file mode 100644 index 50d258d9..00000000 --- a/evals/src/scoring/protocol-adherence-scorer.ts +++ /dev/null @@ -1,150 +0,0 @@ -import type { DimensionScore, JumboLifecycleAudit, SessionRecord } from '../domain/types.js'; - -const DIMENSION = 'protocol-adherence'; - -export type ProtocolStep = - | 'session-start' - | 'goal-start' - | 'in-session-captures' - | 'progress-updates' - | 'goal-submit' - | 'session-end'; - -export const PROTOCOL_STEPS: readonly ProtocolStep[] = [ - 'session-start', - 'goal-start', - 'in-session-captures', - 'progress-updates', - 'goal-submit', - 'session-end', -] as const; - -export interface ProtocolStepResult { - readonly step: ProtocolStep; - readonly passed: boolean; -} - -export interface SessionAdherence { - readonly sessionNumber: number; - readonly steps: readonly ProtocolStepResult[]; - readonly score: number; -} - -function stepResultsForAudit(audit: JumboLifecycleAudit): ProtocolStepResult[] { - return [ - { step: 'session-start', passed: audit.sessionStartExecuted }, - { step: 'goal-start', passed: audit.goalStartExecuted }, - { step: 'in-session-captures', passed: audit.inSessionCapturesExecuted }, - { step: 'progress-updates', passed: audit.progressUpdatesExecuted }, - { step: 'goal-submit', passed: audit.goalSubmitExecuted }, - { step: 'session-end', passed: audit.sessionEndExecuted }, - ]; -} - -/** - * Per-session protocol adherence. Each step is a boolean derived from the - * session's JumboLifecycleAudit (Jumbo's own state, not transcript parsing). - * Aggregate session score = passed steps / total steps. Sessions without an - * audit (e.g. baseline sessions) yield an empty step list and score 0. - */ -export function adherenceForSession(record: SessionRecord): SessionAdherence { - const audit = record.jumboLifecycleAudit; - if (!audit) { - return { sessionNumber: record.sessionNumber, steps: [], score: 0 }; - } - const steps = stepResultsForAudit(audit); - const passed = steps.filter((s) => s.passed).length; - return { - sessionNumber: record.sessionNumber, - steps, - score: steps.length === 0 ? 0 : passed / steps.length, - }; -} - -function summarizeFailures(adherences: readonly SessionAdherence[]): string { - const failureCounts = new Map<ProtocolStep, number[]>(); - for (const adherence of adherences) { - for (const step of adherence.steps) { - if (step.passed) continue; - const sessions = failureCounts.get(step.step) ?? []; - sessions.push(adherence.sessionNumber); - failureCounts.set(step.step, sessions); - } - } - if (failureCounts.size === 0) return 'all-steps-passed'; - return [...failureCounts.entries()] - .map(([step, sessions]) => `${step}-missed-in-sessions:${sessions.join(',')}`) - .join('; '); -} - -/** - * Aggregate protocol adherence across all sessions in the Jumbo arm. Score is - * the mean of per-session scores. Surfaces protocol-failure modes (which - * steps were skipped, in which sessions) in `details` so non-adherence is - * first-class signal — not noise to suppress. - */ -export function scoreProtocolAdherence(records: readonly SessionRecord[]): DimensionScore { - const audited = records.filter((r) => r.jumboLifecycleAudit !== undefined); - if (audited.length === 0) { - return { - dimension: DIMENSION, - score: 0, - maxScore: 1, - details: 'No lifecycle audits recorded; protocol adherence cannot be measured.', - }; - } - const adherences = audited.map(adherenceForSession); - const meanScore = adherences.reduce((sum, a) => sum + a.score, 0) / adherences.length; - const totalSteps = adherences.reduce((sum, a) => sum + a.steps.length, 0); - const passedSteps = adherences.reduce( - (sum, a) => sum + a.steps.filter((s) => s.passed).length, - 0, - ); - - return { - dimension: DIMENSION, - score: Math.round(meanScore * 100) / 100, - maxScore: 1, - details: [ - `sessions-audited=${audited.length}`, - `steps-passed=${passedSteps}/${totalSteps}`, - summarizeFailures(adherences), - ].join('; '), - }; -} - -export function baselineProtocolAdherenceScore(): DimensionScore { - return { - dimension: DIMENSION, - score: 0, - maxScore: 0, - details: 'Not applicable: baseline runs do not execute the Jumbo lifecycle protocol.', - }; -} - -export function scoreProtocolAdherenceTimeline( - records: readonly SessionRecord[], -): DimensionScore[] { - return records.map((record) => { - const adherence = adherenceForSession(record); - if (adherence.steps.length === 0) { - return { - dimension: DIMENSION, - score: 0, - maxScore: 1, - details: 'No lifecycle audit for this session.', - }; - } - const passed = adherence.steps.filter((s) => s.passed).map((s) => s.step); - const failed = adherence.steps.filter((s) => !s.passed).map((s) => s.step); - return { - dimension: DIMENSION, - score: Math.round(adherence.score * 100) / 100, - maxScore: 1, - details: [ - `passed: ${passed.length > 0 ? passed.join(',') : 'none'}`, - `failed: ${failed.length > 0 ? failed.join(',') : 'none'}`, - ].join('; '), - }; - }); -} diff --git a/evals/src/scoring/rubrics.ts b/evals/src/scoring/rubrics.ts deleted file mode 100644 index 9ec3a5ec..00000000 --- a/evals/src/scoring/rubrics.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Locked scoring rubrics for LLM-as-judge evaluation. - * - * Each rubric defines a qualitative dimension with: - * - Specific evaluation questions - * - A fixed scoring scale (1-5) - * - Evidence anchoring requirements (judge must cite transcript/file evidence) - * - * Rubrics are immutable — changes require a new version, not in-place mutation. - */ - -export interface RubricQuestion { - readonly id: string; - readonly question: string; - readonly scoringScale: readonly ScalePoint[]; -} - -export interface ScalePoint { - readonly score: number; - readonly label: string; - readonly description: string; -} - -export interface Rubric { - readonly dimension: string; - readonly version: number; - readonly description: string; - readonly systemPrompt: string; - readonly questions: readonly RubricQuestion[]; -} - -const FIVE_POINT_SCALE: readonly ScalePoint[] = [ - { score: 1, label: 'None', description: 'No evidence of this quality' }, - { score: 2, label: 'Minimal', description: 'Sparse or inconsistent evidence' }, - { score: 3, label: 'Moderate', description: 'Present but with notable gaps' }, - { score: 4, label: 'Strong', description: 'Consistent with minor gaps' }, - { score: 5, label: 'Excellent', description: 'Thorough and consistent throughout' }, -]; - -export const CONSISTENCY_RUBRIC: Rubric = { - dimension: 'consistency', - version: 1, - description: 'Evaluates whether patterns, conventions, and decisions established in early sessions persist through later sessions.', - systemPrompt: [ - 'You are an expert software engineering evaluator.', - 'You are reviewing transcripts from a multi-session coding project.', - 'Your task is to evaluate CONSISTENCY: whether patterns established early in the project are maintained in later sessions.', - 'You MUST cite specific evidence from the transcripts for every score.', - 'For each question, provide: a score (1-5), and evidence (exact quotes or file references from the transcripts).', - 'Respond in valid JSON format.', - ].join(' '), - questions: [ - { - id: 'naming-conventions', - question: 'Are naming conventions (variable names, file names, function names) established in early sessions consistently followed in later sessions?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'architectural-patterns', - question: 'Are architectural patterns (module structure, import patterns, abstraction layers) introduced early maintained throughout later sessions?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'style-coherence', - question: 'Does the code style (formatting, error handling approach, logging patterns) remain coherent across sessions, or does it drift as if written by different authors?', - scoringScale: FIVE_POINT_SCALE, - }, - ], -}; - -export const ERROR_CORRECTION_RUBRIC: Rubric = { - dimension: 'error-correction', - version: 1, - description: 'Evaluates whether mistakes and corrections are properly handled — errors caught, fixed, and not reintroduced in subsequent sessions.', - systemPrompt: [ - 'You are an expert software engineering evaluator.', - 'You are reviewing transcripts from a multi-session coding project.', - 'Your task is to evaluate ERROR CORRECTION: whether mistakes are caught, fixed properly, and not reintroduced.', - 'You MUST cite specific evidence from the transcripts for every score.', - 'For each question, provide: a score (1-5), and evidence (exact quotes or file references from the transcripts).', - 'Respond in valid JSON format.', - ].join(' '), - questions: [ - { - id: 'error-detection', - question: 'When errors or bugs are introduced, are they detected and acknowledged in the same or subsequent session?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'fix-persistence', - question: 'Once a bug is fixed or a correction is applied, does the fix persist in later sessions, or is the same error reintroduced?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'correction-integration', - question: 'When mid-project corrections are given (scope changes, new constraints), are they integrated thoroughly or only partially applied?', - scoringScale: FIVE_POINT_SCALE, - }, - ], -}; - -export const ARCHITECTURAL_QUALITY_RUBRIC: Rubric = { - dimension: 'architectural-quality', - version: 1, - description: 'Evaluates the overall architectural quality of the produced code across sessions — separation of concerns, dependency management, and design coherence.', - systemPrompt: [ - 'You are an expert software engineering evaluator.', - 'You are reviewing transcripts from a multi-session coding project.', - 'Your task is to evaluate ARCHITECTURAL QUALITY: the design coherence and structural quality of the code produced.', - 'You MUST cite specific evidence from the transcripts for every score.', - 'For each question, provide: a score (1-5), and evidence (exact quotes or file references from the transcripts).', - 'Respond in valid JSON format.', - ].join(' '), - questions: [ - { - id: 'separation-of-concerns', - question: 'Does the code maintain clear separation of concerns across modules, or do responsibilities bleed across boundaries over sessions?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'dependency-direction', - question: 'Do dependencies flow in a consistent direction (e.g., domain does not depend on infrastructure), or do circular or inverted dependencies appear in later sessions?', - scoringScale: FIVE_POINT_SCALE, - }, - { - id: 'design-evolution', - question: 'As the project evolves across sessions, does the architecture adapt coherently (refactoring when needed), or does it accumulate ad-hoc patches?', - scoringScale: FIVE_POINT_SCALE, - }, - ], -}; - -export const ALL_RUBRICS: readonly Rubric[] = [ - CONSISTENCY_RUBRIC, - ERROR_CORRECTION_RUBRIC, - ARCHITECTURAL_QUALITY_RUBRIC, -]; - -/** - * Builds the judge prompt for a given rubric and session transcripts. - * The prompt includes the rubric questions, scoring scale, and the transcripts - * to evaluate. The expected response format is enforced via the prompt. - */ -export function buildJudgePrompt( - rubric: Rubric, - transcripts: readonly string[], -): string { - const scaleDescription = rubric.questions[0].scoringScale - .map((sp) => ` ${sp.score} - ${sp.label}: ${sp.description}`) - .join('\n'); - - const questionsBlock = rubric.questions - .map((q, i) => `Question ${i + 1} (id: "${q.id}"): ${q.question}`) - .join('\n\n'); - - const transcriptsBlock = transcripts - .map((t, i) => `=== Session ${i + 1} Transcript ===\n${t}`) - .join('\n\n'); - - return [ - `Evaluate the following multi-session project transcripts for: ${rubric.description}`, - '', - '## Scoring Scale', - scaleDescription, - '', - '## Questions', - questionsBlock, - '', - '## Transcripts', - transcriptsBlock, - '', - '## Required Response Format', - 'Respond with a JSON object matching this structure exactly:', - '```json', - '{', - ' "scores": [', - ' {', - ` "questionId": "${rubric.questions[0].id}",`, - ' "score": <1-5>,', - ' "evidence": "<exact quote or file reference from transcripts>"', - ' }', - ' ]', - '}', - '```', - `You MUST provide a score entry for each of the ${rubric.questions.length} questions.`, - 'You MUST include specific evidence (exact quotes or file references) for every score. A score without evidence is invalid.', - ].join('\n'); -} diff --git a/evals/src/scoring/structural-assertion-scorer.ts b/evals/src/scoring/structural-assertion-scorer.ts deleted file mode 100644 index a7051d2b..00000000 --- a/evals/src/scoring/structural-assertion-scorer.ts +++ /dev/null @@ -1,217 +0,0 @@ -import type { - DimensionScore, - SessionRecord, - StructuralAssertion, - StructuralMatcher, -} from '../domain/index.js'; - -/** - * Pure deterministic scorer for structural assertions — the primary - * knowledge-retention signal (GOAL.md §"Structural assertions vs. keyword - * matching"). Each assertion is a declarative check on the workspace artefacts - * of the session it is due in. Retention is measured on the files the agent - * actually produced, never on transcript keyword presence, so an agent that - * merely echoes a prompt cannot score retention it did not implement. - * - * A session's retention score is the fraction of assertions due in that session - * that pass; the aggregate score is the fraction of all assertions that pass in - * their respective due sessions. - */ - -const DIMENSION = 'structural-retention'; - -/** Converts a restricted glob (`*` within a segment, `**` across segments) to a RegExp. */ -function globToRegExp(glob: string): RegExp { - let out = ''; - for (let i = 0; i < glob.length; i++) { - const ch = glob[i]; - if (ch === '*') { - if (glob[i + 1] === '*') { - out += '.*'; - i++; - // consume a trailing slash so `src/**/x` also matches `src/x` - if (glob[i + 1] === '/') i++; - } else { - out += '[^/]*'; - } - } else if ('\\^$+?.()|[]{}'.includes(ch)) { - out += `\\${ch}`; - } else { - out += ch; - } - } - return new RegExp(`^${out}$`); -} - -interface MatchedFiles { - readonly paths: readonly string[]; - /** Concatenated content of every matched file. */ - readonly content: string; -} - -function matchFiles(record: SessionRecord | undefined, glob: string): MatchedFiles { - const files = record?.workspaceSnapshot?.files ?? []; - const re = globToRegExp(glob); - const matched = files.filter((f) => re.test(f.path)); - return { - paths: matched.map((f) => f.path), - content: matched.map((f) => f.content).join('\n'), - }; -} - -interface MatcherResult { - readonly pass: boolean; - readonly reason?: string; -} - -function evaluateMatcher(matcher: StructuralMatcher, matched: MatchedFiles): MatcherResult { - if (matcher.kind === 'fileExists') { - return matched.paths.length > 0 - ? { pass: true } - : { pass: false, reason: 'no file matched glob' }; - } - - // `notContains` is satisfied vacuously when nothing matches; every other - // matcher needs at least one matched file to evaluate against. - if (matched.paths.length === 0 && matcher.kind !== 'notContains') { - return { pass: false, reason: 'no file matched glob' }; - } - - switch (matcher.kind) { - case 'matchesRegex': { - let re: RegExp; - try { - re = new RegExp(matcher.pattern, matcher.flags); - } catch (err) { - return { pass: false, reason: `invalid regex: ${(err as Error).message}` }; - } - return re.test(matched.content) - ? { pass: true } - : { pass: false, reason: `pattern /${matcher.pattern}/ not found` }; - } - case 'containsAll': { - const missing = matcher.substrings.filter((s) => !matched.content.includes(s)); - return missing.length === 0 - ? { pass: true } - : { pass: false, reason: `missing: ${missing.join(', ')}` }; - } - case 'containsAny': { - const found = matcher.substrings.some((s) => matched.content.includes(s)); - return found - ? { pass: true } - : { pass: false, reason: `none of: ${matcher.substrings.join(', ')}` }; - } - case 'notContains': { - const present = matcher.substrings.filter((s) => matched.content.includes(s)); - return present.length === 0 - ? { pass: true } - : { pass: false, reason: `forbidden present: ${present.join(', ')}` }; - } - case 'exportsSymbol': { - const sym = escapeRegExp(matcher.symbol); - const declared = new RegExp( - `export\\s+(?:default\\s+)?(?:abstract\\s+)?(?:async\\s+)?(?:class|function|const|let|var|interface|type|enum)\\s+${sym}\\b`, - ); - const reExported = new RegExp(`export\\s*(?:type\\s*)?\\{[^}]*\\b${sym}\\b[^}]*\\}`); - return declared.test(matched.content) || reExported.test(matched.content) - ? { pass: true } - : { pass: false, reason: `symbol '${matcher.symbol}' not exported` }; - } - } -} - -function escapeRegExp(s: string): string { - return s.replace(/[\\^$+?.()|[\]{}*]/g, '\\$&'); -} - -function recordForSession( - records: readonly SessionRecord[], - sessionNumber: number, -): SessionRecord | undefined { - return records.find((r) => r.sessionNumber === sessionNumber); -} - -interface AssertionOutcome { - readonly assertion: StructuralAssertion; - readonly pass: boolean; - readonly reason?: string; -} - -function evaluateAssertion( - assertion: StructuralAssertion, - records: readonly SessionRecord[], -): AssertionOutcome { - const record = recordForSession(records, assertion.sessionNumber); - if (!record) { - return { assertion, pass: false, reason: `no record for session ${assertion.sessionNumber}` }; - } - const matched = matchFiles(record, assertion.file); - const result = evaluateMatcher(assertion.matcher, matched); - return { assertion, pass: result.pass, reason: result.reason }; -} - -/** - * Aggregate structural-retention score across all assertions, each evaluated - * against the workspace snapshot of the session it is due in. - */ -export function scoreStructuralAssertions( - sessionRecords: readonly SessionRecord[], - assertions: readonly StructuralAssertion[], -): DimensionScore { - if (assertions.length === 0) { - return { - dimension: DIMENSION, - score: 1, - maxScore: 1, - details: 'No structural assertions defined — trivially satisfied', - }; - } - - const outcomes = assertions.map((a) => evaluateAssertion(a, sessionRecords)); - const passed = outcomes.filter((o) => o.pass).length; - const failed = outcomes.filter((o) => !o.pass); - const score = passed / outcomes.length; - - const details = [ - `${passed}/${outcomes.length} structural assertions passed`, - failed.length > 0 - ? `failed: ${failed.map((o) => `${o.assertion.id} (${o.reason})`).join('; ')}` - : null, - ] - .filter(Boolean) - .join('; '); - - return { - dimension: DIMENSION, - score: Math.round(score * 100) / 100, - maxScore: 1, - details, - }; -} - -/** - * Per-session trajectory: one DimensionScore per session that has assertions - * due, where the score is the fraction of that session's due assertions that - * pass. Sessions with no assertions due are omitted. - */ -export function scoreStructuralAssertionsTimeline( - sessionRecords: readonly SessionRecord[], - assertions: readonly StructuralAssertion[], -): DimensionScore[] { - if (assertions.length === 0) return []; - - const sessionNumbers = [...new Set(assertions.map((a) => a.sessionNumber))].sort((a, b) => a - b); - - return sessionNumbers.map((sessionNumber) => { - const due = assertions.filter((a) => a.sessionNumber === sessionNumber); - const outcomes = due.map((a) => evaluateAssertion(a, sessionRecords)); - const passed = outcomes.filter((o) => o.pass).length; - const score = passed / due.length; - return { - dimension: DIMENSION, - score: Math.round(score * 100) / 100, - maxScore: 1, - details: `session ${sessionNumber}: ${passed}/${due.length} structural assertions passed`, - }; - }); -} diff --git a/evals/src/scoring/token-efficiency-scorer.ts b/evals/src/scoring/token-efficiency-scorer.ts deleted file mode 100644 index 68363194..00000000 --- a/evals/src/scoring/token-efficiency-scorer.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { SessionRecord, DimensionScore } from '../domain/types.js'; - -/** - * Pure scorer: computes token efficiency as total tokens normalized - * by outcome quality. Lower tokens-per-quality-point is better. - * - * Score is expressed as a ratio: baseline_tpq / jumbo_tpq. - * A score > 1 means Jumbo is more token-efficient. - * A score < 1 means baseline is more token-efficient. - * A score of 1 means they're equal. - * - * When used for comparison, the caller passes one run's records - * and the average quality score for that run. - */ -export function scoreTokenEfficiency( - sessionRecords: readonly SessionRecord[], - qualityScore: number, -): DimensionScore { - const totalInput = sessionRecords.reduce((sum, r) => sum + (r.inputTokens ?? 0), 0); - const totalOutput = sessionRecords.reduce((sum, r) => sum + (r.outputTokens ?? 0), 0); - const totalTokens = totalInput + totalOutput; - - if (totalTokens === 0) { - return { - dimension: 'token-efficiency', - score: 0, - maxScore: 1, - details: 'No token data available', - }; - } - - const tokensPerQualityPoint = qualityScore > 0 ? totalTokens / qualityScore : Infinity; - - return { - dimension: 'token-efficiency', - score: totalTokens, - maxScore: totalTokens, - details: `${totalTokens} total tokens (${totalInput} in, ${totalOutput} out); ${qualityScore > 0 ? tokensPerQualityPoint.toFixed(0) : 'N/A'} tokens/quality-point`, - }; -} - -/** - * Computes comparative token efficiency between Jumbo and baseline. - * Returns a DimensionScore where: - * score > 0 means Jumbo is more efficient (fewer tokens per quality point) - * score < 0 means baseline is more efficient - * score = 0 means equal or no data - */ -export function compareTokenEfficiency( - jumboRecords: readonly SessionRecord[], - baselineRecords: readonly SessionRecord[], - jumboQuality: number, - baselineQuality: number, -): DimensionScore { - const jumboTokens = jumboRecords.reduce((sum, r) => sum + (r.inputTokens ?? 0) + (r.outputTokens ?? 0), 0); - const baselineTokens = baselineRecords.reduce((sum, r) => sum + (r.inputTokens ?? 0) + (r.outputTokens ?? 0), 0); - - if (jumboTokens === 0 && baselineTokens === 0) { - return { - dimension: 'token-efficiency', - score: 0, - maxScore: 1, - details: 'No token data available for either run', - }; - } - - const jumboTpq = jumboQuality > 0 ? jumboTokens / jumboQuality : Infinity; - const baselineTpq = baselineQuality > 0 ? baselineTokens / baselineQuality : Infinity; - - // Normalize: positive means Jumbo is better (fewer tokens per quality point) - let efficiency: number; - if (jumboTpq === Infinity && baselineTpq === Infinity) { - efficiency = 0; - } else if (jumboTpq === Infinity) { - efficiency = -1; - } else if (baselineTpq === Infinity) { - efficiency = 1; - } else if (baselineTpq === 0 && jumboTpq === 0) { - efficiency = 0; - } else { - // Ratio-based: how much more efficient is Jumbo? - efficiency = baselineTpq > 0 ? Math.round(((baselineTpq - jumboTpq) / baselineTpq) * 100) / 100 : 0; - } - - return { - dimension: 'token-efficiency', - score: efficiency, - maxScore: 1, - details: `jumbo: ${jumboTokens} tokens (${jumboTpq === Infinity ? 'N/A' : jumboTpq.toFixed(0)} tpq); baseline: ${baselineTokens} tokens (${baselineTpq === Infinity ? 'N/A' : baselineTpq.toFixed(0)} tpq)`, - }; -} - -/** - * Per-session token usage for timeline display. - */ -export function tokenUsageTimeline( - sessionRecords: readonly SessionRecord[], -): DimensionScore[] { - const sorted = [...sessionRecords].sort((a, b) => a.sessionNumber - b.sessionNumber); - - return sorted.map((r) => { - const input = r.inputTokens ?? 0; - const output = r.outputTokens ?? 0; - const total = input + output; - - return { - dimension: 'token-usage', - score: total, - maxScore: total || 1, - details: `session ${r.sessionNumber}: ${total} tokens (${input} in, ${output} out)`, - }; - }); -} diff --git a/evals/src/storage/index.ts b/evals/src/storage/index.ts deleted file mode 100644 index a4c3f198..00000000 --- a/evals/src/storage/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { ResultStore } from './result-store.js'; -export { JsonResultStore } from './json-result-store.js'; diff --git a/evals/src/storage/json-result-store.ts b/evals/src/storage/json-result-store.ts deleted file mode 100644 index 33d5eed5..00000000 --- a/evals/src/storage/json-result-store.ts +++ /dev/null @@ -1,201 +0,0 @@ -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; -import { randomUUID } from 'node:crypto'; -import type { EvalRunRecord, ReplicationReport, RunControlFile, RunHeartbeat, TestScenario, SessionRecord, TestResult } from '../domain/types.js'; -import type { ResultStore } from './result-store.js'; - -function isEnoent(err: unknown): boolean { - return typeof err === 'object' && err !== null && (err as Record<string, unknown>).code === 'ENOENT'; -} - -/** - * JSON file-backed ResultStore implementation. - * Each entity type gets its own directory under the base path. - * Each entity is stored as an individual JSON file keyed by id. - */ -export class JsonResultStore implements ResultStore { - private readonly scenariosDir: string; - private readonly sessionsDir: string; - private readonly resultsDir: string; - private readonly runsDir: string; - private readonly heartbeatWrites = new Map<string, Promise<void>>(); - private readonly controlWrites = new Map<string, Promise<void>>(); - - constructor(private readonly basePath: string) { - this.scenariosDir = path.join(basePath, 'scenarios'); - this.sessionsDir = path.join(basePath, 'sessions'); - this.resultsDir = path.join(basePath, 'results'); - this.runsDir = path.join(basePath, 'runs'); - } - - async initialize(): Promise<void> { - await fs.mkdir(this.scenariosDir, { recursive: true }); - await fs.mkdir(this.sessionsDir, { recursive: true }); - await fs.mkdir(this.resultsDir, { recursive: true }); - await fs.mkdir(this.runsDir, { recursive: true }); - } - - async saveScenario(scenario: TestScenario): Promise<void> { - const filePath = path.join(this.scenariosDir, `${scenario.id}.json`); - await fs.writeFile(filePath, JSON.stringify(scenario, null, 2), 'utf-8'); - } - - async getScenario(id: string): Promise<TestScenario | null> { - return this.readJson<TestScenario>(path.join(this.scenariosDir, `${id}.json`)); - } - - async listScenarios(): Promise<TestScenario[]> { - return this.readAllJson<TestScenario>(this.scenariosDir); - } - - async saveSessionRecord(record: SessionRecord): Promise<void> { - const filePath = path.join(this.sessionsDir, `${record.id}.json`); - await fs.writeFile(filePath, JSON.stringify(record, null, 2), 'utf-8'); - } - - async getSessionRecords(scenarioId: string): Promise<SessionRecord[]> { - const all = await this.readAllJson<SessionRecord>(this.sessionsDir); - return all.filter((r) => r.scenarioId === scenarioId); - } - - async saveTestResult(result: TestResult): Promise<void> { - const filePath = path.join(this.resultsDir, `${result.id}.json`); - await fs.writeFile(filePath, JSON.stringify(result, null, 2), 'utf-8'); - } - - async getTestResult(id: string): Promise<TestResult | null> { - return this.readJson<TestResult>(path.join(this.resultsDir, `${id}.json`)); - } - - async listTestResults(scenarioId?: string): Promise<TestResult[]> { - const all = await this.readAllJson<TestResult>(this.resultsDir); - if (scenarioId) { - return all.filter((r) => r.scenarioId === scenarioId); - } - return all; - } - - async saveRunRecord(record: EvalRunRecord): Promise<void> { - const runDir = this.runDir(record.runId); - await fs.mkdir(runDir, { recursive: true }); - await fs.writeFile(path.join(runDir, 'run.json'), JSON.stringify(record, null, 2), 'utf-8'); - } - - async getRunRecord(runId: string): Promise<EvalRunRecord | null> { - return this.readJson<EvalRunRecord>(path.join(this.runDir(runId), 'run.json')); - } - - async listRunRecords(scenarioId?: string): Promise<EvalRunRecord[]> { - const runDirs = await this.readRunDirs(); - const records = await Promise.all( - runDirs.map((runDir) => this.readJson<EvalRunRecord>(path.join(runDir, 'run.json'))), - ); - const all = records.filter((record): record is EvalRunRecord => record !== null); - if (scenarioId) { - return all.filter((record) => record.scenarioId === scenarioId); - } - return all; - } - - async writeHeartbeat(runId: string, heartbeat: RunHeartbeat): Promise<void> { - const runDir = this.runDir(runId); - await fs.mkdir(runDir, { recursive: true }); - const filePath = path.join(runDir, 'state.json'); - const previous = this.heartbeatWrites.get(filePath) ?? Promise.resolve(); - const next = previous.then(() => this.atomicWriteJson(filePath, heartbeat)); - this.heartbeatWrites.set(filePath, next.catch(() => {})); - await next; - } - - async readHeartbeat(runId: string): Promise<RunHeartbeat | null> { - return this.readJson<RunHeartbeat>(path.join(this.runDir(runId), 'state.json')); - } - - async writeRunControl(runId: string, control: RunControlFile): Promise<void> { - const runDir = this.runDir(runId); - await fs.mkdir(runDir, { recursive: true }); - const filePath = path.join(runDir, 'control.json'); - const previous = this.controlWrites.get(filePath) ?? Promise.resolve(); - const next = previous.then(() => this.atomicWriteJson(filePath, control)); - this.controlWrites.set(filePath, next.catch(() => {})); - await next; - } - - async readRunControl(runId: string): Promise<RunControlFile | null> { - return this.readJson<RunControlFile>(path.join(this.runDir(runId), 'control.json')); - } - - async saveReplicationReport(runId: string, report: ReplicationReport): Promise<void> { - const runDir = this.runDir(runId); - await fs.mkdir(runDir, { recursive: true }); - await this.atomicWriteJson(path.join(runDir, 'replication.json'), report); - } - - async getReplicationReport(runId: string): Promise<ReplicationReport | null> { - return this.readJson<ReplicationReport>(path.join(this.runDir(runId), 'replication.json')); - } - - private runDir(runId: string): string { - return path.join(this.runsDir, runId); - } - - private async atomicWriteJson(filePath: string, value: unknown): Promise<void> { - const tmpPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await fs.writeFile(tmpPath, JSON.stringify(value, null, 2), 'utf-8'); - await renameWithRetry(tmpPath, filePath); - } - - private async readRunDirs(): Promise<string[]> { - try { - const entries = await fs.readdir(this.runsDir, { withFileTypes: true }); - return entries.filter((entry) => entry.isDirectory()).map((entry) => path.join(this.runsDir, entry.name)); - } catch (err: unknown) { - if (isEnoent(err)) return []; - throw err; - } - } - - private async readJson<T>(filePath: string): Promise<T | null> { - try { - const content = await fs.readFile(filePath, 'utf-8'); - return JSON.parse(content) as T; - } catch (err: unknown) { - if (isEnoent(err)) return null; - throw err; - } - } - - private async readAllJson<T>(dirPath: string): Promise<T[]> { - try { - const files = await fs.readdir(dirPath); - const jsonFiles = files.filter((f) => f.endsWith('.json')); - const items = await Promise.all( - jsonFiles.map(async (f) => { - const content = await fs.readFile(path.join(dirPath, f), 'utf-8'); - return JSON.parse(content) as T; - }), - ); - return items; - } catch (err: unknown) { - if (isEnoent(err)) return []; - throw err; - } - } -} - -async function renameWithRetry(from: string, to: string): Promise<void> { - const retryable = new Set(['EPERM', 'EBUSY', 'EACCES']); - let lastErr: unknown; - for (let attempt = 0; attempt < 10; attempt++) { - try { - await fs.rename(from, to); - return; - } catch (err: unknown) { - lastErr = err; - const code = typeof err === 'object' && err !== null ? (err as Record<string, unknown>).code : undefined; - if (!retryable.has(String(code))) throw err; - await new Promise((resolve) => setTimeout(resolve, 5 * (attempt + 1))); - } - } - throw lastErr; -} diff --git a/evals/src/storage/result-store.ts b/evals/src/storage/result-store.ts deleted file mode 100644 index 9a81359c..00000000 --- a/evals/src/storage/result-store.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { EvalRunRecord, ReplicationReport, RunControlFile, RunHeartbeat, TestScenario, SessionRecord, TestResult } from '../domain/types.js'; - -/** - * Abstract storage interface for eval artifacts. - * Backed by JSON files now, designed for SQLite graduation later. - * Consumers depend on this interface, not the implementation. - */ -export interface ResultStore { - saveScenario(scenario: TestScenario): Promise<void>; - getScenario(id: string): Promise<TestScenario | null>; - listScenarios(): Promise<TestScenario[]>; - - saveSessionRecord(record: SessionRecord): Promise<void>; - getSessionRecords(scenarioId: string): Promise<SessionRecord[]>; - - saveTestResult(result: TestResult): Promise<void>; - getTestResult(id: string): Promise<TestResult | null>; - listTestResults(scenarioId?: string): Promise<TestResult[]>; - - saveRunRecord(record: EvalRunRecord): Promise<void>; - getRunRecord(runId: string): Promise<EvalRunRecord | null>; - listRunRecords(scenarioId?: string): Promise<EvalRunRecord[]>; - writeHeartbeat(runId: string, heartbeat: RunHeartbeat): Promise<void>; - readHeartbeat(runId: string): Promise<RunHeartbeat | null>; - - writeRunControl(runId: string, control: RunControlFile): Promise<void>; - readRunControl(runId: string): Promise<RunControlFile | null>; - - /** Persists the Outcome 5 replication report for a run (one statistical artifact per runId). */ - saveReplicationReport(runId: string, report: ReplicationReport): Promise<void>; - getReplicationReport(runId: string): Promise<ReplicationReport | null>; -} diff --git a/evals/tests/integration/adapter-jumbo-reachable.test.ts b/evals/tests/integration/adapter-jumbo-reachable.test.ts deleted file mode 100644 index d9796e2c..00000000 --- a/evals/tests/integration/adapter-jumbo-reachable.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { ClaudeCodeAdapter } from '../../src/harness/claude-code-adapter.js'; -import { CodexCliAdapter } from '../../src/harness/codex-cli-adapter.js'; -import { GeminiCliAdapter } from '../../src/harness/gemini-cli-adapter.js'; -import { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; - -/** - * Goal 26e636a6 success criterion: each adapter passes a regression test - * confirming `jumbo --version` succeeds when invoked via the adapter's - * tool path inside a real session — i.e., a real workdir seeded with the - * adapter's permission/config artifact, with the same LocalExecutor and - * env the adapter will use during sessions. Requires the `jumbo` binary - * on PATH; integration scope only. - */ -describe('adapter jumbo reachability (integration)', () => { - const adapters: ReadonlyArray<readonly [string, HarnessAdapter]> = [ - ['claude-code', new ClaudeCodeAdapter()], - ['codex-cli', new CodexCliAdapter()], - ['gemini-cli', new GeminiCliAdapter()], - ]; - - let workDir: string; - const executor = new LocalExecutor(); - - beforeEach(async () => { - workDir = await mkdtemp(join(tmpdir(), 'jumbo-reachable-')); - }); - - afterEach(async () => { - await rm(workDir, { recursive: true, force: true }); - }); - - for (const [name, adapter] of adapters) { - it(`${name}: jumbo --version succeeds in a real seeded workdir`, async () => { - await adapter.seedToolPermissions(workDir); - - const result = await executor.exec(workDir, ['jumbo', '--version']); - - expect(result.exitCode).toBe(0); - expect(result.stdout.toLowerCase()).toMatch(/jumbo|\d+\.\d+/); - }); - } -}); diff --git a/evals/tests/unit/ab-runner.test.ts b/evals/tests/unit/ab-runner.test.ts deleted file mode 100644 index 93f28525..00000000 --- a/evals/tests/unit/ab-runner.test.ts +++ /dev/null @@ -1,1616 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { - buildJumboLifecyclePrompt, - JumboBaselineLeakError, - JumboInitError, - JumboPlanGoalRegistrationError, - JumboPlanSeedError, - JumboReachabilityError, - getSessionPrompt, - runABComparison, -} from '../../src/ab-runner.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; -import type { SessionRecord, TestResult } from '../../src/domain/types.js'; -import { ClaudeCodeAdapter } from '../../src/harness/claude-code-adapter.js'; -import { CodexCliAdapter } from '../../src/harness/codex-cli-adapter.js'; -import { GeminiCliAdapter } from '../../src/harness/gemini-cli-adapter.js'; - -function createMockExecutor(options?: { - failJumboInit?: boolean; - // Force `jumbo --version` to fail in the jumbo arm (or both). Baseline - // always fails by default because the shim is installed; pass - // `baselineJumboLeak: true` to simulate a leak where baseline jumbo is - // unexpectedly reachable despite the shim. - failJumboVersion?: 'jumbo' | 'both'; - baselineJumboLeak?: boolean; - jumboMemoryOutputs?: Partial<Record<string, unknown>>; - goalAddIds?: readonly string[]; - failJumboPlanEntry?: 'decision' | 'component' | 'invariant' | 'dependency' | 'relation'; - goalShowStatusBefore?: string; - goalShowStatusAfter?: string; - goalVersionBefore?: number; - goalVersionAfter?: number; - sessionsTotalBefore?: number; - sessionsTotalAfter?: number; - sessionsEndedBefore?: number; - sessionsEndedAfter?: number; -}): LocalExecutor & { - execCalls: Array<{ workDir: string; command: string[]; stdin?: string; env?: Record<string, string | undefined> }>; - shimInstalledFor: string[]; -} { - const execCalls: Array<{ workDir: string; command: string[]; stdin?: string; env?: Record<string, string | undefined> }> = []; - const shimInstalledFor: string[] = []; - let dirCounter = 0; - let goalAddCount = 0; - // Tracks how many lifecycle-audit "goal show" calls have run per workDir. - // Odd calls are pre-harness snapshots; even calls are post-harness audits. - const goalShowCallsByDir = new Map<string, number>(); - const sessionsAllCallsByDir = new Map<string, number>(); - const sessionsEndedCallsByDir = new Map<string, number>(); - // First memory-list call for a given (workDir, kind) is the pre-snapshot; - // subsequent calls are post-snapshot (or the lifecycle audit's decisions - // list). The agent's "registrations" are encoded as the diff: pre returns - // empty, post returns the configured output. - const memoryListCallsByDirKind = new Map<string, number>(); - - return { - execCalls, - shimInstalledFor, - createWorkDir: async (prefix?: string) => { - dirCounter++; - return `/tmp/${prefix ?? 'eval-'}${dirCounter}`; - }, - installJumboShim: async (workDir: string) => { - shimInstalledFor.push(workDir); - return { env: { PATH: `${workDir}/.eval-bin:/usr/bin` } }; - }, - exec: async (workDir: string, command: string[], execOptions?: { stdin?: string; env?: Record<string, string | undefined> }) => { - execCalls.push({ workDir, command, stdin: execOptions?.stdin, env: execOptions?.env }); - if (command[0] === 'jumbo' && command[1] === '--version') { - const isJumboArm = workDir.includes('jumbo-eval-jumbo-'); - const shimmed = execOptions?.env?.PATH?.includes('.eval-bin') ?? false; - // Jumbo arm: real binary unless test forces failure. - // Baseline arm: shim fails loudly unless test forces a leak. - const fail = isJumboArm - ? (options?.failJumboVersion === 'both' || options?.failJumboVersion === 'jumbo') - : (shimmed && !options?.baselineJumboLeak); - if (fail) { - return { - stdout: '', - stderr: shimmed - ? 'ERROR: jumbo is not available in the baseline arm (eval shim)' - : 'jumbo: command not found', - exitCode: 127, - }; - } - return { - stdout: 'jumbo 1.2.3', - stderr: '', - exitCode: 0, - }; - } - if (command[0] === 'jumbo' && command[1] === 'init') { - return { - stdout: options?.failJumboInit ? '' : 'jumbo initialized', - stderr: options?.failJumboInit ? 'init failed' : '', - exitCode: options?.failJumboInit ? 2 : 0, - }; - } - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'show') { - const count = (goalShowCallsByDir.get(workDir) ?? 0) + 1; - goalShowCallsByDir.set(workDir, count); - const before = count % 2 === 1; // first call per session is pre - const status = before - ? (options?.goalShowStatusBefore ?? 'refined') - : (options?.goalShowStatusAfter ?? 'submitted'); - const version = before - ? options?.goalVersionBefore - : options?.goalVersionAfter; - const id = command[command.indexOf('--id') + 1]; - const goalPayload: Record<string, unknown> = { goalId: id, status }; - if (version !== undefined) goalPayload.version = version; - return { - stdout: JSON.stringify({ goal: goalPayload }), - stderr: '', - exitCode: 0, - }; - } - if (command[0] === 'jumbo' && command[1] === 'sessions' && command[2] === 'list') { - const statusIdx = command.indexOf('--status'); - const statusFilter = statusIdx >= 0 ? command[statusIdx + 1] : 'all'; - const map = statusFilter === 'ended' ? sessionsEndedCallsByDir : sessionsAllCallsByDir; - const count = (map.get(workDir) ?? 0) + 1; - map.set(workDir, count); - const before = count % 2 === 1; - const total = statusFilter === 'ended' - ? (before ? options?.sessionsEndedBefore ?? 0 : options?.sessionsEndedAfter ?? 1) - : (before ? options?.sessionsTotalBefore ?? 0 : options?.sessionsTotalAfter ?? 1); - const items = Array.from({ length: total }, (_, i) => ({ sessionId: `s-${i + 1}` })); - return { stdout: JSON.stringify(items), stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[3] === '--format' && command[4] === 'json') { - const kind = command[1]; - const callKey = `${workDir}:${kind}`; - const count = (memoryListCallsByDirKind.get(callKey) ?? 0) + 1; - memoryListCallsByDirKind.set(callKey, count); - // Pre-snapshot is empty; post-snapshot and audit reads return the - // configured output. The scorer credits the diff (post - pre) only. - const output = count === 1 ? [] : (options?.jumboMemoryOutputs?.[kind] ?? []); - return { - stdout: JSON.stringify(output), - stderr: '', - exitCode: 0, - }; - } - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'add') { - const id = options?.goalAddIds?.[goalAddCount] ?? `mock-goal-${goalAddCount + 1}`; - goalAddCount++; - return { - stdout: JSON.stringify({ goalId: id }), - stderr: '', - exitCode: 0, - }; - } - if ( - command[0] === 'jumbo' && - command[2] === 'add' && - ['decision', 'component', 'invariant', 'dependency', 'relation'].includes(command[1]) - ) { - const fail = options?.failJumboPlanEntry === command[1]; - return { - stdout: fail ? '' : 'added', - stderr: fail ? `${command[1]} add failed` : '', - exitCode: fail ? 2 : 0, - }; - } - return { - stdout: JSON.stringify({ - result: 'Task completed', - files_modified: ['src/index.ts', 'src/utils.ts'], - }), - stderr: '', - exitCode: 0, - }; - }, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - } as LocalExecutor & { - execCalls: Array<{ workDir: string; command: string[]; stdin?: string; env?: Record<string, string | undefined> }>; - shimInstalledFor: string[]; - }; -} - -function createMockAdapter(overrides?: Partial<HarnessAdapter>): HarnessAdapter & { seededDirs: string[] } { - const seededDirs: string[] = []; - return { - seededDirs, - name: 'mock-harness', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => { - try { - const parsed = JSON.parse(result.stdout); - return { - agentOutput: parsed.result ?? result.stdout, - filesModified: parsed.files_modified ?? [], - transcript: result.stdout, - }; - } catch { - return { agentOutput: result.stdout, filesModified: [], transcript: result.stdout }; - } - }, - seedToolPermissions: async (workDir: string) => { seededDirs.push(workDir); }, - ...overrides, - } as HarnessAdapter & { seededDirs: string[] }; -} - -function createMockStore(): ResultStore { - const records: SessionRecord[] = []; - const results: TestResult[] = []; - return { - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async (r: SessionRecord) => { records.push(r); }, - getSessionRecords: async () => records, - saveTestResult: async (r: TestResult) => { results.push(r); }, - getTestResult: async () => null, - listTestResults: async () => results, - }; -} - -describe('runABComparison', () => { - it('creates two separate working directories', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test scenario', - initialPrompt: 'Build something', - sessionCount: 1, - expectedFiles: ['src/index.ts', 'src/utils.ts'], - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - // Jumbo calls go to dir 1, baseline calls go to dir 2 - const workDirs = new Set(executor.execCalls.map((c) => c.workDir)); - expect(workDirs.size).toBe(2); - }); - - it('includes a structural-retention dimension in scores, deltas, and timelines when the scenario declares structural assertions', async () => { - const scenario = createTestScenario({ - id: 'scenario-structural', - name: 'Structural retention', - initialPrompt: 'Build', - sessionCount: 1, - structuralAssertions: [ - { id: 'index-exists', file: 'src/index.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ], - }); - - const baseExec = createMockExecutor(); - const executor = { - ...baseExec, - captureWorkspaceSnapshot: async () => ({ - capturedAt: new Date().toISOString(), - files: [{ path: 'src/index.ts', content: 'export const x = 1;' }], - }), - } as typeof baseExec; - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - const dims = (scores: readonly { dimension: string }[]) => scores.map((s) => s.dimension); - expect(dims(result.jumboScores)).toContain('structural-retention'); - expect(dims(result.baselineScores)).toContain('structural-retention'); - expect(dims(result.deltas)).toContain('structural-retention'); - expect(result.jumboTimeline?.some((ps) => dims(ps.scores).includes('structural-retention'))).toBe(true); - expect(result.baselineTimeline?.some((ps) => dims(ps.scores).includes('structural-retention'))).toBe(true); - }); - - it('reports token-efficiency as N/A when the two arms are not output-equivalent', async () => { - // A structural assertion neither arm can satisfy (mock snapshots are empty) - // drives both structural scores below the 0.8 equivalence threshold. - const scenario = createTestScenario({ - id: 'scenario-not-equivalent', - name: 'Not equivalent', - initialPrompt: 'Build', - sessionCount: 1, - structuralAssertions: [ - { id: 'needs-file', file: 'src/needed.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ], - }); - const result = await runABComparison({ - scenario, - adapter: createMockAdapter(), - executor: createMockExecutor(), - store: createMockStore(), - }); - const te = result.jumboScores.find((s) => s.dimension === 'token-efficiency'); - expect(te?.maxScore).toBe(0); - expect(te?.details).toContain('N/A'); - expect(te?.details).toContain('not equivalent'); - }); - - it('reports token-efficiency (not N/A) when both arms are output-equivalent', async () => { - const scenario = createTestScenario({ - id: 'scenario-equivalent', - name: 'Equivalent', - initialPrompt: 'Build', - sessionCount: 1, - structuralAssertions: [ - { id: 'index-exists', file: 'src/index.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ], - }); - const baseExec = createMockExecutor(); - const executor = { - ...baseExec, - captureWorkspaceSnapshot: async () => ({ - capturedAt: new Date().toISOString(), - files: [{ path: 'src/index.ts', content: 'export const x = 1;' }], - }), - } as typeof baseExec; - const result = await runABComparison({ - scenario, - adapter: createMockAdapter(), - executor, - store: createMockStore(), - }); - const te = result.jumboScores.find((s) => s.dimension === 'token-efficiency'); - expect(te?.maxScore).toBe(1); - expect(te?.details ?? '').not.toContain('not equivalent'); - }); - - it('surfaces the Jumbo adherence rate in the protocol-adherence details', async () => { - const scenario = createTestScenario({ - id: 'scenario-adherence-rate', - name: 'Adherence rate', - initialPrompt: 'Build', - sessionCount: 1, - }); - const result = await runABComparison({ - scenario, - adapter: createMockAdapter(), - executor: createMockExecutor(), - store: createMockStore(), - }); - const pa = result.jumboScores.find((s) => s.dimension === 'protocol-adherence'); - expect(pa?.details).toContain('adherence-rate='); - expect(pa?.details).toContain('threshold=0.5'); - }); - - it('uses identical scenario prompts for both runs before variant wrapping', async () => { - // Without a jumboPlan, the Jumbo arm has no active goal-id, so its - // prompt is the bare scenario prompt — byte-identical to baseline. - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build a hello world', - sessionCount: 1, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const harnessExecs = executor.execCalls.filter((c) => c.command[0] === 'mock'); - expect(harnessExecs).toHaveLength(2); - // Prompt is delivered via stdin, not argv (cmd.exe newline-truncation fix). - expect(harnessExecs[0].stdin).toBe('Build a hello world'); - expect(harnessExecs[1].stdin).toBe('Build a hello world'); - }); - - it('wraps the Jumbo arm prompt with the lifecycle protocol when an active goal-id exists', async () => { - const scenario = createTestScenario({ - id: 'scenario-lifecycle', - name: 'Lifecycle prompt', - initialPrompt: 'Build a hello world', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'Foundations', - objective: 'Set up the project', - criteria: ['scaffold exists'], - sessionAvailableFrom: 1, - }], - }, - }); - const executor = createMockExecutor({ goalAddIds: ['gid-only'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const harnessExecs = executor.execCalls.filter((c) => c.command[0] === 'mock'); - expect(harnessExecs).toHaveLength(2); - - const jumboStdin = harnessExecs[0].stdin ?? ''; - expect(jumboStdin).toContain('Active goal for this session: gid-only'); - expect(jumboStdin).toContain('Jumbo lifecycle protocol'); - expect(jumboStdin).toContain('jumbo session start'); - expect(jumboStdin).toContain('jumbo goal start --id gid-only'); - expect(jumboStdin).toContain('jumbo goal submit --id gid-only'); - expect(jumboStdin).toContain('jumbo session end'); - expect(jumboStdin).toContain('Build a hello world'); - - // Baseline never sees the lifecycle protocol. - const baselineStdin = harnessExecs[1].stdin ?? ''; - expect(baselineStdin).toBe('Build a hello world'); - expect(baselineStdin).not.toContain('Jumbo lifecycle protocol'); - }); - - it('populates filesModified from workspace-diff fallback when the adapter returns none', async () => { - // Claude/Codex/Gemini CLIs do not surface a files_modified field reliably, - // so the adapter parse yields []. The pre/post workspace snapshot diff - // must be the source of truth for what the agent actually wrote on disk. - const scenario = createTestScenario({ - id: 'scenario-diff', - name: 'Diff test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - // Adapter that always returns empty filesModified, simulating Claude CLI - const emptyAdapter: HarnessAdapter = { - name: 'empty-adapter', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ - agentOutput: result.stdout, - filesModified: [], - transcript: result.stdout, - }), - seedToolPermissions: async () => {}, - }; - - // Executor whose snapshots differ before/after each harness exec. - // Pre-snapshot has only setup.md; post-snapshot adds two new files - // and modifies setup.md content — the diff is exactly those three paths. - const baseExec = createMockExecutor(); - let snapshotCall = 0; - const diffExecutor = { - ...baseExec, - captureWorkspaceSnapshot: async () => { - snapshotCall++; - // Odd calls are pre-exec (1 file), even calls are post-exec (3 files w/ change) - if (snapshotCall % 2 === 1) { - return { - capturedAt: '2026-05-01T00:00:00.000Z', - files: [{ path: 'setup.md', content: 'initial' }], - }; - } - return { - capturedAt: '2026-05-01T00:00:01.000Z', - files: [ - { path: 'setup.md', content: 'edited' }, - { path: 'src/index.ts', content: 'export {}' }, - { path: 'src/utils.ts', content: 'export const x = 1' }, - ], - }; - }, - } as LocalExecutor; - - const store = createMockStore(); - const result = await runABComparison({ - scenario, - adapter: emptyAdapter, - executor: diffExecutor, - store, - }); - - const jumboRecord = result.jumboResult.sessionRecords[0]; - const baselineRecord = result.baselineResult.sessionRecords[0]; - - expect(jumboRecord.filesModified).toEqual(['setup.md', 'src/index.ts', 'src/utils.ts']); - expect(baselineRecord.filesModified).toEqual(['setup.md', 'src/index.ts', 'src/utils.ts']); - }); - - it('preserves adapter-reported filesModified when present (does not overwrite with diff)', async () => { - const scenario = createTestScenario({ - id: 'scenario-adapter-priority', - name: 'Adapter priority test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const reportingAdapter: HarnessAdapter = { - name: 'reporting-adapter', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ - agentOutput: result.stdout, - filesModified: ['adapter/declared.ts'], - transcript: result.stdout, - }), - seedToolPermissions: async () => {}, - }; - - const executor = createMockExecutor(); - const store = createMockStore(); - const result = await runABComparison({ scenario, adapter: reportingAdapter, executor, store }); - - expect(result.jumboResult.sessionRecords[0].filesModified).toEqual(['adapter/declared.ts']); - }); - - it('records the effective prompt and lifecycle audit only for Jumbo runs with an active goal', async () => { - const scenario = createTestScenario({ - id: 'scenario-record', - name: 'Test', - initialPrompt: 'Build a hello world', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'Foundations', - objective: 'Set up the project', - criteria: ['scaffold exists'], - sessionAvailableFrom: 1, - }], - }, - }); - - const executor = createMockExecutor({ goalAddIds: ['gid-record'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - const jumboRecord = result.jumboResult.sessionRecords[0]; - const baselineRecord = result.baselineResult.sessionRecords[0]; - - expect(jumboRecord.variant).toBe('jumbo'); - expect(jumboRecord.scenarioPrompt).toBe('Build a hello world'); - expect(jumboRecord.deliveredContext).toBeUndefined(); - expect(jumboRecord.effectivePrompt).toContain('Active goal for this session: gid-record'); - expect(jumboRecord.effectivePrompt).toContain('Build a hello world'); - expect(jumboRecord.jumboLifecycleAudit).toBeDefined(); - expect(jumboRecord.jumboLifecycleAudit?.activeGoalId).toBe('gid-record'); - - expect(baselineRecord.variant).toBe('baseline'); - expect(baselineRecord.scenarioPrompt).toBe('Build a hello world'); - expect(baselineRecord.deliveredContext).toBeUndefined(); - expect(baselineRecord.effectivePrompt).toBe('Build a hello world'); - expect(baselineRecord.jumboLifecycleAudit).toBeUndefined(); - }); - - it('keeps adapter commands prompt-free so the byte-identical scenario prompt is delivered via stdin', () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build a hello world', - sessionCount: 1, - }); - const scenarioPrompt = getSessionPrompt(scenario, 1); - - // No adapter embeds the prompt in argv any more; run-session.ts pipes - // the prompt through LocalExecutor.exec's stdin option byte-for-byte. - for (const adapter of [new ClaudeCodeAdapter(), new CodexCliAdapter(), new GeminiCliAdapter()]) { - const command = adapter.buildCommand(); - expect(command).not.toContain(scenarioPrompt); - } - }); - - it('composes the Jumbo lifecycle prompt with goal-id, protocol, and scenario task', () => { - const scenarioPrompt = 'Build a hello world'; - const prompt = buildJumboLifecyclePrompt({ - scenarioPrompt, - activeGoalId: 'goal-abc-123', - }); - expect(prompt).toContain('Active goal for this session: goal-abc-123'); - expect(prompt).toContain('Jumbo lifecycle protocol'); - expect(prompt).toContain('jumbo session start'); - expect(prompt).toContain('jumbo goal start --id goal-abc-123'); - expect(prompt).toContain('jumbo goal submit --id goal-abc-123'); - expect(prompt).toContain('jumbo session end'); - expect(prompt).toContain('Scenario task for this session:'); - expect(prompt).toContain(scenarioPrompt); - expect(prompt.indexOf('goal-abc-123')).toBeLessThan(prompt.indexOf('Scenario task for this session:')); - }); - - it('appends operator-injected context to the lifecycle prompt when present', () => { - const prompt = buildJumboLifecyclePrompt({ - scenarioPrompt: 'do work', - activeGoalId: 'g', - injectedContext: 'remember edge case Q', - }); - expect(prompt).toContain('[OPERATOR-INJECTED CONTEXT]:'); - expect(prompt).toContain('remember edge case Q'); - }); - - it('produces a ComparisonResult with scores and deltas', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build something', - sessionCount: 1, - expectedFiles: ['src/index.ts', 'src/utils.ts'], - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - expect(result.scenarioId).toBe('scenario-1'); - expect(result.jumboResult.sessionRecords).toHaveLength(1); - expect(result.baselineResult.sessionRecords).toHaveLength(1); - expect(result.jumboScores).toHaveLength(8); - expect(result.baselineScores).toHaveLength(8); - expect(result.deltas).toHaveLength(8); - expect(result.jumboScores[0].dimension).toBe('file-accuracy'); - expect(result.jumboScores[1].dimension).toBe('knowledge-retention'); - expect(result.jumboScores[2].dimension).toBe('structural-retention'); - expect(result.jumboScores[3].dimension).toBe('disruption-recovery'); - expect(result.jumboScores[4].dimension).toBe('jumbo-memory-capture'); - expect(result.jumboScores[5].dimension).toBe('jumbo-event-capture'); - expect(result.jumboScores[6].dimension).toBe('protocol-adherence'); - expect(result.jumboScores[7].dimension).toBe('token-efficiency'); - }); - - it('captures Jumbo project memory after each session with JSON list commands', async () => { - const scenario = createTestScenario({ - id: 'scenario-memory', - name: 'Memory capture test', - initialPrompt: 'Adopt Commander for the CLI and remember the decision.', - sessionCount: 1, - expectedJumboMemoryCaptures: [ - { kind: 'decision', match: 'Commander for CLI' }, - { kind: 'component', match: 'TaskStore' }, - ], - }); - - const executor = createMockExecutor({ - jumboMemoryOutputs: { - decisions: [{ decisionId: 'dec-1', title: 'Commander for CLI', rationale: 'Matches local conventions.' }], - components: [{ componentId: 'cmp-1', name: 'TaskStore', description: 'Persists tasks.' }], - }, - }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - const memoryCommands = executor.execCalls.filter((call) => - call.command[0] === 'jumbo' && call.command[3] === '--format' && call.command[4] === 'json', - ); - // The agent-driven lifecycle now records a pre-harness memory snapshot - // (all six kinds) before the agent runs, a post-harness snapshot (all - // six kinds again) after, and the lifecycle audit's decisionsListAfter - // probe. Both snapshots must enumerate decisions/guidelines/invariants/ - // components/relations/dependencies in that order so the scorer can - // diff post against pre. - const memorySixKinds = [ - ['jumbo', 'decisions'], - ['jumbo', 'guidelines'], - ['jumbo', 'invariants'], - ['jumbo', 'components'], - ['jumbo', 'relations'], - ['jumbo', 'dependencies'], - ]; - const firstSix = memoryCommands.slice(0, 6).map((call) => call.command.slice(0, 2)); - expect(firstSix).toEqual(memorySixKinds); - // The post-snapshot (the six commands immediately after the harness - // exec) and the audit's trailing decisions list must also appear. - const postSnapshotStart = memoryCommands.findIndex((c, idx) => - idx >= 6 && c.command[1] === 'decisions', - ); - expect(postSnapshotStart).toBeGreaterThanOrEqual(6); - expect( - memoryCommands.slice(postSnapshotStart, postSnapshotStart + 6).map((c) => c.command.slice(0, 2)), - ).toEqual(memorySixKinds); - - // Snapshot diff: pre was empty, post has the configured entities. - const snapshotBefore = result.jumboResult.sessionRecords[0].jumboMemorySnapshotBefore; - const snapshotAfter = result.jumboResult.sessionRecords[0].jumboMemorySnapshot; - expect(snapshotBefore?.entities).toHaveLength(0); - expect(snapshotAfter?.entities).toHaveLength(2); - - expect(result.jumboScores.find((score) => score.dimension === 'jumbo-memory-capture')?.score).toBe(1); - expect(result.baselineScores.find((score) => score.dimension === 'jumbo-memory-capture')?.maxScore).toBe(0); - }); - - it('runs N sessions per working directory for multi-session scenarios', async () => { - const scenario = createTestScenario({ - id: 'scenario-multi', - name: 'Multi-session test', - initialPrompt: 'Build a project', - continuationPrompt: 'Continue the project', - sessionCount: 3, - expectedFiles: ['src/index.ts', 'src/utils.ts'], - retentionPatterns: ['index.ts'], - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - expect(result.jumboResult.sessionRecords).toHaveLength(3); - expect(result.baselineResult.sessionRecords).toHaveLength(3); - expect(result.jumboResult.sessionRecords.map((r) => r.sessionNumber)).toEqual([1, 2, 3]); - expect(result.baselineResult.sessionRecords.map((r) => r.sessionNumber)).toEqual([1, 2, 3]); - }); - - it('uses continuation prompt for sessions 2+', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Initial task', - continuationPrompt: 'Continue the work', - sessionCount: 2, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const harnessExecs = executor.execCalls.filter((c) => c.command[0] === 'mock'); - expect(harnessExecs).toHaveLength(4); - expect(harnessExecs[0].stdin).toContain('Initial task'); - expect(harnessExecs[1].stdin).toContain('Continue the work'); - expect(harnessExecs[2].stdin).toBe('Initial task'); - expect(harnessExecs[3].stdin).toBe('Continue the work'); - }); - - it('does not issue jumbo session start or end from the framework for jumbo runs', async () => { - const scenario = createTestScenario({ - id: 'scenario-no-framework-lifecycle', - name: 'Test', - initialPrompt: 'Build', - sessionCount: 2, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const jumboSessionStarts = executor.execCalls.filter((c) => - c.command[0] === 'jumbo' && c.command[1] === 'session' && c.command[2] === 'start', - ); - const jumboSessionEnds = executor.execCalls.filter((c) => - c.command[0] === 'jumbo' && c.command[1] === 'session' && c.command[2] === 'end', - ); - - expect(jumboSessionStarts).toHaveLength(0); - expect(jumboSessionEnds).toHaveLength(0); - }); - - it('runs the post-session lifecycle audit and records it on the SessionRecord', async () => { - const scenario = createTestScenario({ - id: 'scenario-audit', - name: 'Audit', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - - const executor = createMockExecutor({ - goalAddIds: ['gid-audit'], - goalShowStatusBefore: 'refined', - goalShowStatusAfter: 'submitted', - goalVersionBefore: 1, - goalVersionAfter: 4, // start (+1) + 1 progress (+1) + submit (+1) = +3 - sessionsTotalBefore: 0, - sessionsTotalAfter: 1, - sessionsEndedBefore: 0, - sessionsEndedAfter: 1, - jumboMemoryOutputs: { - decisions: [{ decisionId: 'd1', title: 'New decision the agent registered' }], - }, - }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - const audit = result.jumboResult.sessionRecords[0].jumboLifecycleAudit; - expect(audit).toBeDefined(); - expect(audit?.activeGoalId).toBe('gid-audit'); - expect(audit?.sessionStartExecuted).toBe(true); - expect(audit?.sessionEndExecuted).toBe(true); - expect(audit?.goalStartExecuted).toBe(true); - expect(audit?.goalSubmitExecuted).toBe(true); - // New diff-derived signals: - expect(audit?.inSessionCapturesExecuted).toBe(true); - expect(audit?.progressUpdatesExecuted).toBe(true); - expect(audit?.newEntityCount).toBe(1); - expect(audit?.goalVersionBefore).toBe(1); - expect(audit?.goalVersionAfter).toBe(4); - expect(audit?.goalStatusBefore).toBe('refined'); - expect(audit?.goalStatusAfter).toBe('submitted'); - expect(audit?.sessionsTotalDelta).toBe(1); - expect(audit?.sessionsEndedDelta).toBe(1); - expect(audit?.evidence.goalShowBefore?.command).toEqual([ - 'jumbo', 'goal', 'show', '--id', 'gid-audit', '--format', 'json', - ]); - expect(audit?.evidence.decisionsListAfter?.command).toEqual([ - 'jumbo', 'decisions', 'list', '--format', 'json', - ]); - }); - - it('scores protocol adherence across the six prescribed lifecycle steps', async () => { - const scenario = createTestScenario({ - id: 'scenario-protocol', - name: 'Protocol', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - - const executor = createMockExecutor({ - goalAddIds: ['gid-protocol'], - goalShowStatusBefore: 'refined', - goalShowStatusAfter: 'submitted', - goalVersionBefore: 1, - goalVersionAfter: 4, // implies progress update executed - sessionsTotalBefore: 0, - sessionsTotalAfter: 1, - sessionsEndedBefore: 0, - sessionsEndedAfter: 1, - jumboMemoryOutputs: { - decisions: [{ decisionId: 'd1', title: 'A decision the agent captured' }], - }, - }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - const protocolScore = result.jumboScores.find((s) => s.dimension === 'protocol-adherence'); - const baselineProtocolScore = result.baselineScores.find((s) => s.dimension === 'protocol-adherence'); - - expect(protocolScore?.score).toBe(1); - expect(protocolScore?.maxScore).toBe(1); - expect(protocolScore?.details).toContain('steps-passed=6/6'); - expect(baselineProtocolScore?.maxScore).toBe(0); - expect(baselineProtocolScore?.details).toContain('Not applicable'); - }); - - it('surfaces protocol non-adherence (missing steps) as first-class signal, not noise', async () => { - const scenario = createTestScenario({ - id: 'scenario-protocol-fail', - name: 'Protocol fail', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - - // Agent did not run goal submit and did not record any progress - // updates. Status stays at 'doing'; version bumps only from goal start. - const executor = createMockExecutor({ - goalAddIds: ['gid-partial'], - goalShowStatusBefore: 'refined', - goalShowStatusAfter: 'doing', - goalVersionBefore: 1, - goalVersionAfter: 2, - sessionsTotalBefore: 0, - sessionsTotalAfter: 1, - sessionsEndedBefore: 0, - sessionsEndedAfter: 0, - }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - const audit = result.jumboResult.sessionRecords[0].jumboLifecycleAudit; - expect(audit?.sessionStartExecuted).toBe(true); - expect(audit?.goalStartExecuted).toBe(true); - expect(audit?.goalSubmitExecuted).toBe(false); - expect(audit?.sessionEndExecuted).toBe(false); - expect(audit?.progressUpdatesExecuted).toBe(false); - expect(audit?.inSessionCapturesExecuted).toBe(false); - - const protocolScore = result.jumboScores.find((s) => s.dimension === 'protocol-adherence'); - expect(protocolScore?.score).toBeLessThan(1); - expect(protocolScore?.details).toContain('goal-submit-missed-in-sessions:1'); - expect(protocolScore?.details).toContain('session-end-missed-in-sessions:1'); - expect(protocolScore?.details).toContain('progress-updates-missed-in-sessions:1'); - }); - - it('marks lifecycle steps as not executed when goal status stays refined and no sessions are created', async () => { - const scenario = createTestScenario({ - id: 'scenario-not-executed', - name: 'Not executed', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - - const executor = createMockExecutor({ - goalAddIds: ['gid-skipped'], - goalShowStatusBefore: 'refined', - goalShowStatusAfter: 'refined', - sessionsTotalBefore: 0, - sessionsTotalAfter: 0, - sessionsEndedBefore: 0, - sessionsEndedAfter: 0, - }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - const audit = result.jumboResult.sessionRecords[0].jumboLifecycleAudit; - expect(audit?.sessionStartExecuted).toBe(false); - expect(audit?.sessionEndExecuted).toBe(false); - expect(audit?.goalStartExecuted).toBe(false); - expect(audit?.goalSubmitExecuted).toBe(false); - expect(audit?.inSessionCapturesExecuted).toBe(false); - expect(audit?.progressUpdatesExecuted).toBe(false); - }); - - it('runs jumbo init in the jumbo working directory only', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const jumboInits = executor.execCalls.filter((c) => - c.command[0] === 'jumbo' && c.command[1] === 'init', - ); - expect(jumboInits).toHaveLength(1); - expect(jumboInits[0].command).toEqual([ - 'jumbo', - 'init', - '--purpose', - scenario.name, - '--non-interactive', - '--name', - scenario.name, - '--yolo', - ]); - - // Baseline dir should not have jumbo init - const baselineDir = executor.execCalls - .filter((c) => c.command[0] === 'mock') - .map((c) => c.workDir) - .pop(); // Last mock call is baseline - const baselineJumboInits = executor.execCalls.filter((c) => - c.workDir === baselineDir && c.command[0] === 'jumbo' && c.command[1] === 'init', - ); - expect(baselineJumboInits).toHaveLength(0); - }); - - it('fails fast when jumbo init exits non-zero', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const executor = createMockExecutor({ failJumboInit: true }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await expect(runABComparison({ scenario, adapter, executor, store })).rejects.toThrow(JumboInitError); - - // Setup probes (jumbo --version × 2) run before jumbo init; jumbo init is - // the last call before the throw. No harness exec, no scoring. - const initCall = executor.execCalls.find((c) => c.command[0] === 'jumbo' && c.command[1] === 'init'); - expect(initCall).toBeDefined(); - expect(executor.execCalls.filter((c) => c.command[0] === 'mock')).toHaveLength(0); - expect(await store.listTestResults()).toHaveLength(0); - }); - - it('produces timeline with per-session scores for multi-session', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test', - initialPrompt: 'Build', - sessionCount: 3, - expectedFiles: ['src/index.ts'], - retentionPatterns: ['index.ts'], - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - expect(result.jumboTimeline).toHaveLength(3); - expect(result.baselineTimeline).toHaveLength(3); - expect(result.jumboTimeline![0].sessionNumber).toBe(1); - expect(result.jumboTimeline![2].sessionNumber).toBe(3); - }); - - it('injects disruptions at scheduled sessions', async () => { - const scenario = createTestScenario({ - id: 'scenario-disrupted', - name: 'Disruption test', - initialPrompt: 'Build a project', - continuationPrompt: 'Continue', - sessionCount: 3, - disruptions: [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case not camelCase', - recoveryPatterns: ['snake_case'], - }], - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const harnessExecs = executor.execCalls.filter((c) => c.command[0] === 'mock'); - expect(harnessExecs).toHaveLength(6); - - // Jumbo session 2 (index 1) and baseline session 2 (index 4) should contain disruption - expect(harnessExecs[1].stdin).toContain('[CORRECTION]'); - expect(harnessExecs[1].stdin).toContain('Use snake_case not camelCase'); - expect(harnessExecs[4].stdin).toContain('[CORRECTION]'); - - // Session 1 should NOT contain disruption - expect(harnessExecs[0].stdin).not.toContain('[CORRECTION]'); - }); - - it('seeds adapter tool permissions in both arm workdirs before any session work', async () => { - const scenario = createTestScenario({ - id: 'scenario-seed', - name: 'Seed test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - expect(adapter.seededDirs).toHaveLength(2); - expect(adapter.seededDirs).toEqual(expect.arrayContaining([ - expect.stringContaining('jumbo-eval-jumbo-'), - expect.stringContaining('jumbo-eval-baseline-'), - ])); - }); - - it('runs reachability probes in both arms before jumbo init: jumbo must reach jumbo, baseline must not', async () => { - const scenario = createTestScenario({ - id: 'scenario-probe', - name: 'Probe test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const versionCalls = executor.execCalls.filter( - (c) => c.command[0] === 'jumbo' && c.command[1] === '--version', - ); - const jumboProbe = versionCalls.find((c) => c.workDir.includes('jumbo-eval-jumbo-')); - const baselineProbe = versionCalls.find((c) => c.workDir.includes('jumbo-eval-baseline-')); - expect(jumboProbe).toBeDefined(); - expect(baselineProbe).toBeDefined(); - // The jumbo probe runs without a baseline-shim env; the baseline probe - // must run with the shim PATH so the real binary is shadowed. - expect(jumboProbe?.env?.PATH).toBeUndefined(); - expect(baselineProbe?.env?.PATH).toContain('.eval-bin'); - - // Both probes precede jumbo init - const initIndex = executor.execCalls.findIndex( - (c) => c.command[0] === 'jumbo' && c.command[1] === 'init', - ); - const lastProbeIndex = Math.max(...versionCalls.map((c) => - executor.execCalls.indexOf(c), - )); - expect(lastProbeIndex).toBeLessThan(initIndex); - }); - - it('installs the fail-loud jumbo shim in the baseline workdir before any session work', async () => { - const scenario = createTestScenario({ - id: 'scenario-shim', - name: 'Shim test', - initialPrompt: 'Build', - sessionCount: 1, - }); - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - expect(executor.shimInstalledFor).toHaveLength(1); - expect(executor.shimInstalledFor[0]).toContain('jumbo-eval-baseline-'); - }); - - it('threads the baseline shim env into every baseline subprocess (probe + harness)', async () => { - const scenario = createTestScenario({ - id: 'scenario-thread', - name: 'Thread test', - initialPrompt: 'Build', - sessionCount: 2, - }); - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const baselineCalls = executor.execCalls.filter((c) => - c.workDir.includes('jumbo-eval-baseline-'), - ); - expect(baselineCalls.length).toBeGreaterThan(0); - for (const c of baselineCalls) { - expect(c.env?.PATH).toContain('.eval-bin'); - } - // The jumbo arm must never carry the baseline shim env. - const jumboCalls = executor.execCalls.filter((c) => - c.workDir.includes('jumbo-eval-jumbo-'), - ); - for (const c of jumboCalls) { - expect(c.env?.PATH ?? '').not.toContain('.eval-bin'); - } - }); - - it('throws JumboBaselineLeakError when the shim fails to shadow the real jumbo in baseline', async () => { - const scenario = createTestScenario({ - id: 'scenario-leak', - name: 'Leak test', - initialPrompt: 'Build', - sessionCount: 1, - }); - const executor = createMockExecutor({ baselineJumboLeak: true }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await expect(runABComparison({ scenario, adapter, executor, store })).rejects.toThrow(JumboBaselineLeakError); - // No harness exec, no scoring after a leak failure. - expect(executor.execCalls.filter((c) => c.command[0] === 'mock')).toHaveLength(0); - expect(await store.listTestResults()).toHaveLength(0); - }); - - it('throws JumboReachabilityError when jumbo --version fails in jumbo arm and never runs harness', async () => { - const scenario = createTestScenario({ - id: 'scenario-unreachable', - name: 'Unreachable test', - initialPrompt: 'Build', - sessionCount: 1, - }); - - const executor = createMockExecutor({ failJumboVersion: 'jumbo' }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await expect(runABComparison({ scenario, adapter, executor, store })).rejects.toThrow(JumboReachabilityError); - - // Failure must surface before any session work - expect(executor.execCalls.filter((c) => c.command[0] === 'mock')).toHaveLength(0); - expect(executor.execCalls.find((c) => c.command[0] === 'jumbo' && c.command[1] === 'init')).toBeUndefined(); - expect(await store.listTestResults()).toHaveLength(0); - }); - - it('enforces parity: scenario prompt is byte-equal across arms and the Jumbo arm prompt is a strict superset', async () => { - const scenario = createTestScenario({ - id: 'scenario-parity', - name: 'Parity test', - initialPrompt: 'Build a hello world CLI', - sessionCount: 2, - continuationPrompt: 'Continue the work', - jumboPlan: { - goals: [ - { title: 'A', objective: 'a', criteria: ['c'], sessionAvailableFrom: 1 }, - { title: 'B', objective: 'b', criteria: ['c'], sessionAvailableFrom: 2 }, - ], - }, - }); - const executor = createMockExecutor({ goalAddIds: ['gid-1', 'gid-2'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - for (let i = 0; i < scenario.sessionCount; i++) { - const jumboRecord = result.jumboResult.sessionRecords[i]; - const baselineRecord = result.baselineResult.sessionRecords[i]; - - // (a) Scenario prompt is byte-identical between arms. - expect(jumboRecord.scenarioPrompt).toBe(baselineRecord.scenarioPrompt); - - // (b) The Jumbo collaboration block exists in exactly one arm. - const jumboPrompt = jumboRecord.effectivePrompt ?? ''; - const baselinePrompt = baselineRecord.effectivePrompt ?? ''; - expect(baselinePrompt).toBe(baselineRecord.scenarioPrompt); - expect(baselinePrompt).not.toContain('Jumbo lifecycle protocol'); - expect(jumboPrompt).toContain('Jumbo lifecycle protocol'); - - // Strict superset: the baseline prompt appears verbatim inside the - // Jumbo prompt, and the Jumbo prompt is strictly longer. - expect(jumboPrompt.includes(baselinePrompt)).toBe(true); - expect(jumboPrompt.length).toBeGreaterThan(baselinePrompt.length); - } - }); - - describe('jumboPlan', () => { - it('runs scenarios without a jumboPlan with no pre-seed or goal-add calls (default plan parity)', async () => { - const scenario = createTestScenario({ - id: 'scenario-no-plan', - name: 'No plan', - initialPrompt: 'Build', - sessionCount: 2, - }); - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const addCalls = executor.execCalls.filter( - (c) => c.command[0] === 'jumbo' && c.command[2] === 'add', - ); - expect(addCalls).toHaveLength(0); - }); - - it('seeds preSeededMemory in the jumbo arm after init and before the first harness exec', async () => { - const scenario = createTestScenario({ - id: 'scenario-seeded', - name: 'Seeded plan', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - preSeededMemory: [ - { - kind: 'invariant', - title: 'No mocks in integration tests', - description: 'Integration tests must hit a real database.', - rationale: 'Prior incident.', - }, - { - kind: 'component', - name: 'TaskStore', - type: 'storage', - description: 'Persists tasks to disk.', - responsibility: 'Durable task persistence.', - path: 'src/storage/task-store.ts', - }, - { - kind: 'decision', - title: 'Use Commander', - context: 'Need a CLI parser', - rationale: 'Matches conventions', - alternatives: ['yargs'], - }, - { - kind: 'dependency', - name: 'Express', - ecosystem: 'npm', - packageName: 'express', - versionConstraint: '^4.18.0', - }, - { - kind: 'relation', - fromType: 'component', - fromId: 'TaskStore', - toType: 'dependency', - toId: 'express', - type: 'uses', - description: 'TaskStore depends on Express', - strength: 'medium', - }, - ], - goals: [ - { - title: 'Build CLI', - objective: 'Ship a CLI', - criteria: ['ships'], - sessionAvailableFrom: 1, - }, - ], - }, - }); - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const jumboArm = executor.execCalls.filter((c) => c.workDir.includes('jumbo-eval-jumbo-')); - const initIndex = jumboArm.findIndex( - (c) => c.command[0] === 'jumbo' && c.command[1] === 'init', - ); - const firstHarnessIndex = jumboArm.findIndex((c) => c.command[0] === 'mock'); - const seedKinds: string[] = []; - for (let i = initIndex + 1; i < firstHarnessIndex; i++) { - const call = jumboArm[i]; - if (call.command[0] === 'jumbo' && call.command[2] === 'add' && - ['decision', 'component', 'invariant', 'dependency', 'relation'].includes(call.command[1])) { - seedKinds.push(call.command[1]); - } - } - expect(seedKinds).toEqual(['invariant', 'component', 'decision', 'dependency', 'relation']); - }); - - it('does not seed preSeededMemory in the baseline arm', async () => { - const scenario = createTestScenario({ - id: 'scenario-baseline-clean', - name: 'Baseline clean', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - preSeededMemory: [{ - kind: 'invariant', - title: 'X', - description: 'Y', - }], - goals: [{ - title: 'Foundation', - objective: 'Build it', - criteria: ['ships'], - sessionAvailableFrom: 1, - }], - }, - }); - const executor = createMockExecutor(); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const baselineAdds = executor.execCalls.filter( - (c) => c.workDir.includes('jumbo-eval-baseline-') && - c.command[0] === 'jumbo' && c.command[2] === 'add', - ); - expect(baselineAdds).toHaveLength(0); - }); - - it('fails fast when preSeededMemory registration exits non-zero', async () => { - const scenario = createTestScenario({ - id: 'scenario-seed-fail', - name: 'Seed fail', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - preSeededMemory: [{ - kind: 'invariant', - title: 'X', - description: 'Y', - }], - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - const executor = createMockExecutor({ failJumboPlanEntry: 'invariant' }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await expect(runABComparison({ scenario, adapter, executor, store })).rejects.toThrow(JumboPlanSeedError); - // No harness exec runs after a seed failure. - expect(executor.execCalls.filter((c) => c.command[0] === 'mock')).toHaveLength(0); - }); - - it('registers plan goals only at their sessionAvailableFrom boundary (progressive release)', async () => { - const scenario = createTestScenario({ - id: 'scenario-progressive', - name: 'Progressive release', - initialPrompt: 'Build', - continuationPrompt: 'Continue', - sessionCount: 3, - jumboPlan: { - goals: [ - { - planRef: 'g1', - title: 'Foundations', - objective: 'Set up the project', - criteria: ['scaffold exists'], - sessionAvailableFrom: 1, - }, - { - planRef: 'g2', - title: 'Add command', - objective: 'Implement add', - criteria: ['add works'], - sessionAvailableFrom: 2, - prerequisitePlanRefs: ['g1'], - }, - { - planRef: 'g3', - title: 'Status transitions', - objective: 'Implement transitions', - criteria: ['transitions work'], - sessionAvailableFrom: 3, - prerequisitePlanRefs: ['g2'], - }, - ], - }, - }); - const executor = createMockExecutor({ goalAddIds: ['real-g1', 'real-g2', 'real-g3'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - await runABComparison({ scenario, adapter, executor, store }); - - const jumboArm = executor.execCalls.filter((c) => c.workDir.includes('jumbo-eval-jumbo-')); - const goalAdds = jumboArm.filter((c) => - c.command[0] === 'jumbo' && c.command[1] === 'goal' && c.command[2] === 'add', - ); - expect(goalAdds).toHaveLength(3); - - // Each goal-add must precede its session's harness exec and follow - // the previous session's harness exec (or init for #1). - const harnessExecs = jumboArm - .map((c, idx) => ({ c, idx })) - .filter(({ c }) => c.command[0] === 'mock'); - expect(harnessExecs).toHaveLength(3); - for (let i = 0; i < 3; i++) { - const addIdx = jumboArm.indexOf(goalAdds[i]); - expect(addIdx).toBeLessThan(harnessExecs[i].idx); - if (i > 0) { - expect(addIdx).toBeGreaterThan(harnessExecs[i - 1].idx); - } - } - - // Prerequisite plan refs resolve to the actual goal ids returned by the CLI. - const g2Add = goalAdds[1]; - expect(g2Add.command).toContain('--prerequisite-goals'); - expect(g2Add.command[g2Add.command.indexOf('--prerequisite-goals') + 1]).toBe('real-g1'); - const g3Add = goalAdds[2]; - expect(g3Add.command[g3Add.command.indexOf('--prerequisite-goals') + 1]).toBe('real-g2'); - }); - - it('threads the active goal-id into the jumbo prompt for each session but never the baseline', async () => { - const scenario = createTestScenario({ - id: 'scenario-active-goal', - name: 'Active goal', - initialPrompt: 'Build', - continuationPrompt: 'Continue', - sessionCount: 2, - jumboPlan: { - goals: [ - { - title: 'S1', - objective: 'foundation', - criteria: ['done'], - sessionAvailableFrom: 1, - }, - { - title: 'S2', - objective: 'next', - criteria: ['done'], - sessionAvailableFrom: 2, - }, - ], - }, - }); - const executor = createMockExecutor({ goalAddIds: ['gid-1', 'gid-2'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - expect(result.jumboResult.sessionRecords[0].effectivePrompt).toContain('gid-1'); - expect(result.jumboResult.sessionRecords[1].effectivePrompt).toContain('gid-2'); - expect(result.baselineResult.sessionRecords[0].effectivePrompt).not.toContain('gid-1'); - expect(result.baselineResult.sessionRecords[1].effectivePrompt).not.toContain('gid-2'); - }); - - it('carries the previously active goal forward when no new goal is registered for a session', async () => { - const scenario = createTestScenario({ - id: 'scenario-carry', - name: 'Carry forward', - initialPrompt: 'Build', - continuationPrompt: 'Continue', - sessionCount: 3, - jumboPlan: { - goals: [ - { - title: 'Only goal', - objective: 'one', - criteria: ['done'], - sessionAvailableFrom: 1, - }, - ], - }, - }); - const executor = createMockExecutor({ goalAddIds: ['only-goal-id'] }); - const adapter = createMockAdapter(); - const store = createMockStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - for (const record of result.jumboResult.sessionRecords) { - expect(record.effectivePrompt).toContain('only-goal-id'); - } - }); - - it('throws JumboPlanGoalRegistrationError when goal add returns no parseable id', async () => { - const scenario = createTestScenario({ - id: 'scenario-goal-noid', - name: 'Goal no id', - initialPrompt: 'Build', - sessionCount: 1, - jumboPlan: { - goals: [{ - title: 'G', - objective: 'O', - criteria: ['c'], - sessionAvailableFrom: 1, - }], - }, - }); - const baseExec = createMockExecutor(); - const noIdExecutor = { - ...baseExec, - exec: async (workDir: string, command: string[], execOptions?: { stdin?: string }) => { - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'add') { - baseExec.execCalls.push({ workDir, command, stdin: execOptions?.stdin }); - return { stdout: 'no id here', stderr: '', exitCode: 0 }; - } - return baseExec.exec(workDir, command, execOptions); - }, - } as LocalExecutor; - const adapter = createMockAdapter(); - const store = createMockStore(); - - await expect(runABComparison({ scenario, adapter, executor: noIdExecutor, store })) - .rejects.toThrow(JumboPlanGoalRegistrationError); - }); - }); - - describe('createTestScenario jumboPlan validation', () => { - it('rejects sessionAvailableFrom outside 1..sessionCount', () => { - expect(() => createTestScenario({ - id: 's', - name: 's', - initialPrompt: 'p', - sessionCount: 2, - jumboPlan: { - goals: [{ title: 't', objective: 'o', criteria: ['c'], sessionAvailableFrom: 3 }], - }, - })).toThrow('sessionAvailableFrom'); - - expect(() => createTestScenario({ - id: 's', - name: 's', - initialPrompt: 'p', - sessionCount: 2, - jumboPlan: { - goals: [{ title: 't', objective: 'o', criteria: ['c'], sessionAvailableFrom: 0 }], - }, - })).toThrow('sessionAvailableFrom'); - }); - - it('rejects an empty goals array', () => { - expect(() => createTestScenario({ - id: 's', - name: 's', - initialPrompt: 'p', - sessionCount: 1, - jumboPlan: { goals: [] }, - })).toThrow('at least one goal'); - }); - }); - - it('rejects if jumbo result is missing (invariant: both scores required)', () => { - const { createComparisonResult } = require('../../src/domain/types'); - expect(() => - createComparisonResult({ - id: 'id', - scenarioId: 's1', - harness: 'h', - jumboResult: null, - baselineResult: { id: 'r', scenarioId: 's1', harness: 'h', sessionRecords: [], createdAt: '' }, - jumboScores: [], - baselineScores: [], - deltas: [], - }), - ).toThrow('ComparisonResult requires a jumboResult'); - }); -}); diff --git a/evals/tests/unit/claude-code-adapter.test.ts b/evals/tests/unit/claude-code-adapter.test.ts deleted file mode 100644 index 87db02a3..00000000 --- a/evals/tests/unit/claude-code-adapter.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { ClaudeCodeAdapter } from '../../src/harness/claude-code-adapter.js'; - -describe('ClaudeCodeAdapter', () => { - const adapter = new ClaudeCodeAdapter(); - - it('has the correct name', () => { - expect(adapter.name).toBe('claude-code'); - }); - - it('builds a claude -p command without a positional prompt (prompt goes via stdin)', () => { - const command = adapter.buildCommand(); - expect(command).toEqual(['claude', '-p', '--output-format', 'json', '--dangerously-skip-permissions']); - }); - - it('parses JSON output with result field', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - result: 'Created index.ts with hello world', - files_modified: ['index.ts', 'package.json'], - }), - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Created index.ts with hello world'); - expect(result.filesModified).toEqual(['index.ts', 'package.json']); - expect(result.transcript).toContain('Created index.ts'); - }); - - it('handles non-JSON output gracefully', () => { - const result = adapter.parseOutput({ - stdout: 'Plain text response from claude', - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Plain text response from claude'); - expect(result.filesModified).toEqual([]); - }); - - it('extracts token counts from top-level fields', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - result: 'Done', - input_tokens: 1500, - output_tokens: 800, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(1500); - expect(result.outputTokens).toBe(800); - }); - - it('extracts token counts from nested usage object', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - result: 'Done', - usage: { input_tokens: 2000, output_tokens: 1000 }, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(2000); - expect(result.outputTokens).toBe(1000); - }); - - it('returns undefined tokens for non-JSON output', () => { - const result = adapter.parseOutput({ - stdout: 'plain text', - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBeUndefined(); - expect(result.outputTokens).toBeUndefined(); - }); - - it('includes stderr in transcript', () => { - const result = adapter.parseOutput({ - stdout: 'output', - stderr: 'some warning', - exitCode: 0, - }); - - expect(result.transcript).toContain('output'); - expect(result.transcript).toContain('some warning'); - expect(result.transcript).toContain('---stderr---'); - }); - - describe('seedToolPermissions', () => { - let workDir: string; - - beforeEach(async () => { - workDir = await mkdtemp(join(tmpdir(), 'claude-adapter-seed-')); - }); - - afterEach(async () => { - await rm(workDir, { recursive: true, force: true }); - }); - - it('writes .claude/settings.json with a Bash(jumbo:*) allowlist', async () => { - await adapter.seedToolPermissions(workDir); - const raw = await readFile(join(workDir, '.claude', 'settings.json'), 'utf-8'); - const settings = JSON.parse(raw); - - expect(settings.permissions.allow).toEqual(expect.arrayContaining([ - 'Bash(jumbo)', - 'Bash(jumbo:*)', - ])); - }); - - it('is idempotent: re-seeding overwrites without error', async () => { - await adapter.seedToolPermissions(workDir); - await adapter.seedToolPermissions(workDir); - const raw = await readFile(join(workDir, '.claude', 'settings.json'), 'utf-8'); - expect(JSON.parse(raw).permissions.allow).toContain('Bash(jumbo:*)'); - }); - }); -}); diff --git a/evals/tests/unit/cli-commands.test.ts b/evals/tests/unit/cli-commands.test.ts deleted file mode 100644 index d8cfa2d6..00000000 --- a/evals/tests/unit/cli-commands.test.ts +++ /dev/null @@ -1,1079 +0,0 @@ -import { describe, it, expect, jest, beforeEach, afterEach } from '@jest/globals'; -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; -import * as os from 'node:os'; -import { createProgram } from '../../src/cli/index.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; -import type { TestScenario, SessionRecord, TestResult, ComparisonResult, ReplicationReport } from '../../src/domain/types.js'; -import { handleScenarioCreate } from '../../src/cli/commands/scenario-create.js'; -import { formatScenarioList } from '../../src/cli/commands/scenario-list.js'; -import { validateHarnesses } from '../../src/cli/commands/run.js'; -import { formatScoreOutput } from '../../src/cli/commands/score.js'; -import { filterReportByDimensions, filterComparisonsByHarness } from '../../src/cli/commands/report.js'; -import { formatStatusOutput } from '../../src/cli/commands/status.js'; -import { createTestScenario, createTestResult, createSessionRecord, createComparisonResult } from '../../src/domain/types.js'; -import type { FullReport } from '../../src/output/report-generator.js'; - -describe('createProgram', () => { - it('creates a commander program with eval name', () => { - const program = createProgram(); - expect(program.name()).toBe('eval'); - }); - - it('registers scenario subcommand', () => { - const program = createProgram(); - const scenario = program.commands.find((c) => c.name() === 'scenario'); - expect(scenario).toBeDefined(); - }); - - it('registers top-level commands', () => { - const program = createProgram(); - const names = program.commands.map((c) => c.name()); - expect(names).toContain('scenario'); - expect(names).toContain('run'); - expect(names).toContain('score'); - expect(names).toContain('report'); - expect(names).toContain('status'); - }); - - it('scenario has create and list subcommands', () => { - const program = createProgram(); - const scenario = program.commands.find((c) => c.name() === 'scenario')!; - const subNames = scenario.commands.map((c) => c.name()); - expect(subNames).toContain('create'); - expect(subNames).toContain('list'); - }); -}); - -describe('handleScenarioCreate', () => { - it('creates a scenario from template', () => { - const scenario = handleScenarioCreate({ - name: 'Test Scenario', - initialPrompt: 'Build a REST API', - sessionCount: 3, - }); - - expect(scenario.name).toBe('Test Scenario'); - expect(scenario.initialPrompt).toBe('Build a REST API'); - expect(scenario.sessionCount).toBe(3); - expect(scenario.id).toBeTruthy(); - }); - - it('preserves structuralAssertions from the template (regression: first smoke run silently dropped them)', () => { - const scenario = handleScenarioCreate({ - name: 'With assertions', - initialPrompt: 'Build', - sessionCount: 2, - structuralAssertions: [ - { id: 'a1', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 'a2', file: 'src/b.ts', sessionNumber: 2, matcher: { kind: 'containsAll', substrings: ['x'] } }, - ], - }); - - expect(scenario.structuralAssertions).toHaveLength(2); - expect(scenario.structuralAssertions?.[0].id).toBe('a1'); - }); - - it('applies name override', () => { - const scenario = handleScenarioCreate( - { name: 'Original', initialPrompt: 'prompt', sessionCount: 2 }, - { name: 'Overridden' }, - ); - - expect(scenario.name).toBe('Overridden'); - }); - - it('applies session count override', () => { - const scenario = handleScenarioCreate( - { name: 'Test', initialPrompt: 'prompt', sessionCount: 2 }, - { sessions: 5 }, - ); - - expect(scenario.sessionCount).toBe(5); - }); -}); - -class InMemoryStore implements ResultStore { - scenarios: TestScenario[] = []; - results: TestResult[] = []; - async saveScenario(s: TestScenario): Promise<void> { this.scenarios.push(s); } - async getScenario(id: string): Promise<TestScenario | null> { - return this.scenarios.find((s) => s.id === id) ?? null; - } - async listScenarios(): Promise<TestScenario[]> { return [...this.scenarios]; } - async saveSessionRecord(_r: SessionRecord): Promise<void> {} - async getSessionRecords(_id: string): Promise<SessionRecord[]> { return []; } - async saveTestResult(r: TestResult): Promise<void> { - const existing = this.results.findIndex((candidate) => candidate.id === r.id); - if (existing >= 0) { - this.results[existing] = r; - } else { - this.results.push(r); - } - } - async getTestResult(id: string): Promise<TestResult | null> { - return this.results.find((r) => r.id === id) ?? null; - } - async listTestResults(scenarioId?: string): Promise<TestResult[]> { - return scenarioId ? this.results.filter((r) => r.scenarioId === scenarioId) : [...this.results]; - } - replicationReports = new Map<string, ReplicationReport>(); - async saveReplicationReport(runId: string, report: ReplicationReport): Promise<void> { - this.replicationReports.set(runId, report); - } - async getReplicationReport(runId: string): Promise<ReplicationReport | null> { - return this.replicationReports.get(runId) ?? null; - } -} - -describe('scenario create command', () => { - let store: InMemoryStore; - let tmpDir: string; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(async () => { - store = new InMemoryStore(); - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'eval-cli-test-')); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(async () => { - logSpy.mockRestore(); - await fs.rm(tmpDir, { recursive: true, force: true }); - }); - - it('persists scenario from template and prints UUID', async () => { - const tplPath = path.join(tmpDir, 't.json'); - await fs.writeFile(tplPath, JSON.stringify({ - name: 'X', initialPrompt: 'Build', sessionCount: 4, - })); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'scenario', 'create', '--from-template', tplPath]); - - expect(store.scenarios).toHaveLength(1); - expect(store.scenarios[0].name).toBe('X'); - expect(store.scenarios[0].sessionCount).toBe(4); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain(store.scenarios[0].id); - expect(printed.trim()).toBe(store.scenarios[0].id); - }); - - it('applies --name and --sessions overrides', async () => { - const tplPath = path.join(tmpDir, 't.json'); - await fs.writeFile(tplPath, JSON.stringify({ - name: 'Original', initialPrompt: 'p', sessionCount: 2, - })); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync([ - 'node', 'eval', 'scenario', 'create', - '--from-template', tplPath, - '--name', 'Renamed', - '--sessions', '7', - ]); - - expect(store.scenarios[0].name).toBe('Renamed'); - expect(store.scenarios[0].sessionCount).toBe(7); - }); - - it('rejects unreadable template path with a clear error', async () => { - const program = createProgram({ storeProvider: async () => store }); - await expect(program.parseAsync([ - 'node', 'eval', 'scenario', 'create', - '--from-template', path.join(tmpDir, 'does-not-exist.json'), - ])).rejects.toThrow(/Cannot read template file/); - expect(store.scenarios).toHaveLength(0); - }); - - it('rejects malformed JSON with a clear error', async () => { - const tplPath = path.join(tmpDir, 'bad.json'); - await fs.writeFile(tplPath, '{ not json'); - - const program = createProgram({ storeProvider: async () => store }); - await expect(program.parseAsync([ - 'node', 'eval', 'scenario', 'create', '--from-template', tplPath, - ])).rejects.toThrow(/not valid JSON/); - }); - - it('rejects template missing required fields', async () => { - const tplPath = path.join(tmpDir, 'short.json'); - await fs.writeFile(tplPath, JSON.stringify({ name: 'X' })); - - const program = createProgram({ storeProvider: async () => store }); - await expect(program.parseAsync([ - 'node', 'eval', 'scenario', 'create', '--from-template', tplPath, - ])).rejects.toThrow(/initialPrompt|sessionCount/); - }); -}); - -describe('scenario list command', () => { - let store: InMemoryStore; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(() => { - store = new InMemoryStore(); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(() => { - logSpy.mockRestore(); - }); - - it('prints empty-state message when no scenarios exist', async () => { - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'scenario', 'list']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('No scenarios found'); - }); - - it('prints registered scenarios in human format', async () => { - store.scenarios.push({ - id: 'sid-1', name: 'My Scenario', initialPrompt: 'p', - sessionCount: 3, createdAt: '2026-01-01T00:00:00Z', - } as TestScenario); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'scenario', 'list']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('1 scenario(s)'); - expect(printed).toContain('sid-1'); - expect(printed).toContain('My Scenario'); - }); - - it('--json emits parseable JSON of scenario array', async () => { - store.scenarios.push({ - id: 'sid-2', name: 'JSON Scenario', initialPrompt: 'p', - sessionCount: 2, createdAt: '2026-01-01T00:00:00Z', - } as TestScenario); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'scenario', 'list', '--json']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - const parsed = JSON.parse(printed); - expect(Array.isArray(parsed)).toBe(true); - expect(parsed[0].id).toBe('sid-2'); - }); -}); - -describe('formatScenarioList', () => { - it('shows message for empty list', () => { - const output = formatScenarioList([]); - expect(output).toContain('No scenarios found'); - }); - - it('formats scenarios with details', () => { - const scenario = createTestScenario({ - id: 'test-id', - name: 'My Scenario', - initialPrompt: 'prompt', - sessionCount: 3, - }); - - const output = formatScenarioList([scenario]); - expect(output).toContain('1 scenario(s)'); - expect(output).toContain('My Scenario'); - expect(output).toContain('3'); - }); -}); - -describe('validateHarnesses', () => { - it('accepts valid harness names', () => { - expect(validateHarnesses(['claude-code', 'codex-cli'])).toEqual(['claude-code', 'codex-cli']); - }); - - it('throws on invalid harness name', () => { - expect(() => validateHarnesses(['claude-code', 'invalid-harness'])).toThrow('Unknown harness'); - }); - - it('accepts all three harnesses', () => { - expect(validateHarnesses(['claude-code', 'codex-cli', 'gemini-cli'])).toHaveLength(3); - }); -}); - -describe('formatScoreOutput', () => { - it('formats scores for display', () => { - const record = createSessionRecord({ - id: 'rec-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: 'output', - filesModified: [], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - const result = createTestResult({ - id: 'result-1', - scenarioId: 'scenario-1', - harness: 'claude-code', - sessionRecords: [record], - }); - - const scores = [ - { dimension: 'file-accuracy', score: 0.9, maxScore: 1, details: '9/10 files' }, - ]; - - const output = formatScoreOutput(result, scores); - - expect(output).toContain('result-1'); - expect(output).toContain('file-accuracy'); - expect(output).toContain('0.90/1.00'); - expect(output).toContain('9/10 files'); - }); -}); - -describe('score command', () => { - let store: InMemoryStore; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(() => { - store = new InMemoryStore(); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(() => { - logSpy.mockRestore(); - }); - - function seedScenario(): TestScenario { - const scenario = createTestScenario({ - id: 'scenario-score', - name: 'Score Scenario', - initialPrompt: 'Build a typed config loader', - sessionCount: 2, - expectedFiles: ['src/config.ts'], - retentionPatterns: ['typed config'], - disruptions: [{ - type: 'correction', - sessionNumber: 1, - content: 'Use a typed config parser.', - recoveryPatterns: ['typed config'], - }], - expectedJumboMemoryCaptures: [{ - kind: 'decision', - match: 'typed config', - sessionNumber: 1, - }], - }); - store.scenarios.push(scenario); - return scenario; - } - - function seedResult(params: { - id: string; - scenarioId: string; - variant: 'jumbo' | 'baseline'; - output: string; - }): TestResult { - const records = [1, 2].map((sessionNumber) => createSessionRecord({ - id: `${params.id}-session-${sessionNumber}`, - scenarioId: params.scenarioId, - sessionNumber, - harness: 'claude-code', - variant: params.variant, - agentOutput: params.output, - filesModified: ['src/config.ts'], - transcript: params.output, - workspaceSnapshot: { - capturedAt: '2026-03-21T10:05:00Z', - files: [{ - path: 'src/config.ts', - content: `export const note = "typed config ${params.output}";`, - }], - }, - jumboMemorySnapshot: params.variant === 'jumbo' - ? { - sessionNumber, - capturedAt: '2026-03-21T10:05:00Z', - entities: [{ - kind: 'decision', - id: `decision-${sessionNumber}`, - text: 'Chose typed config parser', - raw: { title: 'Chose typed config parser' }, - }], - commands: [], - } - : undefined, - inputTokens: 100, - outputTokens: 50, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - })); - - const result = createTestResult({ - id: params.id, - scenarioId: params.scenarioId, - harness: 'claude-code', - sessionRecords: records, - }); - store.results.push(result); - return result; - } - - it('scores a TestResult and prints all five deterministic dimensions', async () => { - const scenario = seedScenario(); - seedResult({ - id: 'jumbo-result', - scenarioId: scenario.id, - variant: 'jumbo', - output: 'typed config complete', - }); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'score', '--scenario', scenario.id]); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - expect(printed).toContain('jumbo-result'); - expect(printed).toContain('file-accuracy'); - expect(printed).toContain('knowledge-retention'); - expect(printed).toContain('disruption-recovery'); - expect(printed).toContain('token-efficiency'); - expect(printed).toContain('jumbo-memory-capture'); - }); - - it('--result narrows scoring to a single result', async () => { - const scenario = seedScenario(); - seedResult({ - id: 'jumbo-result', - scenarioId: scenario.id, - variant: 'jumbo', - output: 'typed config complete', - }); - seedResult({ - id: 'baseline-result', - scenarioId: scenario.id, - variant: 'baseline', - output: 'typed config complete', - }); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync([ - 'node', 'eval', 'score', - '--scenario', scenario.id, - '--result', 'baseline-result', - ]); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - expect(printed).toContain('baseline-result'); - expect(printed).not.toContain('jumbo-result'); - expect(printed).toContain('jumbo-memory-capture'); - expect(printed).toContain('0.00/0.00 (N/A)'); - }); - - it('errors clearly when the scenario is missing', async () => { - const program = createProgram({ storeProvider: async () => store }); - - await expect(program.parseAsync([ - 'node', 'eval', 'score', '--scenario', 'missing-scenario', - ])).rejects.toThrow(/Scenario not found: missing-scenario/); - }); -}); - -describe('filterReportByDimensions', () => { - const mockReport: FullReport = { - scenarioId: 'scenario-1', - harnesses: ['claude-code'], - divergenceCurve: [ - { sessionNumber: 1, dimension: 'file-accuracy', jumboScore: 0.9, baselineScore: 0.7, delta: 0.2 }, - { sessionNumber: 1, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.5, delta: 0.3 }, - ], - liftResults: [ - { dimension: 'file-accuracy', jumboScore: 0.9, baselineScore: 0.7, absoluteLift: 0.2, percentageLift: 28.6 }, - { dimension: 'retention', jumboScore: 0.8, baselineScore: 0.5, absoluteLift: 0.3, percentageLift: 60 }, - ], - divergenceOnsets: [ - { dimension: 'file-accuracy', onsetSession: 2, threshold: 0.1, deltaAtOnset: 0.15 }, - { dimension: 'retention', onsetSession: 1, threshold: 0.1, deltaAtOnset: 0.3 }, - ], - disruptionImpacts: [], - memoryCaptureEvidence: [], - harnessAggregation: [], - auditTrails: [], - tamperedComparisons: [], - generatedAt: '2026-03-21T10:00:00Z', - }; - - it('returns full report when no filter', () => { - const filtered = filterReportByDimensions(mockReport, []); - expect(filtered.liftResults).toHaveLength(2); - }); - - it('filters by dimension', () => { - const filtered = filterReportByDimensions(mockReport, ['file-accuracy']); - expect(filtered.liftResults).toHaveLength(1); - expect(filtered.liftResults[0].dimension).toBe('file-accuracy'); - expect(filtered.divergenceCurve).toHaveLength(1); - expect(filtered.divergenceOnsets).toHaveLength(1); - }); -}); - -describe('filterComparisonsByHarness', () => { - it('returns all when no filter', () => { - const record = createSessionRecord({ - id: 'r', scenarioId: 's', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - const result = createTestResult({ id: 'tr', scenarioId: 's', harness: 'claude-code', sessionRecords: [record] }); - const comp = createComparisonResult({ - id: 'c', scenarioId: 's', harness: 'claude-code', - jumboResult: result, baselineResult: result, - jumboScores: [], baselineScores: [], deltas: [], - }); - - expect(filterComparisonsByHarness([comp], [])).toHaveLength(1); - }); - - it('filters by harness', () => { - const record = createSessionRecord({ - id: 'r', scenarioId: 's', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - const result = createTestResult({ id: 'tr', scenarioId: 's', harness: 'claude-code', sessionRecords: [record] }); - const comp1 = createComparisonResult({ - id: 'c1', scenarioId: 's', harness: 'claude-code', - jumboResult: result, baselineResult: result, - jumboScores: [], baselineScores: [], deltas: [], - }); - const comp2 = createComparisonResult({ - id: 'c2', scenarioId: 's', harness: 'codex-cli', - jumboResult: result, baselineResult: result, - jumboScores: [], baselineScores: [], deltas: [], - }); - - const filtered = filterComparisonsByHarness([comp1, comp2], ['claude-code']); - expect(filtered).toHaveLength(1); - expect(filtered[0].harness).toBe('claude-code'); - }); -}); - -describe('report command', () => { - let store: InMemoryStore; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(() => { - store = new InMemoryStore(); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(() => { - logSpy.mockRestore(); - }); - - function seedScenario(id = 'scenario-report'): TestScenario { - const scenario = createTestScenario({ - id, - name: 'Report Scenario', - initialPrompt: 'Build a reportable feature', - sessionCount: 2, - }); - store.scenarios.push(scenario); - return scenario; - } - - function makeComparison(scenarioId: string, harness: string, lift: number): ComparisonResult { - const jumboRecord = createSessionRecord({ - id: `${harness}-jumbo-rec`, - scenarioId, - sessionNumber: 1, - harness, - variant: 'jumbo', - agentOutput: 'kept context', - filesModified: ['src/report.ts'], - transcript: 'kept context', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - const baselineRecord = createSessionRecord({ - id: `${harness}-baseline-rec`, - scenarioId, - sessionNumber: 1, - harness, - variant: 'baseline', - agentOutput: 'lost context', - filesModified: ['src/report.ts'], - transcript: 'lost context', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - const jumboResult = createTestResult({ - id: `${harness}-jumbo`, - scenarioId, - harness, - sessionRecords: [jumboRecord], - }); - const baselineResult = createTestResult({ - id: `${harness}-baseline`, - scenarioId, - harness, - sessionRecords: [baselineRecord], - }); - return createComparisonResult({ - id: `${harness}-comparison`, - scenarioId, - harness, - jumboResult, - baselineResult, - jumboScores: [ - { dimension: 'file-accuracy', score: 0.7 + lift, maxScore: 1 }, - { dimension: 'knowledge-retention', score: 0.6 + lift, maxScore: 1 }, - ], - baselineScores: [ - { dimension: 'file-accuracy', score: 0.7, maxScore: 1 }, - { dimension: 'knowledge-retention', score: 0.6, maxScore: 1 }, - ], - deltas: [ - { dimension: 'file-accuracy', score: lift, maxScore: 1 }, - { dimension: 'knowledge-retention', score: lift, maxScore: 1 }, - ], - jumboTimeline: [{ - sessionNumber: 1, - scores: [ - { dimension: 'file-accuracy', score: 0.7 + lift, maxScore: 1 }, - { dimension: 'knowledge-retention', score: 0.6 + lift, maxScore: 1 }, - ], - }], - baselineTimeline: [{ - sessionNumber: 1, - scores: [ - { dimension: 'file-accuracy', score: 0.7, maxScore: 1 }, - { dimension: 'knowledge-retention', score: 0.6, maxScore: 1 }, - ], - }], - }); - } - - function seedComparison(comparison: ComparisonResult): void { - store.results.push({ - ...comparison.jumboResult, - comparisonResult: comparison, - } as TestResult); - store.results.push({ - ...comparison.baselineResult, - comparisonResult: comparison, - } as TestResult); - } - - it('prints empty-state message when the scenario has no comparisons', async () => { - const program = createProgram({ storeProvider: async () => store }); - - await program.parseAsync(['node', 'eval', 'report', '--scenario', 'missing-scenario']); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - expect(printed).toContain('No comparisons found for scenario: missing-scenario'); - }); - - it('--harness narrows the terminal report to matching comparisons', async () => { - const scenario = seedScenario(); - seedComparison(makeComparison(scenario.id, 'claude-code', 0.2)); - seedComparison(makeComparison(scenario.id, 'codex-cli', 0.4)); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync([ - 'node', 'eval', 'report', - '--scenario', scenario.id, - '--harness', 'claude-code', - ]); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - expect(printed).toContain('claude-code'); - expect(printed).not.toContain('codex-cli'); - }); - - it('--dimension narrows the JSON report to matching dimensions', async () => { - const scenario = seedScenario(); - seedComparison(makeComparison(scenario.id, 'claude-code', 0.2)); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync([ - 'node', 'eval', 'report', - '--scenario', scenario.id, - '--dimension', 'file-accuracy', - '--json', - ]); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - const parsed = JSON.parse(printed); - expect(parsed.lift.byDimension).toHaveLength(1); - expect(parsed.lift.byDimension[0].dimension).toBe('file-accuracy'); - expect(parsed.audit.trails[0].scoringEvidence).toHaveLength(1); - expect(parsed.audit.trails[0].scoringEvidence[0].dimension).toBe('file-accuracy'); - }); - - it('--json emits a parseable v1 jumbo-eval-report envelope', async () => { - const scenario = seedScenario(); - seedComparison(makeComparison(scenario.id, 'claude-code', 0.2)); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'report', '--scenario', scenario.id, '--json']); - - const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n'); - const parsed = JSON.parse(printed); - expect(parsed.meta.format).toBe('jumbo-eval-report'); - expect(parsed.meta.version).toBe(1); - expect(parsed.meta.scenarioId).toBe(scenario.id); - }); - - it('rejects invalid filters before opening the store', async () => { - const storeProvider = jest.fn(async () => store); - const program = createProgram({ storeProvider }); - - await expect(program.parseAsync([ - 'node', 'eval', 'report', - '--scenario', 'scenario-report', - '--harness', 'bogus-harness', - ])).rejects.toThrow(/Unknown harness/); - - expect(storeProvider).not.toHaveBeenCalled(); - - await expect(program.parseAsync([ - 'node', 'eval', 'report', - '--scenario', 'scenario-report', - '--dimension', 'bogus-dimension', - ])).rejects.toThrow(/Unknown dimension/); - - expect(storeProvider).not.toHaveBeenCalled(); - }); -}); - -describe('status command', () => { - let store: InMemoryStore; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(() => { - store = new InMemoryStore(); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(() => { - logSpy.mockRestore(); - }); - - function seedResult(scenarioId: string, id: string): TestResult { - const record = createSessionRecord({ - id: `${id}-rec`, scenarioId, sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - return createTestResult({ id, scenarioId, harness: 'claude-code', sessionRecords: [record] }); - } - - it('prints empty-state message when store has no results', async () => { - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'status']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('No runs found'); - }); - - it('prints result IDs from a populated store', async () => { - store.results.push(seedResult('scenario-a', 'res-a-1')); - store.results.push(seedResult('scenario-b', 'res-b-1')); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'status']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('res-a-1'); - expect(printed).toContain('res-b-1'); - expect(printed).toContain('2 test result(s)'); - }); - - it('--scenario narrows output to matching results', async () => { - store.results.push(seedResult('scenario-a', 'res-a-1')); - store.results.push(seedResult('scenario-b', 'res-b-1')); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'status', '--scenario', 'scenario-a']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('res-a-1'); - expect(printed).not.toContain('res-b-1'); - }); - - it('--scenario with no matches prints empty-state message', async () => { - store.results.push(seedResult('scenario-a', 'res-a-1')); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'status', '--scenario', 'does-not-exist']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(printed).toContain('No runs found'); - }); - - it('--json emits parseable JSON envelope with results and comparisons', async () => { - store.results.push(seedResult('scenario-a', 'res-a-1')); - - const program = createProgram({ storeProvider: async () => store }); - await program.parseAsync(['node', 'eval', 'status', '--json']); - const printed = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - const parsed = JSON.parse(printed); - expect(Array.isArray(parsed.results)).toBe(true); - expect(Array.isArray(parsed.comparisons)).toBe(true); - expect(parsed.results[0].id).toBe('res-a-1'); - }); -}); - -describe('formatStatusOutput', () => { - it('shows message for empty results', () => { - const output = formatStatusOutput([]); - expect(output).toContain('No runs found'); - }); - - it('formats results with details', () => { - const record = createSessionRecord({ - id: 'r', scenarioId: 'scenario-1', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - const result = createTestResult({ - id: 'result-1', scenarioId: 'scenario-1', harness: 'claude-code', - sessionRecords: [record], - }); - - const output = formatStatusOutput([result]); - expect(output).toContain('1 test result(s)'); - expect(output).toContain('scenario-1'); - expect(output).toContain('result-1'); - expect(output).toContain('claude-code'); - }); - - it('includes comparison summaries', () => { - const record = createSessionRecord({ - id: 'r', scenarioId: 'scenario-1', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - const result = createTestResult({ - id: 'r1', scenarioId: 'scenario-1', harness: 'claude-code', - sessionRecords: [record], - }); - const comp = createComparisonResult({ - id: 'comp-1', scenarioId: 'scenario-1', harness: 'claude-code', - jumboResult: result, baselineResult: result, - jumboScores: [{ dimension: 'test', score: 0.9, maxScore: 1 }], - baselineScores: [{ dimension: 'test', score: 0.6, maxScore: 1 }], - deltas: [{ dimension: 'test', score: 0.3, maxScore: 1 }], - }); - - const output = formatStatusOutput([result], [comp]); - expect(output).toContain('1 comparison(s)'); - expect(output).toContain('Avg Lift'); - expect(output).toContain('+0.300'); - }); -}); - -describe('run command', () => { - let store: InMemoryStore; - let logSpy: jest.SpiedFunction<typeof console.log>; - - beforeEach(() => { - store = new InMemoryStore(); - logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - }); - - afterEach(() => { - logSpy.mockRestore(); - }); - - function makeFakeComparison(scenarioId: string, harness: string): ComparisonResult { - const record = createSessionRecord({ - id: `${harness}-rec`, scenarioId, sessionNumber: 1, harness, - agentOutput: '', filesModified: [], transcript: '', - startedAt: '2026-03-21T10:00:00Z', completedAt: '2026-03-21T10:05:00Z', - }); - const result = createTestResult({ - id: `${harness}-tr`, scenarioId, harness, sessionRecords: [record], - }); - return createComparisonResult({ - id: `${harness}-comp`, scenarioId, harness, - jumboResult: result, baselineResult: result, - jumboScores: [], baselineScores: [], deltas: [], - }); - } - - it('exits with clear error when scenario does not exist', async () => { - const abRunner = jest.fn(); - const program = createProgram({ - storeProvider: async () => store, - abRunner: abRunner as never, - }); - - await expect(program.parseAsync([ - 'node', 'eval', 'run', '--scenario', 'missing-id', - ])).rejects.toThrow(/Scenario not found: missing-id/); - - expect(abRunner).not.toHaveBeenCalled(); - }); - - it('exits with clear error on invalid harness before any execution', async () => { - const scenario = createTestScenario({ - id: 'sid', name: 'S', initialPrompt: 'p', sessionCount: 2, - }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(); - const program = createProgram({ - storeProvider: async () => store, - abRunner: abRunner as never, - }); - - await expect(program.parseAsync([ - 'node', 'eval', 'run', - '--scenario', 'sid', - '--harness', 'claude-code', 'bogus-harness', - ])).rejects.toThrow(/Unknown harness/); - - expect(abRunner).not.toHaveBeenCalled(); - }); - - it('calls runABComparison once per harness with resolved sessions value', async () => { - const scenario = createTestScenario({ - id: 'sid', name: 'S', initialPrompt: 'p', sessionCount: 3, - }); - store.scenarios.push(scenario); - - const calls: Array<{ harness: string; sessionCount: number }> = []; - const abRunner = jest.fn(async (config: { scenario: TestScenario; adapter: { name: string } }) => { - calls.push({ harness: config.adapter.name, sessionCount: config.scenario.sessionCount }); - return makeFakeComparison(config.scenario.id, config.adapter.name); - }); - - const program = createProgram({ - storeProvider: async () => store, - abRunner: abRunner as never, - }); - - await program.parseAsync([ - 'node', 'eval', 'run', - '--scenario', 'sid', - '--harness', 'claude-code', 'codex-cli', - '--sessions', '5', - ]); - - expect(abRunner).toHaveBeenCalledTimes(2); - expect(calls).toEqual([ - { harness: 'claude-code', sessionCount: 5 }, - { harness: 'codex-cli', sessionCount: 5 }, - ]); - }); - - it('prints ReportGenerator output after a successful run', async () => { - const scenario = createTestScenario({ - id: 'sid-report', name: 'S', initialPrompt: 'p', sessionCount: 1, - }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(async (config: { scenario: TestScenario; adapter: { name: string } }) => { - return makeFakeComparison(config.scenario.id, config.adapter.name); - }); - - const program = createProgram({ - storeProvider: async () => store, - abRunner: abRunner as never, - }); - - await program.parseAsync([ - 'node', 'eval', 'run', '--scenario', 'sid-report', - ]); - - const output = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(output).toContain('JUMBO EVALUATION REPORT'); - expect(output).toContain('sid-report'); - }); - - it('runs K replications per harness and persists a ReplicationReport retrievable by runId (--replicate 5)', async () => { - const scenario = createTestScenario({ id: 'sid-rep', name: 'S', initialPrompt: 'p', sessionCount: 1 }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(async (config: { scenario: TestScenario; adapter: { name: string } }) => - makeFakeComparison(config.scenario.id, config.adapter.name), - ); - const program = createProgram({ storeProvider: async () => store, abRunner: abRunner as never }); - - await program.parseAsync(['node', 'eval', 'run', '--scenario', 'sid-rep', '--replicate', '5']); - - expect(abRunner).toHaveBeenCalledTimes(5); // 5 replications × 1 harness - - const runIdLine = logSpy.mock.calls.map((c) => String(c[0])).find((l) => l.startsWith('Run ID: ')); - const runId = runIdLine!.replace('Run ID: ', '').trim(); - const report = await store.getReplicationReport(runId); - expect(report).not.toBeNull(); - expect(report!.k).toBe(5); - expect(report!.harness).toBe('claude-code'); - - const output = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(output).toContain('REPLICATION REPORT'); - }); - - it('keeps single-run behavior and produces no replication report for --replicate 1', async () => { - const scenario = createTestScenario({ id: 'sid-rep1', name: 'S', initialPrompt: 'p', sessionCount: 1 }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(async (config: { scenario: TestScenario; adapter: { name: string } }) => - makeFakeComparison(config.scenario.id, config.adapter.name), - ); - const program = createProgram({ storeProvider: async () => store, abRunner: abRunner as never }); - - await program.parseAsync(['node', 'eval', 'run', '--scenario', 'sid-rep1', '--replicate', '1']); - - expect(abRunner).toHaveBeenCalledTimes(1); - const output = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); - expect(output).toContain('JUMBO EVALUATION REPORT'); - expect(output).not.toContain('REPLICATION REPORT'); - expect(store.replicationReports.size).toBe(0); - }); - - it('errors when --replicate is below 1', async () => { - const scenario = createTestScenario({ id: 'sid-rep0', name: 'S', initialPrompt: 'p', sessionCount: 1 }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(); - const program = createProgram({ storeProvider: async () => store, abRunner: abRunner as never }); - - await expect( - program.parseAsync(['node', 'eval', 'run', '--scenario', 'sid-rep0', '--replicate', '0']), - ).rejects.toThrow(/--replicate must be an integer >= 1/); - expect(abRunner).not.toHaveBeenCalled(); - }); - - it('emits empty-state message when no comparisons were produced', async () => { - const scenario = createTestScenario({ - id: 'sid-empty', name: 'S', initialPrompt: 'p', sessionCount: 1, - }); - store.scenarios.push(scenario); - - const program = createProgram({ - storeProvider: async () => store, - abRunner: (async () => { throw new Error('boom'); }) as never, - }); - - await expect(program.parseAsync([ - 'node', 'eval', 'run', '--scenario', 'sid-empty', - ])).rejects.toThrow(/boom/); - }); - - it('uses scenario sessionCount when --sessions is omitted', async () => { - const scenario = createTestScenario({ - id: 'sid', name: 'S', initialPrompt: 'p', sessionCount: 4, - }); - store.scenarios.push(scenario); - - const abRunner = jest.fn(async (config: { scenario: TestScenario; adapter: { name: string } }) => { - return makeFakeComparison(config.scenario.id, config.adapter.name); - }); - - const program = createProgram({ - storeProvider: async () => store, - abRunner: abRunner as never, - }); - - await program.parseAsync([ - 'node', 'eval', 'run', '--scenario', 'sid', - ]); - - expect(abRunner).toHaveBeenCalledTimes(1); - const callArg = abRunner.mock.calls[0][0] as { scenario: TestScenario }; - expect(callArg.scenario.sessionCount).toBe(4); - }); -}); diff --git a/evals/tests/unit/cli-entry-point.test.ts b/evals/tests/unit/cli-entry-point.test.ts deleted file mode 100644 index 30409f80..00000000 --- a/evals/tests/unit/cli-entry-point.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { isMainModule } from '../../src/cli/index.js'; - -describe('isMainModule', () => { - it('returns false when argv1 is undefined', () => { - expect(isMainModule('/anything/index.js', undefined)).toBe(false); - }); - - it('returns false when argv1 is empty string', () => { - expect(isMainModule('/anything/index.js', '')).toBe(false); - }); - - it('returns true when both paths resolve to the same absolute location', () => { - if (process.platform === 'win32') { - const a = 'C:\\projects\\jumbo\\evals\\dist\\cli\\index.js'; - const b = 'C:/projects/jumbo/evals/dist/cli/index.js'; - expect(isMainModule(a, a)).toBe(true); - expect(isMainModule(a, b)).toBe(true); - expect(isMainModule(b, a)).toBe(true); - } else { - const p = '/home/u/proj/dist/cli/index.js'; - expect(isMainModule(p, p)).toBe(true); - } - }); - - it('returns false when paths point at different files', () => { - if (process.platform === 'win32') { - expect(isMainModule( - 'C:\\projects\\jumbo\\evals\\dist\\cli\\index.js', - 'C:\\projects\\jumbo\\evals\\dist\\cli\\other.js', - )).toBe(false); - } else { - expect(isMainModule('/a/index.js', '/a/other.js')).toBe(false); - } - }); -}); diff --git a/evals/tests/unit/codex-cli-adapter.test.ts b/evals/tests/unit/codex-cli-adapter.test.ts deleted file mode 100644 index 1dc81593..00000000 --- a/evals/tests/unit/codex-cli-adapter.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { CodexCliAdapter } from '../../src/harness/codex-cli-adapter.js'; - -describe('CodexCliAdapter', () => { - const adapter = new CodexCliAdapter(); - - it('has the correct name', () => { - expect(adapter.name).toBe('codex-cli'); - }); - - it('builds a codex command without a positional prompt (prompt goes via stdin)', () => { - const command = adapter.buildCommand(); - expect(command).toEqual([ - 'codex', - '--ask-for-approval', - 'never', - '--sandbox', - 'workspace-write', - 'exec', - '--json', - '--skip-git-repo-check', - ]); - }); - - it('parses JSON output with response field', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - response: 'Created index.ts with hello world', - files_modified: ['index.ts', 'package.json'], - }), - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Created index.ts with hello world'); - expect(result.filesModified).toEqual(['index.ts', 'package.json']); - expect(result.transcript).toContain('Created index.ts'); - }); - - it('handles non-JSON output gracefully', () => { - const result = adapter.parseOutput({ - stdout: 'Plain text response from codex', - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Plain text response from codex'); - expect(result.filesModified).toEqual([]); - }); - - it('extracts token counts from usage object', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - response: 'Done', - usage: { prompt_tokens: 2000, completion_tokens: 1000 }, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(2000); - expect(result.outputTokens).toBe(1000); - }); - - it('extracts token counts from top-level fields', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - response: 'Done', - prompt_tokens: 1500, - completion_tokens: 800, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(1500); - expect(result.outputTokens).toBe(800); - }); - - it('parses Codex exec JSONL events', () => { - const result = adapter.parseOutput({ - stdout: [ - JSON.stringify({ type: 'session.started', id: 'session-1' }), - JSON.stringify({ - type: 'item.completed', - item: { - type: 'message', - role: 'assistant', - content: [{ type: 'output_text', text: 'Implemented the inventory events.' }], - }, - }), - JSON.stringify({ - type: 'turn.completed', - usage: { input_tokens: 2400, output_tokens: 900 }, - }), - ].join('\n'), - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Implemented the inventory events.'); - expect(result.filesModified).toEqual([]); - expect(result.inputTokens).toBe(2400); - expect(result.outputTokens).toBe(900); - }); - - it('returns undefined tokens for non-JSON output', () => { - const result = adapter.parseOutput({ - stdout: 'plain text', - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBeUndefined(); - expect(result.outputTokens).toBeUndefined(); - }); - - it('includes stderr in transcript', () => { - const result = adapter.parseOutput({ - stdout: 'output', - stderr: 'some warning', - exitCode: 0, - }); - - expect(result.transcript).toContain('output'); - expect(result.transcript).toContain('some warning'); - expect(result.transcript).toContain('---stderr---'); - }); - - it('does not embed any prompt in argv (identical-prompts invariant via stdin delivery)', () => { - const command = adapter.buildCommand(); - expect(command).not.toContain('Build a REST API with authentication'); - expect(command.at(-1)).toBe('--skip-git-repo-check'); - }); - - describe('seedToolPermissions', () => { - let workDir: string; - - beforeEach(async () => { - workDir = await mkdtemp(join(tmpdir(), 'codex-adapter-seed-')); - }); - - afterEach(async () => { - await rm(workDir, { recursive: true, force: true }); - }); - - it('writes .codex/config.toml with non-interactive approval and workspace-write sandbox', async () => { - await adapter.seedToolPermissions(workDir); - const toml = await readFile(join(workDir, '.codex', 'config.toml'), 'utf-8'); - expect(toml).toContain('approval_policy = "never"'); - expect(toml).toContain('sandbox_mode = "workspace-write"'); - }); - - it('is idempotent: re-seeding overwrites without error', async () => { - await adapter.seedToolPermissions(workDir); - await adapter.seedToolPermissions(workDir); - const toml = await readFile(join(workDir, '.codex', 'config.toml'), 'utf-8'); - expect(toml).toContain('approval_policy = "never"'); - }); - }); -}); diff --git a/evals/tests/unit/comparison-display.test.ts b/evals/tests/unit/comparison-display.test.ts deleted file mode 100644 index fe281852..00000000 --- a/evals/tests/unit/comparison-display.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { formatComparisonOutput } from '../../src/output/comparison-display.js'; -import { createComparisonResult, createTestResult, createSessionRecord } from '../../src/domain/types.js'; -import type { DimensionScore } from '../../src/domain/types.js'; - -describe('formatComparisonOutput', () => { - it('produces formatted terminal output with scores and deltas', () => { - const jumboRecord = createSessionRecord({ - id: 'jr-1', scenarioId: 'scenario-1', sessionNumber: 1, harness: 'claude-code', - agentOutput: 'output', filesModified: ['src/index.ts', 'src/utils.ts'], - transcript: '', startedAt: '', completedAt: '', - }); - - const baselineRecord = createSessionRecord({ - id: 'br-1', scenarioId: 'scenario-1', sessionNumber: 1, harness: 'claude-code', - agentOutput: 'output', filesModified: ['src/index.ts'], - transcript: '', startedAt: '', completedAt: '', - }); - - const jumboResult = createTestResult({ - id: 'jr', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [jumboRecord], - }); - - const baselineResult = createTestResult({ - id: 'br', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [baselineRecord], - }); - - const jumboScores: DimensionScore[] = [{ dimension: 'file-accuracy', score: 1.0, maxScore: 1 }]; - const baselineScores: DimensionScore[] = [{ dimension: 'file-accuracy', score: 0.67, maxScore: 1 }]; - const deltas: DimensionScore[] = [{ dimension: 'file-accuracy', score: 0.33, maxScore: 1 }]; - - const comparison = createComparisonResult({ - id: 'cmp-1', scenarioId: 'scenario-1', harness: 'claude-code', - jumboResult, baselineResult, jumboScores, baselineScores, deltas, - }); - - const output = formatComparisonOutput(comparison); - - expect(output).toContain('A/B Comparison: scenario-1'); - expect(output).toContain('JUMBO RUN'); - expect(output).toContain('BASELINE RUN'); - expect(output).toContain('file-accuracy'); - expect(output).toContain('1.00/1.00'); - expect(output).toContain('0.67/1.00'); - expect(output).toContain('+0.33'); - expect(output).toContain('2 files modified'); - expect(output).toContain('1 files modified'); - }); - - it('shows negative delta when baseline outperforms jumbo', () => { - const jumboRecord = createSessionRecord({ - id: 'jr-1', scenarioId: 's1', sessionNumber: 1, harness: 'h', - agentOutput: '', filesModified: [], transcript: '', startedAt: '', completedAt: '', - }); - const baselineRecord = createSessionRecord({ - id: 'br-1', scenarioId: 's1', sessionNumber: 1, harness: 'h', - agentOutput: '', filesModified: ['file.ts'], transcript: '', startedAt: '', completedAt: '', - }); - - const comparison = createComparisonResult({ - id: 'cmp', scenarioId: 's1', harness: 'h', - jumboResult: createTestResult({ id: 'jr', scenarioId: 's1', harness: 'h', sessionRecords: [jumboRecord] }), - baselineResult: createTestResult({ id: 'br', scenarioId: 's1', harness: 'h', sessionRecords: [baselineRecord] }), - jumboScores: [{ dimension: 'file-accuracy', score: 0, maxScore: 1 }], - baselineScores: [{ dimension: 'file-accuracy', score: 0.5, maxScore: 1 }], - deltas: [{ dimension: 'file-accuracy', score: -0.5, maxScore: 1 }], - }); - - const output = formatComparisonOutput(comparison); - expect(output).toContain('-0.50'); - }); - - it('shows Jumbo memory capture evidence separately from the scores table', () => { - const jumboRecord = createSessionRecord({ - id: 'jr-1', scenarioId: 's1', sessionNumber: 1, harness: 'h', - agentOutput: '', filesModified: [], transcript: '', startedAt: '', completedAt: '', - }); - const baselineRecord = createSessionRecord({ - id: 'br-1', scenarioId: 's1', sessionNumber: 1, harness: 'h', - agentOutput: '', filesModified: [], transcript: '', startedAt: '', completedAt: '', - }); - - const comparison = createComparisonResult({ - id: 'cmp', scenarioId: 's1', harness: 'h', - jumboResult: createTestResult({ id: 'jr', scenarioId: 's1', harness: 'h', sessionRecords: [jumboRecord] }), - baselineResult: createTestResult({ id: 'br', scenarioId: 's1', harness: 'h', sessionRecords: [baselineRecord] }), - jumboScores: [{ dimension: 'jumbo-memory-capture', score: 1, maxScore: 1, details: 'precision=1.00; recall=1.00' }], - baselineScores: [{ dimension: 'jumbo-memory-capture', score: 0, maxScore: 0, details: 'Not applicable: baseline runs do not use Jumbo project memory.' }], - deltas: [{ dimension: 'jumbo-memory-capture', score: 1, maxScore: 1 }], - }); - - const output = formatComparisonOutput(comparison); - expect(output).toContain('JUMBO MEMORY CAPTURE EVIDENCE'); - expect(output).toContain('Jumbo: precision=1.00; recall=1.00'); - expect(output).toContain('Baseline: Not applicable'); - }); -}); diff --git a/evals/tests/unit/control.test.ts b/evals/tests/unit/control.test.ts deleted file mode 100644 index 1f66e24d..00000000 --- a/evals/tests/unit/control.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { Command } from 'commander'; -import { registerControlCommand } from '../../src/cli/commands/control.js'; -import type { RunControlFile } from '../../src/domain/types.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; - -function makeMemoryStore(): ResultStore & { control: Map<string, RunControlFile> } { - const control = new Map<string, RunControlFile>(); - const store: Partial<ResultStore> & { control: Map<string, RunControlFile> } = { - control, - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async () => {}, - getSessionRecords: async () => [], - saveTestResult: async () => {}, - getTestResult: async () => null, - listTestResults: async () => [], - saveRunRecord: async () => {}, - getRunRecord: async () => null, - listRunRecords: async () => [], - writeHeartbeat: async () => {}, - readHeartbeat: async () => null, - writeRunControl: async (runId: string, file: RunControlFile) => { - control.set(runId, file); - }, - readRunControl: async (runId: string) => control.get(runId) ?? null, - }; - return store as ResultStore & { control: Map<string, RunControlFile> }; -} - -function buildProgram(store: ResultStore): Command { - const program = new Command(); - program.exitOverride(); - registerControlCommand(program, { storeProvider: async () => store }); - return program; -} - -describe('eval control', () => { - it('refuses mutating actions without --allow-tampering', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await expect(program.parseAsync(['node', 'eval', 'control', 'run-1', 'pause'])) - .rejects.toThrow(/--allow-tampering/); - expect(store.control.size).toBe(0); - }); - - it('rejects unknown actions', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await expect(program.parseAsync(['node', 'eval', 'control', 'run-1', 'flip', '--allow-tampering'])) - .rejects.toThrow(/Unknown control action/); - }); - - it('records pause action with pauseRequested=true', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await program.parseAsync(['node', 'eval', 'control', 'run-1', 'pause', '--allow-tampering']); - const file = store.control.get('run-1'); - expect(file?.pauseRequested).toBe(true); - expect(file?.abortRequested).toBe(false); - expect(file?.pendingActions).toHaveLength(1); - expect(file?.pendingActions[0].action).toBe('pause'); - }); - - it('resume clears pauseRequested', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await program.parseAsync(['node', 'eval', 'control', 'run-1', 'pause', '--allow-tampering']); - const program2 = buildProgram(store); - await program2.parseAsync(['node', 'eval', 'control', 'run-1', 'resume', '--allow-tampering']); - const file = store.control.get('run-1'); - expect(file?.pauseRequested).toBe(false); - expect(file?.pendingActions.map((e) => e.action)).toEqual(['pause', 'resume']); - }); - - it('abort sets abortRequested', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await program.parseAsync([ - 'node', 'eval', 'control', 'run-1', 'abort', - '--allow-tampering', '--operator', 'alice', - ]); - const file = store.control.get('run-1'); - expect(file?.abortRequested).toBe(true); - expect(file?.pendingActions[0].operator).toBe('alice'); - }); - - it('inject-context requires a payload', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await expect(program.parseAsync([ - 'node', 'eval', 'control', 'run-1', 'inject-context', - '--allow-tampering', - ])).rejects.toThrow(/payload/); - }); - - it('inject-context refuses --variant baseline', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await expect(program.parseAsync([ - 'node', 'eval', 'control', 'run-1', 'inject-context', 'remember X', - '--allow-tampering', '--variant', 'baseline', - ])).rejects.toThrow(/jumbo variant/); - }); - - it('inject-context defaults variant to jumbo and stores payload', async () => { - const store = makeMemoryStore(); - const program = buildProgram(store); - await program.parseAsync([ - 'node', 'eval', 'control', 'run-1', 'inject-context', 'edge case Y', - '--allow-tampering', - ]); - const file = store.control.get('run-1'); - expect(file?.pendingActions).toHaveLength(1); - expect(file?.pendingActions[0].action).toBe('inject-context'); - expect(file?.pendingActions[0].payload).toBe('edge case Y'); - expect(file?.pendingActions[0].variant).toBe('jumbo'); - expect(file?.pauseRequested).toBe(false); - expect(file?.abortRequested).toBe(false); - }); - - it('appends multiple pending actions across invocations', async () => { - const store = makeMemoryStore(); - await buildProgram(store).parseAsync([ - 'node', 'eval', 'control', 'run-1', 'inject-context', 'first', - '--allow-tampering', - ]); - await buildProgram(store).parseAsync([ - 'node', 'eval', 'control', 'run-1', 'inject-context', 'second', - '--allow-tampering', - ]); - expect(store.control.get('run-1')?.pendingActions.map((e) => e.payload)).toEqual(['first', 'second']); - }); -}); diff --git a/evals/tests/unit/cross-harness-display.test.ts b/evals/tests/unit/cross-harness-display.test.ts deleted file mode 100644 index dad96df4..00000000 --- a/evals/tests/unit/cross-harness-display.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { formatCrossHarnessComparison } from '../../src/output/cross-harness-display.js'; -import { createComparisonResult, createTestResult, createSessionRecord } from '../../src/domain/types.js'; -import type { ComparisonResult } from '../../src/domain/types.js'; - -function makeComparison(harness: string, jumboScoreValue: number, baselineScoreValue: number): ComparisonResult { - const record = createSessionRecord({ - id: `rec-${harness}`, - scenarioId: 'scenario-1', - sessionNumber: 1, - harness, - agentOutput: 'output', - filesModified: ['src/index.ts'], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - const jumboResult = createTestResult({ - id: `jumbo-${harness}`, - scenarioId: 'scenario-1', - harness, - sessionRecords: [record], - }); - - const baselineResult = createTestResult({ - id: `baseline-${harness}`, - scenarioId: 'scenario-1', - harness, - sessionRecords: [record], - }); - - return createComparisonResult({ - id: `comp-${harness}`, - scenarioId: 'scenario-1', - harness, - jumboResult, - baselineResult, - jumboScores: [ - { dimension: 'file-accuracy', score: jumboScoreValue, maxScore: 1 }, - { dimension: 'knowledge-retention', score: jumboScoreValue, maxScore: 1 }, - ], - baselineScores: [ - { dimension: 'file-accuracy', score: baselineScoreValue, maxScore: 1 }, - { dimension: 'knowledge-retention', score: baselineScoreValue, maxScore: 1 }, - ], - deltas: [ - { dimension: 'file-accuracy', score: Math.round((jumboScoreValue - baselineScoreValue) * 100) / 100, maxScore: 1 }, - { dimension: 'knowledge-retention', score: Math.round((jumboScoreValue - baselineScoreValue) * 100) / 100, maxScore: 1 }, - ], - }); -} - -describe('formatCrossHarnessComparison', () => { - it('returns message for empty comparisons', () => { - const output = formatCrossHarnessComparison([]); - expect(output).toContain('No comparison results'); - }); - - it('returns message for single comparison', () => { - const comp = makeComparison('claude-code', 0.9, 0.6); - const output = formatCrossHarnessComparison([comp]); - expect(output).toContain('at least 2 harnesses'); - }); - - it('shows cross-harness comparison for two harnesses', () => { - const claude = makeComparison('claude-code', 0.9, 0.6); - const codex = makeComparison('codex-cli', 0.8, 0.5); - - const output = formatCrossHarnessComparison([claude, codex]); - - expect(output).toContain('Cross-Harness Comparison'); - expect(output).toContain('claude-code'); - expect(output).toContain('codex-cli'); - expect(output).toContain('scenario-1'); - }); - - it('shows lift deltas per harness', () => { - const claude = makeComparison('claude-code', 0.9, 0.6); // delta = +0.30 - const codex = makeComparison('codex-cli', 0.8, 0.5); // delta = +0.30 - - const output = formatCrossHarnessComparison([claude, codex]); - - expect(output).toContain('JUMBO LIFT BY HARNESS'); - expect(output).toContain('+0.30'); - }); - - it('shows absolute Jumbo scores per harness', () => { - const claude = makeComparison('claude-code', 0.9, 0.6); - const codex = makeComparison('codex-cli', 0.8, 0.5); - - const output = formatCrossHarnessComparison([claude, codex]); - - expect(output).toContain('ABSOLUTE SCORES'); - expect(output).toContain('0.90/1.00'); - expect(output).toContain('0.80/1.00'); - }); - - it('shows summary with highest Jumbo lift', () => { - const claude = makeComparison('claude-code', 0.9, 0.6); // avg lift = 0.30 - const codex = makeComparison('codex-cli', 0.85, 0.4); // avg lift = 0.45 - - const output = formatCrossHarnessComparison([claude, codex]); - - expect(output).toContain('SUMMARY'); - expect(output).toContain('Highest Jumbo lift: codex-cli'); - }); - - it('rejects comparisons spanning multiple scenarios', () => { - const comp1 = makeComparison('claude-code', 0.9, 0.6); - const comp2: ComparisonResult = { - ...makeComparison('codex-cli', 0.8, 0.5), - scenarioId: 'different-scenario', - }; - - const output = formatCrossHarnessComparison([comp1, comp2]); - expect(output).toContain('multiple scenarios'); - }); - - it('handles dimensions present in one harness but not another', () => { - const claude = makeComparison('claude-code', 0.9, 0.6); - const codex: ComparisonResult = { - ...makeComparison('codex-cli', 0.8, 0.5), - jumboScores: [{ dimension: 'file-accuracy', score: 0.8, maxScore: 1 }], - baselineScores: [{ dimension: 'file-accuracy', score: 0.5, maxScore: 1 }], - deltas: [{ dimension: 'file-accuracy', score: 0.3, maxScore: 1 }], - }; - - const output = formatCrossHarnessComparison([claude, codex]); - - // Should show n/a for knowledge-retention under codex - expect(output).toContain('n/a'); - }); -}); diff --git a/evals/tests/unit/disruption-recovery-scorer.test.ts b/evals/tests/unit/disruption-recovery-scorer.test.ts deleted file mode 100644 index 15716f91..00000000 --- a/evals/tests/unit/disruption-recovery-scorer.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { scoreDisruptionRecovery, scoreDisruptionRecoveryTimeline } from '../../src/scoring/disruption-recovery-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; -import type { Disruption, WorkspaceSnapshot } from '../../src/domain/types.js'; - -function makeRecord(sessionNumber: number, opts: { - agentOutput?: string; - filesModified?: string[]; - workspaceSnapshot?: WorkspaceSnapshot; -} = {}) { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'scenario-1', - sessionNumber, - harness: 'claude-code', - agentOutput: opts.agentOutput ?? '', - filesModified: opts.filesModified ?? [], - transcript: '', - workspaceSnapshot: opts.workspaceSnapshot, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -function makeSnapshot(files: Record<string, string>): WorkspaceSnapshot { - return { - capturedAt: '2026-03-21T10:05:00Z', - files: Object.entries(files).map(([path, content]) => ({ path, content })), - }; -} - -describe('scoreDisruptionRecovery', () => { - it('returns perfect score when all recovery patterns persist in post-disruption sessions', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case not camelCase', - recoveryPatterns: ['snake_case'], - }]; - - const records = [ - makeRecord(1, { agentOutput: 'initial setup' }), - makeRecord(2, { agentOutput: 'switching to snake_case' }), - makeRecord(3, { agentOutput: 'continued with snake_case conventions' }), - ]; - - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(1); - expect(score.dimension).toBe('disruption-recovery'); - }); - - it('returns zero when recovery patterns are lost after disruption', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case not camelCase', - recoveryPatterns: ['snake_case'], - }]; - - const records = [ - makeRecord(1, { agentOutput: 'initial' }), - makeRecord(2, { agentOutput: 'applied snake_case' }), - makeRecord(3, { agentOutput: 'reverted to camelCase everywhere' }), - ]; - - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(0); - expect(score.details).toContain('lost "snake_case"'); - }); - - it('handles multiple disruptions with different recovery patterns', () => { - const disruptions: Disruption[] = [ - { - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case', - recoveryPatterns: ['snake_case'], - }, - { - type: 'new-constraint', - sessionNumber: 3, - content: 'Add error handling', - recoveryPatterns: ['try', 'catch'], - }, - ]; - - const records = [ - makeRecord(1, { agentOutput: 'initial' }), - makeRecord(2, { agentOutput: 'snake_case' }), - makeRecord(3, { agentOutput: 'snake_case with try catch' }), - makeRecord(4, { agentOutput: 'snake_case but no error handling' }), - ]; - - // Disruption 1 (session 2): post-sessions 3,4. snake_case in both → 2/2 - // Disruption 2 (session 3): post-session 4 only. try=no, catch=no → 0/2 - // Total: 2 pass out of 4 checks = 0.5 - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(0.5); - }); - - it('returns trivially satisfied when no disruptions defined', () => { - const score = scoreDisruptionRecovery([makeRecord(1)], []); - expect(score.score).toBe(1); - expect(score.details).toContain('trivially satisfied'); - }); - - it('uses workspace snapshot as primary evidence for recovery (correct code, terse transcript)', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case', - recoveryPatterns: ['snake_case'], - }]; - - // No mention of snake_case in output — but the file uses snake_case naming - const records = [ - makeRecord(1, { agentOutput: 'initial' }), - makeRecord(2, { agentOutput: 'updated' }), - makeRecord(3, { - agentOutput: 'done', - workspaceSnapshot: makeSnapshot({ 'src/utils.ts': 'const snake_case_var = getValue();' }), - }), - ]; - - // snake_case is not in agentOutput but is in the workspace file — should pass - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(1); - }); - - it('rejects keyword-only transcript when workspace snapshot does not contain pattern', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case', - recoveryPatterns: ['snake_case'], - }]; - - // Transcript/output claims snake_case but workspace file only has camelCase - const records = [ - makeRecord(1, { agentOutput: 'initial' }), - makeRecord(2, { agentOutput: 'switched to snake_case' }), - makeRecord(3, { - agentOutput: 'still using snake_case', - workspaceSnapshot: makeSnapshot({ 'src/utils.ts': 'const userName = getUserName();' }), - }), - ]; - - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(0); - expect(score.details).toContain('lost "snake_case"'); - }); - - it('handles disruption at last session (no post-sessions to check)', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 3, - content: 'Fix something', - recoveryPatterns: ['fix'], - }]; - - const records = [ - makeRecord(1, {}), - makeRecord(2, {}), - makeRecord(3, { agentOutput: 'applied fix' }), - ]; - - // No sessions after disruption to check - const score = scoreDisruptionRecovery(records, disruptions); - expect(score.score).toBe(1); - expect(score.details).toContain('No recovery checks applicable'); - }); -}); - -describe('scoreDisruptionRecoveryTimeline', () => { - it('produces per-session timeline with recovery status', () => { - const disruptions: Disruption[] = [{ - type: 'correction', - sessionNumber: 2, - content: 'Use snake_case', - recoveryPatterns: ['snake_case'], - }]; - - const records = [ - makeRecord(1, { agentOutput: 'initial' }), - makeRecord(2, { agentOutput: 'snake_case' }), - makeRecord(3, { agentOutput: 'snake_case still' }), - makeRecord(4, { agentOutput: 'forgot everything' }), - ]; - - const timeline = scoreDisruptionRecoveryTimeline(records, disruptions); - - expect(timeline).toHaveLength(4); - expect(timeline[0].score).toBe(1); // Session 1: no active disruptions yet - expect(timeline[1].score).toBe(1); // Session 2: disruption injected here, no active yet (< not <=) - expect(timeline[2].score).toBe(1); // Session 3: snake_case present - expect(timeline[3].score).toBe(0); // Session 4: lost - }); - - it('returns empty for no disruptions', () => { - const timeline = scoreDisruptionRecoveryTimeline([makeRecord(1)], []); - expect(timeline).toEqual([]); - }); -}); diff --git a/evals/tests/unit/domain.test.ts b/evals/tests/unit/domain.test.ts deleted file mode 100644 index 1e6d3cfe..00000000 --- a/evals/tests/unit/domain.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { createTestScenario, createSessionRecord, createTestResult } from '../../src/domain/types.js'; - -describe('TestScenario', () => { - it('creates a valid scenario with all required fields', () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Basic TypeScript project', - initialPrompt: 'Build a hello world TypeScript project', - sessionCount: 1, - }); - - expect(scenario.id).toBe('scenario-1'); - expect(scenario.name).toBe('Basic TypeScript project'); - expect(scenario.initialPrompt).toBe('Build a hello world TypeScript project'); - expect(scenario.sessionCount).toBe(1); - expect(scenario.createdAt).toBeDefined(); - }); - - it('rejects empty id', () => { - expect(() => - createTestScenario({ id: '', name: 'test', initialPrompt: 'prompt', sessionCount: 1 }), - ).toThrow('TestScenario requires an id'); - }); - - it('rejects empty name', () => { - expect(() => - createTestScenario({ id: 'id', name: '', initialPrompt: 'prompt', sessionCount: 1 }), - ).toThrow('TestScenario requires a name'); - }); - - it('rejects empty initialPrompt', () => { - expect(() => - createTestScenario({ id: 'id', name: 'name', initialPrompt: '', sessionCount: 1 }), - ).toThrow('TestScenario requires an initialPrompt'); - }); - - it('rejects sessionCount < 1', () => { - expect(() => - createTestScenario({ id: 'id', name: 'name', initialPrompt: 'prompt', sessionCount: 0 }), - ).toThrow('TestScenario requires sessionCount >= 1'); - }); - - it('accepts valid structuralAssertions', () => { - const scenario = createTestScenario({ - id: 'id', - name: 'name', - initialPrompt: 'prompt', - sessionCount: 3, - structuralAssertions: [ - { id: 'a1', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 'a2', file: 'src/b.ts', sessionNumber: 3, matcher: { kind: 'matchesRegex', pattern: 'x' } }, - ], - }); - expect(scenario.structuralAssertions).toHaveLength(2); - }); - - it('rejects a structural assertion with no id', () => { - expect(() => - createTestScenario({ - id: 'id', - name: 'name', - initialPrompt: 'prompt', - sessionCount: 1, - structuralAssertions: [{ id: '', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }], - }), - ).toThrow('require an id'); - }); - - it('rejects duplicate structural assertion ids', () => { - expect(() => - createTestScenario({ - id: 'id', - name: 'name', - initialPrompt: 'prompt', - sessionCount: 1, - structuralAssertions: [ - { id: 'dup', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 'dup', file: 'src/b.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ], - }), - ).toThrow('duplicate id: dup'); - }); - - it('rejects a structural assertion with no file glob', () => { - expect(() => - createTestScenario({ - id: 'id', - name: 'name', - initialPrompt: 'prompt', - sessionCount: 1, - structuralAssertions: [{ id: 'a1', file: '', sessionNumber: 1, matcher: { kind: 'fileExists' } }], - }), - ).toThrow('requires a file glob'); - }); - - it('rejects a structural assertion whose sessionNumber exceeds sessionCount', () => { - expect(() => - createTestScenario({ - id: 'id', - name: 'name', - initialPrompt: 'prompt', - sessionCount: 3, - structuralAssertions: [{ id: 'a1', file: 'src/a.ts', sessionNumber: 4, matcher: { kind: 'fileExists' } }], - }), - ).toThrow('sessionNumber must be an integer in 1..3'); - }); -}); - -describe('SessionRecord', () => { - it('creates a valid session record', () => { - const record = createSessionRecord({ - id: 'session-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - variant: 'baseline', - scenarioPrompt: 'Build the app', - effectivePrompt: 'Build the app', - agentOutput: 'Created index.ts', - filesModified: ['index.ts'], - transcript: 'full transcript...', - jumboLifecycleAudit: { - sessionStartExecuted: true, - goalStartExecuted: true, - goalSubmitExecuted: true, - sessionEndExecuted: true, - activeGoalId: 'goal-1', - goalStatusBefore: 'refined', - goalStatusAfter: 'submitted', - sessionsTotalDelta: 1, - sessionsEndedDelta: 1, - evidence: {}, - }, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - expect(record.id).toBe('session-1'); - expect(record.scenarioId).toBe('scenario-1'); - expect(record.sessionNumber).toBe(1); - expect(record.harness).toBe('claude-code'); - expect(record.variant).toBe('baseline'); - expect(record.scenarioPrompt).toBe('Build the app'); - expect(record.effectivePrompt).toBe('Build the app'); - expect(record.filesModified).toEqual(['index.ts']); - expect(record.jumboLifecycleAudit?.goalSubmitExecuted).toBe(true); - expect(record.jumboLifecycleAudit?.activeGoalId).toBe('goal-1'); - }); - - it('rejects empty id', () => { - expect(() => - createSessionRecord({ - id: '', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - startedAt: '', - completedAt: '', - }), - ).toThrow('SessionRecord requires an id'); - }); - - it('rejects sessionNumber < 1', () => { - expect(() => - createSessionRecord({ - id: 'id', - scenarioId: 'scenario-1', - sessionNumber: 0, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - startedAt: '', - completedAt: '', - }), - ).toThrow('SessionRecord requires sessionNumber >= 1'); - }); -}); - -describe('TestResult', () => { - it('creates a valid test result with session records', () => { - const record = createSessionRecord({ - id: 'session-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: 'output', - filesModified: [], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - const result = createTestResult({ - id: 'result-1', - scenarioId: 'scenario-1', - harness: 'claude-code', - sessionRecords: [record], - }); - - expect(result.id).toBe('result-1'); - expect(result.sessionRecords).toHaveLength(1); - expect(result.createdAt).toBeDefined(); - }); - - it('rejects empty id', () => { - expect(() => - createTestResult({ id: '', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [] }), - ).toThrow('TestResult requires an id'); - }); -}); diff --git a/evals/tests/unit/end-to-end-validation.test.ts b/evals/tests/unit/end-to-end-validation.test.ts deleted file mode 100644 index 023d26ac..00000000 --- a/evals/tests/unit/end-to-end-validation.test.ts +++ /dev/null @@ -1,407 +0,0 @@ -import { describe, it, expect, afterEach } from '@jest/globals'; -import { mkdir, writeFile } from 'node:fs/promises'; -import * as path from 'node:path'; -import { runABComparison } from '../../src/ab-runner.js'; -import { HarnessExecutionError } from '../../src/run-session.js'; -import { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { SessionRecord, TestResult, TestScenario, ComparisonResult } from '../../src/domain/types.js'; -import { generateFullReport } from '../../src/output/report-generator.js'; - -/** - * Deterministic end-to-end validation. - * - * These tests prove the eval harness measures Jumbo context effects rather - * than prompt drift, filesystem leakage, scorer artifacts, or partial-run - * persistence. They use a fake harness whose behavior depends entirely on - * the delivered Jumbo context, and a fake executor that simulates the - * jumbo CLI deterministically while using real temp directories so that - * workspace isolation is exercised — not mocked. - */ - -const PRIOR_SESSION_FACT = 'snake_case'; -const FACT_FOLLOWUP = `Use ${PRIOR_SESSION_FACT} for all identifiers.`; - -interface FakeOptions { - /** When true, the simulated agent does not consult or update Jumbo memory (control). */ - readonly emptyJumboMemory?: boolean; - /** When set, the harness command exits with this code on session N. */ - readonly failHarnessOnSession?: { variant: 'jumbo' | 'baseline'; sessionNumber: number }; -} - -/** - * Executor that uses real fs for temp dirs and snapshots, but simulates - * the jumbo CLI and the harness CLI deterministically. Each variant gets - * its own work dir; each work dir gets its own simulated jumbo memory. - */ -class FakeJumboExecutor extends LocalExecutor { - private readonly memoryByDir = new Map<string, string[]>(); - private readonly sessionByDir = new Map<string, number>(); - private readonly sessionsByDir = new Map<string, { id: string; ended: boolean }[]>(); - private readonly goalSubmittedByDir = new Map<string, boolean>(); - readonly execCalls: Array<{ workDir: string; command: string[] }> = []; - - constructor(private readonly options: FakeOptions = {}) { - super(); - } - - async exec( - workDir: string, - command: string[], - options?: { stdin?: string; env?: Record<string, string | undefined> }, - ): Promise<ExecResult> { - this.execCalls.push({ workDir, command }); - - if (command[0] === 'jumbo') { - return this.handleJumbo(workDir, command, options?.env); - } - - if (command[0] === 'fake-harness') { - return this.handleHarness(workDir, options?.stdin ?? ''); - } - - return { stdout: '', stderr: `unknown command: ${command[0]}`, exitCode: 1 }; - } - - private async handleJumbo( - workDir: string, - command: string[], - env?: Record<string, string | undefined>, - ): Promise<ExecResult> { - if (command[1] === '--version') { - // Baseline arm runs with a shimmed PATH; simulate the fail-loud shim. - if (env?.PATH?.includes('.eval-bin')) { - return { - stdout: '', - stderr: 'ERROR: jumbo is not available in the baseline arm (eval shim)', - exitCode: 127, - }; - } - return { stdout: 'jumbo 1.2.3', stderr: '', exitCode: 0 }; - } - - if (command[1] === 'init') { - this.memoryByDir.set(workDir, []); - return { stdout: 'jumbo initialized', stderr: '', exitCode: 0 }; - } - - if (command[1] === 'goal' && command[2] === 'add') { - const titleIdx = command.indexOf('--title'); - const title = titleIdx >= 0 ? command[titleIdx + 1] : 'goal'; - const goalId = `goal-${title.replace(/\s+/g, '-').toLowerCase()}`; - return { stdout: JSON.stringify({ goalId }), stderr: '', exitCode: 0 }; - } - - if (command[1] === 'goal' && command[2] === 'show') { - // The agent simulation in handleHarness toggles goalSubmittedByDir - // to 'submitted' after running its protocol; the framework's - // post-session audit reads that state through goal show. - const status = this.goalSubmittedByDir.get(workDir) ? 'submitted' : 'refined'; - const idIdx = command.indexOf('--id'); - const id = idIdx >= 0 ? command[idIdx + 1] : undefined; - return { stdout: JSON.stringify({ goal: { goalId: id, status } }), stderr: '', exitCode: 0 }; - } - - if (command[1] === 'sessions' && command[2] === 'list') { - const statusIdx = command.indexOf('--status'); - const filter = statusIdx >= 0 ? command[statusIdx + 1] : 'all'; - const sessions = this.sessionsByDir.get(workDir) ?? []; - const filtered = filter === 'ended' - ? sessions.filter((s) => s.ended) - : sessions; - return { - stdout: JSON.stringify(filtered.map((s) => ({ sessionId: s.id, ended: s.ended }))), - stderr: '', - exitCode: 0, - }; - } - - // jumbo decisions/guidelines/.../relations list --format json - if (command[2] === 'list' && command[3] === '--format' && command[4] === 'json') { - const memory = this.memoryByDir.get(workDir) ?? []; - const items = command[1] === 'decisions' - ? memory.map((text, i) => ({ decisionId: `dec-${i}`, title: text })) - : []; - return { stdout: JSON.stringify(items), stderr: '', exitCode: 0 }; - } - - return { stdout: '', stderr: `unsupported jumbo command: ${command.join(' ')}`, exitCode: 1 }; - } - - private async handleHarness(workDir: string, prompt: string): Promise<ExecResult> { - const isJumboDir = (this.memoryByDir.get(workDir) ?? null) !== null; - const variant: 'jumbo' | 'baseline' = isJumboDir ? 'jumbo' : 'baseline'; - const session = (this.sessionByDir.get(workDir) ?? 0) + 1; - this.sessionByDir.set(workDir, session); - - const fail = this.options.failHarnessOnSession; - if (fail && fail.variant === variant && fail.sessionNumber === session) { - return { stdout: '', stderr: 'simulated harness failure', exitCode: 1 }; - } - - // Simulate a protocol-compliant agent on the Jumbo arm: when the prompt - // includes the lifecycle protocol, the agent runs `jumbo session start`, - // pulls prior memory, and runs `jumbo session end` recording observed - // facts. This makes the fake harness exercise the agent-driven lifecycle - // rather than relying on the framework to inject memory into the prompt. - const followsProtocol = isJumboDir - && !this.options.emptyJumboMemory - && prompt.includes('Jumbo lifecycle protocol'); - - let effectivePrompt = prompt; - if (followsProtocol) { - const memory = this.memoryByDir.get(workDir) ?? []; - const priorContext = memory.length === 0 - ? 'No prior project memory.' - : `Prior project memory:\n${memory.join('\n')}`; - effectivePrompt = `${priorContext}\n\n${prompt}`; - // Record an active session for this work cycle so the post-session - // audit can observe sessionStartExecuted via jumbo sessions list. - const sessions = this.sessionsByDir.get(workDir) ?? []; - sessions.push({ id: `s-${session}`, ended: false }); - this.sessionsByDir.set(workDir, sessions); - } - - const factPresent = effectivePrompt.includes(PRIOR_SESSION_FACT); - const filename = 'src/module.ts'; - const fileContent = factPresent - ? `// Honors prior decision: ${PRIOR_SESSION_FACT} naming.\nexport const item_count = 0;\n` - : `// No prior context; using default naming.\nexport const itemCount = 0;\n`; - - const fullPath = path.join(workDir, filename); - await mkdir(path.dirname(fullPath), { recursive: true }); - await writeFile(fullPath, fileContent, 'utf-8'); - - if (followsProtocol) { - // Agent runs `jumbo session end` recording the observed fact, and - // `jumbo goal submit` marking the goal complete. - if (factPresent || effectivePrompt.includes(PRIOR_SESSION_FACT)) { - const memory = this.memoryByDir.get(workDir) ?? []; - memory.push(`Decision: ${FACT_FOLLOWUP}`); - this.memoryByDir.set(workDir, memory); - } - const sessions = this.sessionsByDir.get(workDir) ?? []; - const last = sessions[sessions.length - 1]; - if (last) last.ended = true; - this.goalSubmittedByDir.set(workDir, true); - } - - const json = JSON.stringify({ - result: factPresent - ? `Implemented module honoring ${PRIOR_SESSION_FACT} naming.` - : 'Implemented module with default naming.', - files_modified: [filename], - }); - return { stdout: json, stderr: '', exitCode: 0 }; - } -} - -function createFakeAdapter(): HarnessAdapter { - return { - name: 'fake-harness', - buildCommand: () => ['fake-harness'], - parseOutput: (result: ExecResult) => { - try { - const parsed = JSON.parse(result.stdout) as { result?: string; files_modified?: string[] }; - return { - agentOutput: parsed.result ?? result.stdout, - filesModified: parsed.files_modified ?? [], - transcript: result.stdout, - inputTokens: 100, - outputTokens: 50, - }; - } catch { - return { agentOutput: result.stdout, filesModified: [], transcript: result.stdout }; - } - }, - seedToolPermissions: async () => {}, - }; -} - -function createInMemoryStore(): ResultStore & { - readonly records: SessionRecord[]; - readonly results: TestResult[]; -} { - const records: SessionRecord[] = []; - const results: TestResult[] = []; - return { - records, - results, - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async (r) => { records.push(r); }, - getSessionRecords: async (scenarioId: string) => records.filter((r) => r.scenarioId === scenarioId), - saveTestResult: async (r) => { results.push(r); }, - getTestResult: async () => null, - listTestResults: async () => results, - }; -} - -function createScenario(overrides?: Partial<TestScenario>): TestScenario { - return createTestScenario({ - id: 'e2e-scenario', - name: 'E2E deterministic validation', - initialPrompt: `Begin the project. Adopt ${PRIOR_SESSION_FACT} for naming and remember the decision.`, - continuationPrompt: 'Continue the project, honoring any prior decisions.', - sessionCount: 2, - expectedFiles: ['src/module.ts'], - retentionPatterns: [PRIOR_SESSION_FACT], - jumboPlan: { - goals: [ - { - title: 'Establish naming convention', - objective: 'Implement the module using the agreed naming convention', - criteria: ['module file exists'], - sessionAvailableFrom: 1, - }, - ], - }, - ...overrides, - }); -} - -describe('Deterministic end-to-end validation', () => { - const created: ComparisonResult[] = []; - - afterEach(() => { - created.length = 0; - }); - - it('produces a positive Jumbo lift only when the prior-session fact is delivered', async () => { - const scenario = createScenario(); - const executor = new FakeJumboExecutor(); - const adapter = createFakeAdapter(); - const store = createInMemoryStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - created.push(result); - - // Session 1 of Jumbo's agent simulation runs the protocol, captures the - // fact via its (simulated) session end, and the audit verifies it. The - // agent in session 2 then reads the prior memory through `jumbo session - // start` and writes snake_case files — so retention rises in session 2. - const jumboSession2 = result.jumboResult.sessionRecords.find((r) => r.sessionNumber === 2)!; - expect(jumboSession2.jumboLifecycleAudit?.sessionStartExecuted).toBe(true); - expect(jumboSession2.jumboLifecycleAudit?.sessionEndExecuted).toBe(true); - - const baselineSession2 = result.baselineResult.sessionRecords.find((r) => r.sessionNumber === 2)!; - expect(baselineSession2.deliveredContext).toBeUndefined(); - expect(baselineSession2.jumboLifecycleAudit).toBeUndefined(); - - const retentionDelta = result.deltas.find((d) => d.dimension === 'knowledge-retention'); - expect(retentionDelta).toBeDefined(); - expect(retentionDelta!.score).toBeGreaterThan(0); - - // The retention pattern only appears in the file content when the fact - // was injected into the prompt. The Jumbo final session's file content - // honors snake_case naming; the baseline final session's does not. - const jumboFinalFile = result.jumboResult.sessionRecords.at(-1)! - .workspaceSnapshot!.files.find((f) => f.path === 'src/module.ts')!; - const baselineFinalFile = result.baselineResult.sessionRecords.at(-1)! - .workspaceSnapshot!.files.find((f) => f.path === 'src/module.ts')!; - expect(jumboFinalFile.content).toContain(PRIOR_SESSION_FACT); - expect(baselineFinalFile.content).not.toContain(PRIOR_SESSION_FACT); - }); - - it('produces zero lift when the Jumbo context carries no prior-session fact', async () => { - const scenario = createScenario(); - const executor = new FakeJumboExecutor({ emptyJumboMemory: true }); - const adapter = createFakeAdapter(); - const store = createInMemoryStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - created.push(result); - - // Same fake harness, but Jumbo session-start delivers no fact. The Jumbo - // variant cannot outperform baseline on retention or file-accuracy because - // the harness behavior depends only on the delivered fact. - const retentionDelta = result.deltas.find((d) => d.dimension === 'knowledge-retention'); - const fileDelta = result.deltas.find((d) => d.dimension === 'file-accuracy'); - expect(retentionDelta!.score).toBe(0); - expect(fileDelta!.score).toBe(0); - }); - - it('isolates variants in separate temp workdirs with no cross-variant filesystem leakage', async () => { - const scenario = createScenario(); - const executor = new FakeJumboExecutor(); - const adapter = createFakeAdapter(); - const store = createInMemoryStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - created.push(result); - - const workDirs = new Set(executor.execCalls.map((c) => c.workDir)); - expect(workDirs.size).toBe(2); - - const jumboFinal = result.jumboResult.sessionRecords.at(-1)!; - const baselineFinal = result.baselineResult.sessionRecords.at(-1)!; - - // No cross-variant leakage: the file content reflecting the prior-session - // fact only exists in the Jumbo dir. The baseline file content uses the - // default naming because the fact was never delivered to it. - const jumboFile = jumboFinal.workspaceSnapshot!.files - .find((f) => f.path === 'src/module.ts')!; - const baselineFile = baselineFinal.workspaceSnapshot!.files - .find((f) => f.path === 'src/module.ts')!; - expect(jumboFile.content).toContain(PRIOR_SESSION_FACT); - expect(baselineFile.content).not.toContain(PRIOR_SESSION_FACT); - expect(baselineFile.content).toContain('itemCount'); - }); - - it('invalidates the comparison when the baseline harness fails (no partial result persisted)', async () => { - const scenario = createScenario({ sessionCount: 1 }); - const executor = new FakeJumboExecutor({ - failHarnessOnSession: { variant: 'baseline', sessionNumber: 1 }, - }); - const adapter = createFakeAdapter(); - const store = createInMemoryStore(); - - await expect( - runABComparison({ scenario, adapter, executor, store }), - ).rejects.toThrow(HarnessExecutionError); - - // Invariant: a partial result (only one variant) must not be persisted - // and must not be aggregated into a ComparisonResult. - expect(store.results).toHaveLength(0); - }); - - it('emits audit trails that explain measured deltas via context, snapshot, and scoring evidence', async () => { - const scenario = createScenario(); - const executor = new FakeJumboExecutor(); - const adapter = createFakeAdapter(); - const store = createInMemoryStore(); - - const result = await runABComparison({ scenario, adapter, executor, store }); - - const report = generateFullReport([result], scenario.disruptions ?? []); - expect(report.auditTrails).toHaveLength(1); - - const trail = report.auditTrails[0]; - - // Effective context audit: the Jumbo final session's effective prompt - // carries the lifecycle protocol and active goal-id (the prompt-prefix - // injection path is gone; prior memory is delivered to the agent via - // its own `jumbo session start` call, not via framework wrapping). - const jumboLast = trail.jumboContext.at(-1)!; - expect(jumboLast.effectivePrompt).toContain('Jumbo lifecycle protocol'); - - // Final snapshot evidence: the module file content reflecting the fact - // is captured for the Jumbo side; the baseline content is not. - const jumboFile = trail.jumboFinalSnapshot!.files.find((f) => f.path === 'src/module.ts')!; - const baselineFile = trail.baselineFinalSnapshot!.files.find((f) => f.path === 'src/module.ts')!; - expect(jumboFile.content).toContain(PRIOR_SESSION_FACT); - expect(baselineFile.content).not.toContain(PRIOR_SESSION_FACT); - - // Scoring evidence: each dimension's details and computed delta are present. - const retentionEvidence = trail.scoringEvidence.find((e) => e.dimension === 'knowledge-retention'); - expect(retentionEvidence).toBeDefined(); - expect(retentionEvidence!.delta).toBeGreaterThan(0); - expect(retentionEvidence!.jumboDetails).toBeDefined(); - expect(retentionEvidence!.baselineDetails).toBeDefined(); - }); - -}); diff --git a/evals/tests/unit/file-accuracy-scorer.test.ts b/evals/tests/unit/file-accuracy-scorer.test.ts deleted file mode 100644 index 5ef13dc2..00000000 --- a/evals/tests/unit/file-accuracy-scorer.test.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { scoreFileAccuracy, producedAllExpectedFiles } from '../../src/scoring/file-accuracy-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; -import type { WorkspaceSnapshot } from '../../src/domain/types.js'; - -function makeRecord(filesModified: string[], workspaceSnapshot?: WorkspaceSnapshot): ReturnType<typeof createSessionRecord> { - return createSessionRecord({ - id: 'rec-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: '', - filesModified, - transcript: '', - workspaceSnapshot, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -function makeSnapshot(paths: string[]): WorkspaceSnapshot { - return { - capturedAt: '2026-03-21T10:05:00Z', - files: paths.map((path) => ({ path, content: '' })), - }; -} - -describe('producedAllExpectedFiles', () => { - it('is true when every expected file was produced (recall = 1), ignoring extras', () => { - const record = makeRecord(['src/index.ts', 'src/utils.ts', 'src/extra.ts']); - expect(producedAllExpectedFiles([record], ['src/index.ts', 'src/utils.ts'])).toBe(true); - }); - - it('is false when any expected file is missing', () => { - const record = makeRecord(['src/index.ts']); - expect(producedAllExpectedFiles([record], ['src/index.ts', 'src/utils.ts'])).toBe(false); - }); - - it('is vacuously true when no files are expected', () => { - expect(producedAllExpectedFiles([makeRecord([])], [])).toBe(true); - }); - - it('falls back to workspace snapshot paths when filesModified is empty', () => { - const record = makeRecord([], makeSnapshot(['src/index.ts', 'src/utils.ts'])); - expect(producedAllExpectedFiles([record], ['src/index.ts', 'src/utils.ts'])).toBe(true); - }); -}); - -describe('scoreFileAccuracy', () => { - it('returns perfect score when all expected files are modified and nothing extra', () => { - const record = makeRecord(['src/index.ts', 'src/utils.ts']); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - expect(score.dimension).toBe('file-accuracy'); - expect(score.score).toBe(1); - expect(score.maxScore).toBe(1); - }); - - it('returns zero when no expected files are modified', () => { - const record = makeRecord(['src/other.ts']); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - expect(score.score).toBe(0); - }); - - it('handles partial match', () => { - const record = makeRecord(['src/index.ts']); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - // precision=1 (1/1), recall=0.5 (1/2), f1=0.67 - expect(score.score).toBeCloseTo(0.67, 1); - expect(score.details).toContain('1/2 expected files modified'); - expect(score.details).toContain('missed: src/utils.ts'); - }); - - it('penalizes unexpected files via precision', () => { - const record = makeRecord(['src/index.ts', 'src/utils.ts', 'src/junk.ts']); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - // precision=2/3, recall=1, f1=0.8 - expect(score.score).toBeCloseTo(0.8, 1); - expect(score.details).toContain('unexpected: src/junk.ts'); - }); - - it('returns trivial score when no expected files defined', () => { - const record = makeRecord(['anything.ts']); - const score = scoreFileAccuracy([record], []); - - expect(score.score).toBe(1); - expect(score.details).toContain('trivially satisfied'); - }); - - it('aggregates files across multiple session records', () => { - const rec1 = makeRecord(['src/index.ts']); - const rec2 = createSessionRecord({ - id: 'rec-2', - scenarioId: 'scenario-1', - sessionNumber: 2, - harness: 'claude-code', - agentOutput: '', - filesModified: ['src/utils.ts'], - transcript: '', - startedAt: '2026-03-21T10:05:00Z', - completedAt: '2026-03-21T10:10:00Z', - }); - - const score = scoreFileAccuracy([rec1, rec2], ['src/index.ts', 'src/utils.ts']); - expect(score.score).toBe(1); - }); - - it('returns zero for empty session records and non-empty expected files', () => { - const score = scoreFileAccuracy([], ['src/index.ts']); - - expect(score.score).toBe(0); - expect(score.details).toContain('missed: src/index.ts'); - }); - - it('uses workspace snapshot file paths when harness metadata is missing', () => { - // filesModified is empty (harness did not report it), but workspace snapshot has the files - const record = makeRecord([], makeSnapshot(['src/index.ts', 'src/utils.ts'])); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - expect(score.score).toBe(1); - expect(score.details).toContain('2/2 expected files modified'); - }); - - it('prefers harness filesModified over workspace snapshot when both present', () => { - // Harness says only index.ts was modified; snapshot has both — should trust harness - const record = makeRecord(['src/index.ts'], makeSnapshot(['src/index.ts', 'src/utils.ts'])); - const score = scoreFileAccuracy([record], ['src/index.ts', 'src/utils.ts']); - - // precision=1 (1/1), recall=0.5 (1/2), f1≈0.67 — harness data wins - expect(score.score).toBeCloseTo(0.67, 1); - expect(score.details).toContain('missed: src/utils.ts'); - }); - - it('returns zero with empty filesModified and no workspace snapshot', () => { - const record = makeRecord([]); - const score = scoreFileAccuracy([record], ['src/index.ts']); - - expect(score.score).toBe(0); - expect(score.details).toContain('missed: src/index.ts'); - }); -}); diff --git a/evals/tests/unit/gemini-cli-adapter.test.ts b/evals/tests/unit/gemini-cli-adapter.test.ts deleted file mode 100644 index 3cd5d4af..00000000 --- a/evals/tests/unit/gemini-cli-adapter.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { GeminiCliAdapter } from '../../src/harness/gemini-cli-adapter.js'; - -describe('GeminiCliAdapter', () => { - const adapter = new GeminiCliAdapter(); - - it('has the correct name', () => { - expect(adapter.name).toBe('gemini-cli'); - }); - - it('builds a gemini command without a positional prompt (prompt goes via stdin)', () => { - const command = adapter.buildCommand(); - expect(command).toEqual(['gemini', '--json']); - }); - - it('parses JSON output with text field', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - text: 'Created index.ts with hello world', - files_modified: ['index.ts', 'package.json'], - }), - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Created index.ts with hello world'); - expect(result.filesModified).toEqual(['index.ts', 'package.json']); - }); - - it('handles non-JSON output gracefully', () => { - const result = adapter.parseOutput({ - stdout: 'Plain text response', - stderr: '', - exitCode: 0, - }); - - expect(result.agentOutput).toBe('Plain text response'); - expect(result.filesModified).toEqual([]); - }); - - it('extracts token counts from usage_metadata', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - text: 'Done', - usage_metadata: { prompt_token_count: 1800, candidates_token_count: 900 }, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(1800); - expect(result.outputTokens).toBe(900); - }); - - it('extracts token counts from top-level fields', () => { - const result = adapter.parseOutput({ - stdout: JSON.stringify({ - text: 'Done', - prompt_token_count: 1500, - candidates_token_count: 700, - }), - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBe(1500); - expect(result.outputTokens).toBe(700); - }); - - it('returns undefined tokens for non-JSON output', () => { - const result = adapter.parseOutput({ - stdout: 'text', - stderr: '', - exitCode: 0, - }); - - expect(result.inputTokens).toBeUndefined(); - expect(result.outputTokens).toBeUndefined(); - }); - - it('includes stderr in transcript', () => { - const result = adapter.parseOutput({ - stdout: 'output', - stderr: 'warning', - exitCode: 0, - }); - - expect(result.transcript).toContain('output'); - expect(result.transcript).toContain('warning'); - expect(result.transcript).toContain('---stderr---'); - }); - - it('does not embed any prompt in argv (prompt is delivered via stdin)', () => { - const command = adapter.buildCommand(); - expect(command).not.toContain('Build a REST API'); - expect(command).toEqual(['gemini', '--json']); - }); - - describe('seedToolPermissions', () => { - let workDir: string; - - beforeEach(async () => { - workDir = await mkdtemp(join(tmpdir(), 'gemini-adapter-seed-')); - }); - - afterEach(async () => { - await rm(workDir, { recursive: true, force: true }); - }); - - it('writes .gemini/settings.json allowlisting run_shell_command(jumbo)', async () => { - await adapter.seedToolPermissions(workDir); - const raw = await readFile(join(workDir, '.gemini', 'settings.json'), 'utf-8'); - const settings = JSON.parse(raw); - - expect(settings.tools.allowed).toContain('run_shell_command(jumbo)'); - expect(settings.general.defaultApprovalMode).toBe('auto_edit'); - }); - - it('does not set tools.core (which would disable other built-in tools)', async () => { - await adapter.seedToolPermissions(workDir); - const settings = JSON.parse(await readFile(join(workDir, '.gemini', 'settings.json'), 'utf-8')); - expect(settings.tools.core).toBeUndefined(); - }); - - it('is idempotent: re-seeding overwrites without error', async () => { - await adapter.seedToolPermissions(workDir); - await adapter.seedToolPermissions(workDir); - const settings = JSON.parse(await readFile(join(workDir, '.gemini', 'settings.json'), 'utf-8')); - expect(settings.tools.allowed).toContain('run_shell_command(jumbo)'); - }); - }); -}); diff --git a/evals/tests/unit/heartbeat.test.ts b/evals/tests/unit/heartbeat.test.ts deleted file mode 100644 index 0e980351..00000000 --- a/evals/tests/unit/heartbeat.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { describe, it, expect, jest } from '@jest/globals'; -import { runABComparison } from '../../src/ab-runner.js'; -import { formatHeartbeatDisplay } from '../../src/output/heartbeat-display.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { HeartbeatWriter } from '../../src/infrastructure/heartbeat-writer.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { EvalRunRecord, RunHeartbeat, SessionRecord, TestResult } from '../../src/domain/types.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; - -describe('heartbeat emission', () => { - it('emits phase transitions for jumbo and baseline variants', async () => { - const writer = createWriter(); - await runABComparison({ - scenario: createTestScenario({ id: 'scenario-heartbeat', name: 'Heartbeat', initialPrompt: 'Build', sessionCount: 1 }), - adapter: createAdapter(), - executor: createExecutor(), - store: createStore(), - runId: 'run-heartbeat', - heartbeatWriter: writer, - }); - - const transitions = writer.heartbeats.flatMap((heartbeat) => - heartbeat.harnesses.flatMap((harness) => - harness.sessions.map((session) => `${harness.variant}:${session.status}:${session.phase ?? 'none'}`), - ), - ); - - expect(transitions).toEqual(expect.arrayContaining([ - 'jumbo:running:harness-exec', - 'jumbo:running:lifecycle-audit', - 'jumbo:completed:none', - 'baseline:running:harness-exec', - 'baseline:completed:none', - ])); - }); - - it('never reads heartbeat state inside runABComparison', async () => { - const store = createStore(); - const readHeartbeat = jest.spyOn(store, 'readHeartbeat'); - - await runABComparison({ - scenario: createTestScenario({ id: 'scenario-read', name: 'No reads', initialPrompt: 'Build', sessionCount: 1 }), - adapter: createAdapter(), - executor: createExecutor(), - store, - }); - - expect(readHeartbeat).not.toHaveBeenCalled(); - }); - - it('does not change persisted artifacts except nondeterministic timing fields', async () => { - const scenario = createTestScenario({ id: 'scenario-equal', name: 'Equal', initialPrompt: 'Build', sessionCount: 1 }); - const disabledStore = createStore(); - const enabledStore = createStore(); - - await runABComparison({ - scenario, - adapter: createAdapter(), - executor: createExecutor(), - store: disabledStore, - runId: 'run-disabled', - }); - await runABComparison({ - scenario, - adapter: createAdapter(), - executor: createExecutor(), - store: enabledStore, - runId: 'run-enabled', - heartbeatWriter: createWriter(), - }); - - expect(normalizePersistedArtifacts(enabledStore)).toEqual(normalizePersistedArtifacts(disabledStore)); - }); - - it('marks both variants failed when scoring persistence fails', async () => { - const writer = createWriter(); - - await expect(runABComparison({ - scenario: createTestScenario({ id: 'scenario-score-fail', name: 'Score fail', initialPrompt: 'Build', sessionCount: 1 }), - adapter: createAdapter(), - executor: createExecutor(), - store: createStore({ failSaveTestResult: true }), - runId: 'run-score-fail', - heartbeatWriter: writer, - })).rejects.toThrow('save failed'); - - const failedScoring = writer.heartbeats - .flatMap((heartbeat) => heartbeat.harnesses) - .flatMap((harness) => harness.sessions.map((session) => `${harness.variant}:${session.status}:${session.phase}:${session.errorMessage}`)) - .filter((transition) => transition.includes(':failed:scoring:')); - expect(failedScoring).toEqual(expect.arrayContaining([ - 'jumbo:failed:scoring:save failed', - 'baseline:failed:scoring:save failed', - ])); - }); - - it('renders per-session heartbeat rows', () => { - const output = formatHeartbeatDisplay({ - runId: 'run-display', - scenarioId: 'scenario-display', - updatedAt: '2026-04-29T10:00:00.000Z', - harnesses: [{ - harness: 'claude-code', - variant: 'jumbo', - sessions: [ - { sessionNumber: 1, status: 'completed', startedAt: '2026-04-29T10:00:00.000Z', completedAt: '2026-04-29T10:00:01.000Z' }, - { sessionNumber: 2, status: 'running', phase: 'harness-exec', startedAt: new Date().toISOString() }, - { sessionNumber: 3, status: 'pending' }, - ], - }], - }); - - expect(output).toContain('Sessions'); - expect(output).toContain('claude-code'); - expect(output).toContain('jumbo'); - expect(output).toContain('completed'); - expect(output).toContain('running'); - expect(output).toContain('harness-exec'); - expect(output).toContain('pending'); - }); -}); - -function createWriter(): HeartbeatWriter & { heartbeats: RunHeartbeat[] } { - return { - heartbeats: [], - async writeHeartbeat(_runId: string, heartbeat: RunHeartbeat): Promise<void> { - this.heartbeats.push(heartbeat); - }, - }; -} - -function normalizePersistedArtifacts(store: RecordingStore): unknown { - return { - sessions: store.sessionRecords.map(normalizeSession), - testResults: store.testResults.map(normalizeTestResult), - }; -} - -function normalizeTestResult(result: TestResult): unknown { - return { - ...result, - id: '<id>', - createdAt: '<time>', - sessionRecords: result.sessionRecords.map(normalizeSession), - }; -} - -function normalizeSession(record: SessionRecord): unknown { - return { - ...record, - id: '<id>', - startedAt: '<time>', - completedAt: '<time>', - workspaceSnapshot: record.workspaceSnapshot ? { ...record.workspaceSnapshot, capturedAt: '<time>' } : undefined, - jumboMemorySnapshot: record.jumboMemorySnapshot ? { ...record.jumboMemorySnapshot, capturedAt: '<time>' } : undefined, - jumboMemorySnapshotBefore: record.jumboMemorySnapshotBefore ? { ...record.jumboMemorySnapshotBefore, capturedAt: '<time>' } : undefined, - phaseTimings: '<timing>', - }; -} - -function createExecutor(): LocalExecutor & { execCalls: Array<{ command: string[] }> } { - let dir = 0; - const execCalls: Array<{ command: string[] }> = []; - return { - execCalls, - createWorkDir: async (prefix?: string) => `/tmp/${prefix ?? 'eval'}${++dir}`, - installJumboShim: async (workDir: string) => ({ env: { PATH: `${workDir}/.eval-bin:/usr/bin` } }), - exec: async (_workDir: string, command: string[], options?: { env?: Record<string, string | undefined> }) => { - execCalls.push({ command }); - if (command[0] === 'jumbo' && command[1] === '--version') { - const shimmed = options?.env?.PATH?.includes('.eval-bin') ?? false; - return shimmed - ? { stdout: '', stderr: 'shim', exitCode: 127 } - : { stdout: 'jumbo 1.2.3', stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'show') { - return { stdout: JSON.stringify({ goal: { status: 'submitted' } }), stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[1] === 'sessions' && command[2] === 'list') { - return { stdout: '[]', stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[3] === '--format') { - return { stdout: '[]', stderr: '', exitCode: 0 }; - } - if (command[0] === 'mock') { - return { stdout: '{"result":"ok","files_modified":[]}', stderr: '', exitCode: 0 }; - } - return { stdout: 'ok', stderr: '', exitCode: 0 }; - }, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - } as LocalExecutor; -} - -function createAdapter(): HarnessAdapter { - return { - name: 'mock', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ agentOutput: result.stdout, filesModified: [], transcript: result.stdout }), - seedToolPermissions: async () => {}, - }; -} - -function createStore(options?: { failSaveTestResult?: boolean }): RecordingStore { - return new RecordingStore(options); -} - -class RecordingStore implements ResultStore { - readonly sessionRecords: SessionRecord[] = []; - readonly testResults: TestResult[] = []; - - constructor(private readonly options?: { failSaveTestResult?: boolean }) {} - - async saveScenario(): Promise<void> {} - async getScenario(): Promise<null> { return null; } - async listScenarios(): Promise<[]> { return []; } - - async saveSessionRecord(record: SessionRecord): Promise<void> { - const index = this.sessionRecords.findIndex((existing) => existing.id === record.id); - if (index === -1) { - this.sessionRecords.push(record); - } else { - this.sessionRecords[index] = record; - } - } - - async getSessionRecords(): Promise<SessionRecord[]> { - return [...this.sessionRecords]; - } - - async saveTestResult(result: TestResult): Promise<void> { - if (this.options?.failSaveTestResult) { - throw new Error('save failed'); - } - this.testResults.push(result); - } - - async getTestResult(): Promise<null> { return null; } - async listTestResults(): Promise<TestResult[]> { return [...this.testResults]; } - async saveRunRecord(_record: EvalRunRecord): Promise<void> {} - async getRunRecord(): Promise<null> { return null; } - async listRunRecords(): Promise<[]> { return []; } - async writeHeartbeat(_runId: string, _heartbeat: RunHeartbeat): Promise<void> {} - async readHeartbeat(): Promise<null> { return null; } -} diff --git a/evals/tests/unit/json-report.test.ts b/evals/tests/unit/json-report.test.ts deleted file mode 100644 index f34f1868..00000000 --- a/evals/tests/unit/json-report.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { exportReportAsJson, parseJsonReport } from '../../src/output/json-report.js'; -import { generateFullReport } from '../../src/output/report-generator.js'; -import { createComparisonResult, createTestResult, createSessionRecord } from '../../src/domain/types.js'; - -function makeReport() { - const record = createSessionRecord({ - id: 'rec-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: 'output', - filesModified: ['src/index.ts'], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - const result = createTestResult({ id: 'r1', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [record] }); - - const comp = createComparisonResult({ - id: 'comp-1', - scenarioId: 'scenario-1', - harness: 'claude-code', - jumboResult: result, - baselineResult: result, - jumboScores: [{ dimension: 'file-accuracy', score: 0.9, maxScore: 1 }], - baselineScores: [{ dimension: 'file-accuracy', score: 0.6, maxScore: 1 }], - deltas: [{ dimension: 'file-accuracy', score: 0.3, maxScore: 1 }], - }); - - return generateFullReport([comp]); -} - -describe('exportReportAsJson', () => { - it('produces valid JSON with correct structure', () => { - const report = makeReport(); - const json = exportReportAsJson(report); - const parsed = JSON.parse(json); - - expect(parsed.meta.format).toBe('jumbo-eval-report'); - expect(parsed.meta.version).toBe(1); - expect(parsed.meta.scenarioId).toBe('scenario-1'); - }); - - it('includes lift data', () => { - const report = makeReport(); - const json = exportReportAsJson(report); - const parsed = JSON.parse(json); - - expect(parsed.lift.byDimension).toHaveLength(1); - expect(parsed.lift.byDimension[0].dimension).toBe('file-accuracy'); - expect(parsed.lift.byDimension[0].absoluteLift).toBeCloseTo(0.3, 2); - }); - - it('includes divergence data', () => { - const report = makeReport(); - const json = exportReportAsJson(report); - const parsed = JSON.parse(json); - - expect(parsed.divergence).toBeDefined(); - expect(parsed.divergence.onsets).toBeDefined(); - }); - - it('includes harness comparison data', () => { - const report = makeReport(); - const json = exportReportAsJson(report); - const parsed = JSON.parse(json); - - expect(parsed.harnessComparison).toHaveLength(1); - expect(parsed.harnessComparison[0].harness).toBe('claude-code'); - }); -}); - -describe('parseJsonReport', () => { - it('round-trips through export and parse', () => { - const report = makeReport(); - const json = exportReportAsJson(report); - const parsed = parseJsonReport(json); - - expect(parsed.meta.format).toBe('jumbo-eval-report'); - expect(parsed.meta.version).toBe(1); - }); - - it('throws on invalid format', () => { - expect(() => parseJsonReport('{}')).toThrow('Invalid report format'); - }); - - it('throws on missing version', () => { - const bad = JSON.stringify({ meta: { format: 'jumbo-eval-report' } }); - expect(() => parseJsonReport(bad)).toThrow('missing meta.version'); - }); - - it('throws on invalid JSON', () => { - expect(() => parseJsonReport('not json')).toThrow(); - }); -}); diff --git a/evals/tests/unit/json-result-store.test.ts b/evals/tests/unit/json-result-store.test.ts deleted file mode 100644 index 680a041c..00000000 --- a/evals/tests/unit/json-result-store.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; -import * as os from 'node:os'; -import { JsonResultStore } from '../../src/storage/json-result-store.js'; -import { createTestScenario, createSessionRecord, createTestResult } from '../../src/domain/types.js'; -import type { ReplicationReport } from '../../src/domain/types.js'; - -describe('JsonResultStore', () => { - let tmpDir: string; - let store: JsonResultStore; - - beforeEach(async () => { - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'jumbo-evals-test-')); - store = new JsonResultStore(tmpDir); - await store.initialize(); - }); - - afterEach(async () => { - await fs.rm(tmpDir, { recursive: true, force: true }); - }); - - describe('scenarios', () => { - it('saves and retrieves a scenario', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Test scenario', - initialPrompt: 'Build something', - sessionCount: 3, - }); - - await store.saveScenario(scenario); - const retrieved = await store.getScenario('scenario-1'); - - expect(retrieved).not.toBeNull(); - expect(retrieved!.id).toBe('scenario-1'); - expect(retrieved!.name).toBe('Test scenario'); - expect(retrieved!.sessionCount).toBe(3); - }); - - it('returns null for non-existent scenario', async () => { - const result = await store.getScenario('does-not-exist'); - expect(result).toBeNull(); - }); - - it('lists all scenarios', async () => { - await store.saveScenario(createTestScenario({ id: 's1', name: 'A', initialPrompt: 'p1', sessionCount: 1 })); - await store.saveScenario(createTestScenario({ id: 's2', name: 'B', initialPrompt: 'p2', sessionCount: 2 })); - - const scenarios = await store.listScenarios(); - expect(scenarios).toHaveLength(2); - }); - }); - - describe('session records', () => { - it('saves and retrieves session records by scenario', async () => { - const record = createSessionRecord({ - id: 'rec-1', - scenarioId: 'scenario-1', - sessionNumber: 1, - harness: 'claude-code', - agentOutput: 'output', - filesModified: ['file.ts'], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); - - await store.saveSessionRecord(record); - const records = await store.getSessionRecords('scenario-1'); - - expect(records).toHaveLength(1); - expect(records[0].id).toBe('rec-1'); - expect(records[0].filesModified).toEqual(['file.ts']); - }); - - it('filters session records by scenarioId', async () => { - await store.saveSessionRecord(createSessionRecord({ - id: 'rec-1', scenarioId: 'scenario-1', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', startedAt: '', completedAt: '', - })); - await store.saveSessionRecord(createSessionRecord({ - id: 'rec-2', scenarioId: 'scenario-2', sessionNumber: 1, harness: 'claude-code', - agentOutput: '', filesModified: [], transcript: '', startedAt: '', completedAt: '', - })); - - const records = await store.getSessionRecords('scenario-1'); - expect(records).toHaveLength(1); - expect(records[0].scenarioId).toBe('scenario-1'); - }); - }); - - describe('test results', () => { - it('saves and retrieves a test result', async () => { - const result = createTestResult({ - id: 'result-1', - scenarioId: 'scenario-1', - harness: 'claude-code', - sessionRecords: [], - }); - - await store.saveTestResult(result); - const retrieved = await store.getTestResult('result-1'); - - expect(retrieved).not.toBeNull(); - expect(retrieved!.id).toBe('result-1'); - }); - - it('returns null for non-existent result', async () => { - const result = await store.getTestResult('nope'); - expect(result).toBeNull(); - }); - - it('lists results filtered by scenario', async () => { - await store.saveTestResult(createTestResult({ - id: 'r1', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], - })); - await store.saveTestResult(createTestResult({ - id: 'r2', scenarioId: 'scenario-2', harness: 'claude-code', sessionRecords: [], - })); - - const results = await store.listTestResults('scenario-1'); - expect(results).toHaveLength(1); - expect(results[0].scenarioId).toBe('scenario-1'); - }); - - it('lists all results when no filter', async () => { - await store.saveTestResult(createTestResult({ - id: 'r1', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], - })); - await store.saveTestResult(createTestResult({ - id: 'r2', scenarioId: 'scenario-2', harness: 'claude-code', sessionRecords: [], - })); - - const results = await store.listTestResults(); - expect(results).toHaveLength(2); - }); - }); - - describe('replication reports', () => { - const report: ReplicationReport = { - scenarioId: 's1', - harness: 'claude-code', - k: 5, - dimensions: [ - { - dimension: 'file-accuracy', - k: 5, - applicableReplications: 5, - meanJumbo: 0.9, - meanBaseline: 0.5, - meanLift: 0.4, - sdLift: 0.2, - tStatistic: 4.47, - isSignal: true, - }, - ], - significance: { rule: 'isSignal = |meanLift| > sdLift', tCriticalOneTailed05: 2.132, note: 'note' }, - createdAt: '2026-03-21T10:05:00Z', - }; - - it('persists and retrieves a replication report by runId (survives a fresh store instance)', async () => { - await store.saveReplicationReport('run-123', report); - const reopened = new JsonResultStore(tmpDir); - expect(await reopened.getReplicationReport('run-123')).toEqual(report); - }); - - it('returns null when no replication report exists for the runId', async () => { - expect(await store.getReplicationReport('absent')).toBeNull(); - }); - }); -}); diff --git a/evals/tests/unit/jumbo-event-capture-scorer.test.ts b/evals/tests/unit/jumbo-event-capture-scorer.test.ts deleted file mode 100644 index 4b2e7224..00000000 --- a/evals/tests/unit/jumbo-event-capture-scorer.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { - scoreJumboEventCapture, - baselineJumboEventCaptureScore, - scoreJumboEventCaptureTimeline, - resolveAddedEventTypeForKind, - addedEventTypeByKind, -} from '../../src/scoring/jumbo-event-capture-scorer.js'; -import { createSessionRecord } from '../../src/domain/index.js'; -import type { ExpectedJumboMemoryCapture, SessionRecord } from '../../src/domain/index.js'; - -function record(sessionNumber: number, countsByType?: Record<string, number>): SessionRecord { - const files: { path: string; content: string }[] = []; - return createSessionRecord({ - id: `r${sessionNumber}`, - scenarioId: 's', - sessionNumber, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - workspaceSnapshot: { - capturedAt: '2026-03-21T10:05:00Z', - files, - ...(countsByType - ? { - jumboEvents: { - capturedAt: '2026-03-21T10:05:00Z', - aggregateCount: Object.keys(countsByType).length, - eventCount: Object.values(countsByType).reduce((a, b) => a + b, 0), - countsByType, - fileNames: [], - }, - } - : {}), - }, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -const expected: ExpectedJumboMemoryCapture[] = [ - { kind: 'decision', match: 'event sourcing' }, - { kind: 'component', match: 'EventStore' }, -]; - -describe('scoreJumboEventCapture', () => { - it('scores 1.0 when every expected kind is evidenced in the latest event log', () => { - const records = [record(1, { DecisionAddedEvent: 1, ComponentAddedEvent: 2, WorkerIdentifiedEvent: 3 })]; - expect(scoreJumboEventCapture(records, expected).score).toBe(1); - }); - - it('scores partially when only some expected kinds are evidenced', () => { - const records = [record(1, { DecisionAddedEvent: 1 })]; - const score = scoreJumboEventCapture(records, expected); - expect(score.score).toBe(0.5); - expect(score.details).toContain('ComponentAddedEvent'); - }); - - it('scores 0 when the latest snapshot has no jumboEvents', () => { - const records = [record(1)]; // workspace snapshot but no .jumbo/events - expect(scoreJumboEventCapture(records, expected).score).toBe(0); - }); - - it('uses the latest session that carries an event log', () => { - const records = [ - record(1, { DecisionAddedEvent: 1 }), - record(2, { DecisionAddedEvent: 1, ComponentAddedEvent: 1 }), - ]; - expect(scoreJumboEventCapture(records, expected).score).toBe(1); - }); - - it('is trivially satisfied when no captures are expected', () => { - const records = [record(1)]; - expect(scoreJumboEventCapture(records, []).score).toBe(1); - }); - - it('counts distinct kinds, not individual expectations', () => { - const twoDecisions: ExpectedJumboMemoryCapture[] = [ - { kind: 'decision', match: 'a' }, - { kind: 'decision', match: 'b' }, - ]; - const records = [record(1, { DecisionAddedEvent: 1 })]; - expect(scoreJumboEventCapture(records, twoDecisions).score).toBe(1); - }); -}); - -describe('resolveAddedEventTypeForKind', () => { - it('maps every memory kind to its explicit Added-event type', () => { - expect(resolveAddedEventTypeForKind('decision')).toBe('DecisionAddedEvent'); - expect(resolveAddedEventTypeForKind('guideline')).toBe('GuidelineAddedEvent'); - expect(resolveAddedEventTypeForKind('invariant')).toBe('InvariantAddedEvent'); - expect(resolveAddedEventTypeForKind('component')).toBe('ComponentAddedEvent'); - expect(resolveAddedEventTypeForKind('relation')).toBe('RelationAddedEvent'); - expect(resolveAddedEventTypeForKind('dependency')).toBe('DependencyAddedEvent'); - }); - - it('covers exactly the six memory kinds (registry is exhaustive)', () => { - expect(Object.keys(addedEventTypeByKind).sort()).toEqual( - ['component', 'decision', 'dependency', 'guideline', 'invariant', 'relation'], - ); - }); -}); - -describe('baselineJumboEventCaptureScore', () => { - it('is a trivial zero for the baseline arm (no .jumbo/events)', () => { - const score = baselineJumboEventCaptureScore(expected); - expect(score.dimension).toBe('jumbo-event-capture'); - expect(score.score).toBe(0); - }); -}); - -describe('scoreJumboEventCaptureTimeline', () => { - it('produces one score per session, cumulative to that session', () => { - const records = [ - record(1, { DecisionAddedEvent: 1 }), - record(2, { DecisionAddedEvent: 1, ComponentAddedEvent: 1 }), - ]; - const timeline = scoreJumboEventCaptureTimeline(records, expected); - expect(timeline).toHaveLength(2); - expect(timeline[0].score).toBe(0.5); - expect(timeline[1].score).toBe(1); - }); -}); diff --git a/evals/tests/unit/jumbo-event-summary.test.ts b/evals/tests/unit/jumbo-event-summary.test.ts deleted file mode 100644 index 8a2f3540..00000000 --- a/evals/tests/unit/jumbo-event-summary.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { summarizeJumboEvents } from '../../src/infrastructure/local-executor.js'; - -const TS = '2026-03-21T10:05:00Z'; - -describe('summarizeJumboEvents', () => { - it('returns an empty summary for no event files', () => { - const summary = summarizeJumboEvents([], TS); - expect(summary.capturedAt).toBe(TS); - expect(summary.aggregateCount).toBe(0); - expect(summary.eventCount).toBe(0); - expect(summary.countsByType).toEqual({}); - expect(summary.fileNames).toEqual([]); - }); - - it('counts events and aggregates from a typical layout', () => { - const paths = [ - 'project/000001.ProjectInitializedEvent.json', - 'cd3a2dc3-e242-4165-ad51-5587fd3adc11/000001.DecisionAddedEvent.json', - 'd2f936e3-25f7-4784-81d2-65f1da30be07/000001.ComponentAddedEvent.json', - 'c3cb11f3-754e-4346-800f-11c64d9b143b/000001.WorkerIdentifiedEvent.json', - ]; - const summary = summarizeJumboEvents(paths, TS); - expect(summary.eventCount).toBe(4); - expect(summary.aggregateCount).toBe(4); - expect(summary.countsByType).toEqual({ - ProjectInitializedEvent: 1, - DecisionAddedEvent: 1, - ComponentAddedEvent: 1, - WorkerIdentifiedEvent: 1, - }); - }); - - it('aggregates type counts across multiple events of the same type', () => { - const paths = [ - 'a/000001.DecisionAddedEvent.json', - 'b/000001.DecisionAddedEvent.json', - 'a/000002.DecisionAddedEvent.json', - ]; - const summary = summarizeJumboEvents(paths, TS); - expect(summary.countsByType.DecisionAddedEvent).toBe(3); - expect(summary.eventCount).toBe(3); - expect(summary.aggregateCount).toBe(2); // a and b - }); - - it('parses the event type by stripping the sequence prefix and .json suffix', () => { - const summary = summarizeJumboEvents(['agg/000007.InvariantAddedEvent.json'], TS); - expect(summary.countsByType).toEqual({ InvariantAddedEvent: 1 }); - }); - - it('buckets unparseable filenames under "unknown"', () => { - const summary = summarizeJumboEvents(['agg/snapshot.bin', 'agg/000001.DecisionAddedEvent.json'], TS); - expect(summary.countsByType.unknown).toBe(1); - expect(summary.countsByType.DecisionAddedEvent).toBe(1); - }); - - it('returns fileNames sorted for deterministic evidence', () => { - const paths = [ - 'z/000001.DecisionAddedEvent.json', - 'a/000001.ComponentAddedEvent.json', - 'm/000001.InvariantAddedEvent.json', - ]; - const summary = summarizeJumboEvents(paths, TS); - expect(summary.fileNames).toEqual([ - 'a/000001.ComponentAddedEvent.json', - 'm/000001.InvariantAddedEvent.json', - 'z/000001.DecisionAddedEvent.json', - ]); - }); -}); diff --git a/evals/tests/unit/jumbo-memory-capture-scorer.test.ts b/evals/tests/unit/jumbo-memory-capture-scorer.test.ts deleted file mode 100644 index 867d43f3..00000000 --- a/evals/tests/unit/jumbo-memory-capture-scorer.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { describe, expect, it } from '@jest/globals'; -import { baselineJumboMemoryCaptureScore, scoreJumboMemoryCapture } from '../../src/scoring/jumbo-memory-capture-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; -import type { JumboMemoryEntity, JumboMemorySnapshot, SessionRecord } from '../../src/domain/types.js'; - -function snap(sessionNumber: number, entities: JumboMemoryEntity[]): JumboMemorySnapshot { - return { - sessionNumber, - capturedAt: '2026-04-26T10:00:00Z', - entities, - commands: [], - }; -} - -function record( - sessionNumber: number, - before: JumboMemoryEntity[], - after: JumboMemoryEntity[], -): SessionRecord { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'scenario-memory', - sessionNumber, - harness: 'mock', - variant: 'jumbo', - agentOutput: 'done', - filesModified: [], - transcript: 'transcript', - jumboMemorySnapshotBefore: snap(sessionNumber, before), - jumboMemorySnapshot: snap(sessionNumber, after), - startedAt: '2026-04-26T10:00:00Z', - completedAt: '2026-04-26T10:01:00Z', - }); -} - -describe('scoreJumboMemoryCapture (snapshot-diff)', () => { - it('credits entities that appear only in the post-session snapshot', () => { - const score = scoreJumboMemoryCapture( - [ - record( - 1, - [], - [ - { kind: 'decision', id: 'dec-1', text: 'Commander for CLI framework', raw: {} }, - { kind: 'component', id: 'cmp-1', text: 'TaskStore persists tasks', raw: {} }, - ], - ), - ], - [ - { kind: 'decision', match: 'Commander for CLI' }, - { kind: 'component', match: 'TaskStore' }, - ], - ); - - expect(score.dimension).toBe('jumbo-memory-capture'); - expect(score.score).toBe(1); - expect(score.details).toContain('precision=1.00'); - expect(score.details).toContain('recall=1.00'); - expect(score.details).toContain('new-entities=2'); - expect(score.details).toContain('spurious: none'); - }); - - it('ignores entities already present in the pre-session snapshot (no harness-side mirroring)', () => { - // The agent did not register anything new this session — both decisions - // were already in the pre-snapshot (preSeededMemory or prior session). - const preExisting: JumboMemoryEntity[] = [ - { kind: 'decision', id: 'dec-1', text: 'Commander for CLI framework', raw: {} }, - ]; - const score = scoreJumboMemoryCapture( - [record(1, preExisting, preExisting)], - [{ kind: 'decision', match: 'Commander for CLI' }], - ); - - expect(score.score).toBe(0); - expect(score.details).toContain('new-entities=0'); - expect(score.details).toContain('missing: decision:Commander for CLI'); - }); - - it('flags missing and spurious memories among new entities only', () => { - const score = scoreJumboMemoryCapture( - [ - record( - 1, - [], - [ - { kind: 'decision', id: 'dec-1', text: 'Use ad hoc argument parsing', raw: {} }, - ], - ), - ], - [{ kind: 'decision', match: 'Commander for CLI' }], - ); - - expect(score.score).toBe(0); - expect(score.details).toContain('missing: decision:Commander for CLI'); - expect(score.details).toContain('spurious: decision:Use ad hoc argument parsing'); - }); - - it('matches expected captures with sessionNumber against the session window in which they appeared', () => { - const score = scoreJumboMemoryCapture( - [ - record(1, [], [{ kind: 'decision', id: 'd1', text: 'Adopt event sourcing', raw: {} }]), - record(2, - [{ kind: 'decision', id: 'd1', text: 'Adopt event sourcing', raw: {} }], - [ - { kind: 'decision', id: 'd1', text: 'Adopt event sourcing', raw: {} }, - { kind: 'invariant', id: 'i1', text: 'No mocks in integration tests', raw: {} }, - ], - ), - ], - [ - { kind: 'decision', match: 'event sourcing', sessionNumber: 1 }, - { kind: 'invariant', match: 'No mocks', sessionNumber: 2 }, - ], - ); - - expect(score.score).toBe(1); - expect(score.details).toContain('recall=1.00'); - }); - - it('does not penalize baseline runs for missing Jumbo memory', () => { - const score = baselineJumboMemoryCaptureScore([ - { kind: 'invariant', match: 'Use append-only history' }, - ]); - - expect(score.dimension).toBe('jumbo-memory-capture'); - expect(score.score).toBe(0); - expect(score.maxScore).toBe(0); - expect(score.details).toContain('baseline runs do not use Jumbo project memory'); - }); -}); diff --git a/evals/tests/unit/knowledge-retention-scorer.test.ts b/evals/tests/unit/knowledge-retention-scorer.test.ts deleted file mode 100644 index 46c61cb6..00000000 --- a/evals/tests/unit/knowledge-retention-scorer.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { scoreKnowledgeRetention, scoreKnowledgeRetentionTimeline } from '../../src/scoring/knowledge-retention-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; -import type { WorkspaceSnapshot } from '../../src/domain/types.js'; - -function makeRecord(sessionNumber: number, opts: { - agentOutput?: string; - filesModified?: string[]; - transcript?: string; - workspaceSnapshot?: WorkspaceSnapshot; -} = {}) { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'scenario-1', - sessionNumber, - harness: 'claude-code', - agentOutput: opts.agentOutput ?? '', - filesModified: opts.filesModified ?? [], - transcript: opts.transcript ?? '', - workspaceSnapshot: opts.workspaceSnapshot, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -function makeSnapshot(files: Record<string, string>): WorkspaceSnapshot { - return { - capturedAt: '2026-03-21T10:05:00Z', - files: Object.entries(files).map(([path, content]) => ({ path, content })), - }; -} - -describe('scoreKnowledgeRetention', () => { - it('returns perfect score when all patterns are found in latest session', () => { - const records = [ - makeRecord(1, { agentOutput: 'Using DependencyInversion and InterfaceSegregation' }), - makeRecord(2, { agentOutput: 'Applied DependencyInversion pattern, followed InterfaceSegregation' }), - ]; - - const score = scoreKnowledgeRetention(records, ['DependencyInversion', 'InterfaceSegregation']); - expect(score.score).toBe(1); - expect(score.dimension).toBe('knowledge-retention'); - }); - - it('returns zero when no patterns found in latest session', () => { - const records = [ - makeRecord(1, { agentOutput: 'Using DependencyInversion and InterfaceSegregation' }), - makeRecord(2, { agentOutput: 'Did something completely different' }), - ]; - - const score = scoreKnowledgeRetention(records, ['DependencyInversion', 'InterfaceSegregation']); - expect(score.score).toBe(0); - expect(score.details).toContain('lost: DependencyInversion, InterfaceSegregation'); - }); - - it('handles partial retention', () => { - const records = [ - makeRecord(1, { agentOutput: 'Pattern A and Pattern B' }), - makeRecord(3, { agentOutput: 'Still using Pattern A but forgot B' }), - ]; - - const score = scoreKnowledgeRetention(records, ['Pattern A', 'Pattern B']); - expect(score.score).toBe(0.5); - }); - - it('checks filesModified and transcript too', () => { - const records = [ - makeRecord(1, { filesModified: ['src/auth-service.ts'], transcript: 'Created auth service' }), - makeRecord(2, { filesModified: ['src/auth-service.ts'] }), - ]; - - const score = scoreKnowledgeRetention(records, ['auth-service']); - expect(score.score).toBe(1); - }); - - it('is case-insensitive', () => { - const records = [ - makeRecord(1, { agentOutput: 'SOLID principles' }), - makeRecord(2, { agentOutput: 'Applied solid Principles' }), - ]; - - const score = scoreKnowledgeRetention(records, ['SOLID']); - expect(score.score).toBe(1); - }); - - it('returns trivial score when no patterns defined', () => { - const score = scoreKnowledgeRetention([makeRecord(1)], []); - expect(score.score).toBe(1); - expect(score.details).toContain('trivially satisfied'); - }); - - it('uses workspace snapshot content as primary evidence (correct code, terse transcript)', () => { - // Transcript says nothing — but the file actually implements the pattern - const records = [ - makeRecord(1, { agentOutput: 'Setup done' }), - makeRecord(2, { - agentOutput: 'done', - transcript: '', - workspaceSnapshot: makeSnapshot({ - 'src/service.ts': 'export class AuthService { constructor() {} }', - }), - }), - ]; - - const score = scoreKnowledgeRetention(records, ['AuthService']); - expect(score.score).toBe(1); - }); - - it('rejects keyword-only transcript when workspace snapshot exists without the pattern', () => { - // Transcript mentions the keyword, but the actual file does not implement it - const records = [ - makeRecord(1, { agentOutput: 'Added DependencyInversion' }), - makeRecord(2, { - agentOutput: 'Applied DependencyInversion pattern', - transcript: 'I used DependencyInversion throughout', - workspaceSnapshot: makeSnapshot({ - 'src/index.ts': 'function doStuff() { return 42; }', - }), - }), - ]; - - const score = scoreKnowledgeRetention(records, ['DependencyInversion']); - expect(score.score).toBe(0); - expect(score.details).toContain('lost: DependencyInversion'); - }); - - it('falls back to transcript/output when no workspace snapshot present', () => { - const records = [ - makeRecord(1, { agentOutput: 'Initial setup' }), - makeRecord(2, { agentOutput: 'Applied DependencyInversion' }), - ]; - - const score = scoreKnowledgeRetention(records, ['DependencyInversion']); - expect(score.score).toBe(1); - }); - - it('evaluates latest session by sessionNumber, not array position', () => { - const records = [ - makeRecord(3, { agentOutput: 'Has the pattern' }), - makeRecord(1, { agentOutput: 'Also has the pattern' }), - makeRecord(2, { agentOutput: 'No pattern here' }), - ]; - - // Latest is session 3 which has the pattern - const score = scoreKnowledgeRetention(records, ['pattern']); - expect(score.score).toBe(1); - }); -}); - -describe('scoreKnowledgeRetentionTimeline', () => { - it('produces one score per session in order', () => { - const records = [ - makeRecord(1, { agentOutput: 'Pattern A and Pattern B' }), - makeRecord(2, { agentOutput: 'Pattern A only' }), - makeRecord(3, { agentOutput: 'Neither pattern' }), - ]; - - const timeline = scoreKnowledgeRetentionTimeline(records, ['Pattern A', 'Pattern B']); - - expect(timeline).toHaveLength(3); - expect(timeline[0].score).toBe(1); // Session 1: both patterns - expect(timeline[1].score).toBe(0.5); // Session 2: one pattern - expect(timeline[2].score).toBe(0); // Session 3: no patterns - }); - - it('returns empty array when no patterns defined', () => { - const timeline = scoreKnowledgeRetentionTimeline([makeRecord(1)], []); - expect(timeline).toEqual([]); - }); - - it('sorts by session number', () => { - const records = [ - makeRecord(3, { agentOutput: 'no match' }), - makeRecord(1, { agentOutput: 'Pattern A' }), - makeRecord(2, { agentOutput: 'Pattern A' }), - ]; - - const timeline = scoreKnowledgeRetentionTimeline(records, ['Pattern A']); - expect(timeline[0].details).toContain('session 1'); - expect(timeline[1].details).toContain('session 2'); - expect(timeline[2].details).toContain('session 3'); - }); -}); diff --git a/evals/tests/unit/llm-judge-scorer.test.ts b/evals/tests/unit/llm-judge-scorer.test.ts deleted file mode 100644 index e0191ee9..00000000 --- a/evals/tests/unit/llm-judge-scorer.test.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { createSessionRecord } from '../../src/domain/types.js'; -import { - scoreWithJudge, - scoreAllJudgeDimensions, - validateJudgeConfig, - parseJudgeResponse, -} from '../../src/scoring/llm-judge-scorer.js'; -import type { JudgeConfig, JudgeFn } from '../../src/scoring/llm-judge-scorer.js'; -import { - CONSISTENCY_RUBRIC, - ERROR_CORRECTION_RUBRIC, - ARCHITECTURAL_QUALITY_RUBRIC, - ALL_RUBRICS, - buildJudgePrompt, -} from '../../src/scoring/rubrics.js'; - -function makeRecords(count: number): ReturnType<typeof createSessionRecord>[] { - return Array.from({ length: count }, (_, i) => - createSessionRecord({ - id: `rec-${i + 1}`, - scenarioId: 'scenario-1', - sessionNumber: i + 1, - harness: 'claude-code', - agentOutput: `Session ${i + 1} output: implemented feature ${i + 1}`, - filesModified: [`src/feature-${i + 1}.ts`], - transcript: `User: Continue the project.\nAgent: I'll work on feature ${i + 1}. Creating src/feature-${i + 1}.ts with the naming convention from session 1.`, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }), - ); -} - -function makeMockJudgeResponse(rubricDimension: string): string { - const questionIds: Record<string, string[]> = { - consistency: ['naming-conventions', 'architectural-patterns', 'style-coherence'], - 'error-correction': ['error-detection', 'fix-persistence', 'correction-integration'], - 'architectural-quality': ['separation-of-concerns', 'dependency-direction', 'design-evolution'], - }; - - const ids = questionIds[rubricDimension] ?? []; - const scores = ids.map((id) => ({ - questionId: id, - score: 4, - evidence: `Evidence for ${id}: consistent pattern observed in transcript`, - })); - - return JSON.stringify({ scores }); -} - -const validConfig: JudgeConfig = { - judgeModel: 'gpt-4o', - runnerModel: 'claude-sonnet-4-6', -}; - -const mockJudgeFn: JudgeFn = async (_system, _user, _model) => { - // Default mock — overridden per-test where needed - return makeMockJudgeResponse('consistency'); -}; - -describe('validateJudgeConfig', () => { - it('passes when judge and runner models differ', () => { - expect(() => validateJudgeConfig(validConfig)).not.toThrow(); - }); - - it('throws when judge and runner models match', () => { - expect(() => - validateJudgeConfig({ judgeModel: 'claude-sonnet-4-6', runnerModel: 'claude-sonnet-4-6' }), - ).toThrow('must differ from runner model'); - }); - - it('throws when judge model is empty', () => { - expect(() => - validateJudgeConfig({ judgeModel: '', runnerModel: 'claude-sonnet-4-6' }), - ).toThrow('Judge model must be specified'); - }); - - it('throws when runner model is empty', () => { - expect(() => - validateJudgeConfig({ judgeModel: 'gpt-4o', runnerModel: '' }), - ).toThrow('Runner model must be specified'); - }); -}); - -describe('parseJudgeResponse', () => { - it('parses valid JSON response', () => { - const raw = makeMockJudgeResponse('consistency'); - const result = parseJudgeResponse(raw, CONSISTENCY_RUBRIC); - - expect(result.scores).toHaveLength(3); - expect(result.scores[0].questionId).toBe('naming-conventions'); - expect(result.scores[0].score).toBe(4); - expect(result.scores[0].evidence).toContain('Evidence for naming-conventions'); - }); - - it('handles markdown code fences', () => { - const raw = '```json\n' + makeMockJudgeResponse('consistency') + '\n```'; - const result = parseJudgeResponse(raw, CONSISTENCY_RUBRIC); - - expect(result.scores).toHaveLength(3); - }); - - it('throws on invalid JSON', () => { - expect(() => parseJudgeResponse('not json', CONSISTENCY_RUBRIC)).toThrow('not valid JSON'); - }); - - it('throws on missing scores array', () => { - expect(() => parseJudgeResponse('{}', CONSISTENCY_RUBRIC)).toThrow('missing "scores" array'); - }); - - it('throws on invalid score value', () => { - const raw = JSON.stringify({ - scores: [ - { questionId: 'naming-conventions', score: 6, evidence: 'test' }, - { questionId: 'architectural-patterns', score: 4, evidence: 'test' }, - { questionId: 'style-coherence', score: 4, evidence: 'test' }, - ], - }); - expect(() => parseJudgeResponse(raw, CONSISTENCY_RUBRIC)).toThrow('Invalid score'); - }); - - it('throws on missing evidence', () => { - const raw = JSON.stringify({ - scores: [ - { questionId: 'naming-conventions', score: 4, evidence: '' }, - { questionId: 'architectural-patterns', score: 4, evidence: 'test' }, - { questionId: 'style-coherence', score: 4, evidence: 'test' }, - ], - }); - expect(() => parseJudgeResponse(raw, CONSISTENCY_RUBRIC)).toThrow('Missing evidence'); - }); - - it('throws on unexpected questionId', () => { - const raw = JSON.stringify({ - scores: [ - { questionId: 'bogus-id', score: 4, evidence: 'test' }, - ], - }); - expect(() => parseJudgeResponse(raw, CONSISTENCY_RUBRIC)).toThrow('Unexpected or missing questionId'); - }); - - it('throws on wrong number of scores', () => { - const raw = JSON.stringify({ - scores: [ - { questionId: 'naming-conventions', score: 4, evidence: 'test' }, - ], - }); - expect(() => parseJudgeResponse(raw, CONSISTENCY_RUBRIC)).toThrow('Expected 3 scores, got 1'); - }); -}); - -describe('scoreWithJudge', () => { - it('returns normalized score from judge response', async () => { - const records = makeRecords(3); - const judgeFn: JudgeFn = async () => makeMockJudgeResponse('consistency'); - - const result = await scoreWithJudge(records, CONSISTENCY_RUBRIC, validConfig, judgeFn); - - expect(result.dimension).toBe('consistency'); - // All scores are 4/5, so average = 12/15 = 0.8 - expect(result.score).toBe(0.8); - expect(result.maxScore).toBe(1); - expect(result.details).toContain('naming-conventions=4/5'); - expect(result.details).toContain('Evidence for naming-conventions'); - }); - - it('returns zero score for empty session records', async () => { - const result = await scoreWithJudge([], CONSISTENCY_RUBRIC, validConfig, mockJudgeFn); - - expect(result.score).toBe(0); - expect(result.details).toContain('No session records'); - }); - - it('passes correct model to judge function', async () => { - const records = makeRecords(2); - let capturedModel = ''; - const judgeFn: JudgeFn = async (_sys, _user, model) => { - capturedModel = model; - return makeMockJudgeResponse('consistency'); - }; - - await scoreWithJudge(records, CONSISTENCY_RUBRIC, validConfig, judgeFn); - expect(capturedModel).toBe('gpt-4o'); - }); - - it('passes system prompt from rubric to judge function', async () => { - const records = makeRecords(2); - let capturedSystem = ''; - const judgeFn: JudgeFn = async (sys, _user, _model) => { - capturedSystem = sys; - return makeMockJudgeResponse('consistency'); - }; - - await scoreWithJudge(records, CONSISTENCY_RUBRIC, validConfig, judgeFn); - expect(capturedSystem).toBe(CONSISTENCY_RUBRIC.systemPrompt); - }); - - it('throws when judge and runner model match', async () => { - const records = makeRecords(2); - const badConfig: JudgeConfig = { judgeModel: 'same-model', runnerModel: 'same-model' }; - - await expect( - scoreWithJudge(records, CONSISTENCY_RUBRIC, badConfig, mockJudgeFn), - ).rejects.toThrow('must differ from runner model'); - }); -}); - -describe('scoreAllJudgeDimensions', () => { - it('scores all three qualitative dimensions', async () => { - const records = makeRecords(3); - let callCount = 0; - const judgeFn: JudgeFn = async (_sys, _user, _model) => { - const dimensions = ['consistency', 'error-correction', 'architectural-quality']; - return makeMockJudgeResponse(dimensions[callCount++]); - }; - - const scores = await scoreAllJudgeDimensions(records, validConfig, judgeFn); - - expect(scores).toHaveLength(3); - expect(scores[0].dimension).toBe('consistency'); - expect(scores[1].dimension).toBe('error-correction'); - expect(scores[2].dimension).toBe('architectural-quality'); - expect(scores.every((s) => s.score === 0.8)).toBe(true); - }); - - it('accepts custom rubric subset', async () => { - const records = makeRecords(2); - const judgeFn: JudgeFn = async () => makeMockJudgeResponse('consistency'); - - const scores = await scoreAllJudgeDimensions( - records, - validConfig, - judgeFn, - [CONSISTENCY_RUBRIC], - ); - - expect(scores).toHaveLength(1); - expect(scores[0].dimension).toBe('consistency'); - }); -}); - -describe('buildJudgePrompt', () => { - it('includes rubric questions and transcripts', () => { - const transcripts = ['Session 1 transcript', 'Session 2 transcript']; - const prompt = buildJudgePrompt(CONSISTENCY_RUBRIC, transcripts); - - expect(prompt).toContain('naming conventions'); - expect(prompt).toContain('Session 1 Transcript'); - expect(prompt).toContain('Session 2 Transcript'); - expect(prompt).toContain('Session 1 transcript'); - expect(prompt).toContain('Session 2 transcript'); - expect(prompt).toContain('1 - None'); - expect(prompt).toContain('5 - Excellent'); - }); - - it('includes response format instructions', () => { - const prompt = buildJudgePrompt(ERROR_CORRECTION_RUBRIC, ['transcript']); - - expect(prompt).toContain('questionId'); - expect(prompt).toContain('evidence'); - expect(prompt).toContain('MUST include specific evidence'); - }); -}); - -describe('rubrics', () => { - it('ALL_RUBRICS contains exactly 3 rubrics', () => { - expect(ALL_RUBRICS).toHaveLength(3); - }); - - it('each rubric has 3 questions with 5-point scales', () => { - for (const rubric of ALL_RUBRICS) { - expect(rubric.questions).toHaveLength(3); - for (const q of rubric.questions) { - expect(q.scoringScale).toHaveLength(5); - expect(q.scoringScale[0].score).toBe(1); - expect(q.scoringScale[4].score).toBe(5); - } - } - }); - - it('rubric dimensions are unique', () => { - const dimensions = ALL_RUBRICS.map((r) => r.dimension); - expect(new Set(dimensions).size).toBe(dimensions.length); - }); - - it('rubric question ids are unique within each rubric', () => { - for (const rubric of ALL_RUBRICS) { - const ids = rubric.questions.map((q) => q.id); - expect(new Set(ids).size).toBe(ids.length); - } - }); -}); diff --git a/evals/tests/unit/local-executor.test.ts b/evals/tests/unit/local-executor.test.ts deleted file mode 100644 index ec385dde..00000000 --- a/evals/tests/unit/local-executor.test.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import { existsSync } from 'node:fs'; -import { writeFile, mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; - -describe('LocalExecutor', () => { - const executor = new LocalExecutor(); - - it('creates a temp working directory', async () => { - const workDir = await executor.createWorkDir('test-exec-'); - expect(workDir).toBeTruthy(); - expect(existsSync(workDir)).toBe(true); - await executor.cleanup(workDir); - }); - - it('creates unique directories per call', async () => { - const dir1 = await executor.createWorkDir('test-unique-'); - const dir2 = await executor.createWorkDir('test-unique-'); - expect(dir1).not.toBe(dir2); - await executor.cleanup(dir1); - await executor.cleanup(dir2); - }); - - it('executes a command with cwd and captures stdout', async () => { - const workDir = await executor.createWorkDir('test-exec-'); - const result = await executor.exec(workDir, ['echo', 'hello']); - - expect(result.stdout.trim()).toBe('hello'); - expect(result.exitCode).toBe(0); - await executor.cleanup(workDir); - }); - - it('captures stderr', async () => { - const workDir = await executor.createWorkDir('test-stderr-'); - const result = await executor.exec(workDir, ['node', '--input-type=commonjs', '-e', "console.error('warning')"]); - - expect(result.stderr).toContain('warning'); - await executor.cleanup(workDir); - }); - - it('captures non-zero exit code', async () => { - const workDir = await executor.createWorkDir('test-exit-'); - const result = await executor.exec(workDir, ['node', '--input-type=commonjs', '-e', 'process.exit(42)']); - - expect(result.exitCode).toBe(42); - await executor.cleanup(workDir); - }); - - it('returns error result for invalid command', async () => { - const workDir = await executor.createWorkDir('test-invalid-'); - const result = await executor.exec(workDir, ['nonexistent_command_xyz']); - - expect(result.exitCode).not.toBe(0); - await executor.cleanup(workDir); - }); - - describe('stdin delivery (multi-line prompt safety)', () => { - // Single-line node script that echoes stdin verbatim. Kept on one line so - // the script itself never contains an embedded \n that cmd.exe could trip - // over — the multi-line content under test arrives via stdin, not argv. - const ECHO_STDIN_SCRIPT = - "let b='';process.stdin.setEncoding('utf-8');process.stdin.on('data',d=>{b+=d});process.stdin.on('end',()=>{process.stdout.write(b)})"; - - it('delivers a multi-line stdin payload to a node child intact', async () => { - const workDir = await executor.createWorkDir('test-stdin-multiline-'); - const multiLinePayload = [ - 'line one of jumbo context wrapper', - 'line two — scenario continuation', - 'line three with embedded "quotes" and \\backslashes', - '', - 'line five after a blank', - ].join('\n'); - - const result = await executor.exec( - workDir, - ['node', '-e', ECHO_STDIN_SCRIPT], - { stdin: multiLinePayload }, - ); - - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(multiLinePayload); - await executor.cleanup(workDir); - }); - - it('omits stdin entirely when the option is not provided (stdin closes immediately)', async () => { - const workDir = await executor.createWorkDir('test-stdin-none-'); - const result = await executor.exec(workDir, ['node', '-e', ECHO_STDIN_SCRIPT]); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(''); - await executor.cleanup(workDir); - }); - - // Windows is where this bug actually manifests: cmd.exe /d /s /c truncates - // joined argv at the first newline. This test documents that the new - // delivery path bypasses cmd.exe's tokenizer by going through stdin. - const describeWindows = process.platform === 'win32' ? describe : describe.skip; - describeWindows('Windows cmd.exe newline handling', () => { - it('preserves Jumbo-context-shaped multi-line prompt across cmd.exe via stdin', async () => { - const workDir = await executor.createWorkDir('test-stdin-win-'); - const jumboShapedPrompt = [ - '## Jumbo session context', - '', - 'Project: Jumbo Evals', - 'Active goal: eaf11d9e — repair LocalExecutor command-delivery path', - '', - '## Scenario', - '', - 'Build a hello-world TypeScript module that exports `greet(name)`.', - ].join('\n'); - - const result = await executor.exec( - workDir, - ['node', '-e', ECHO_STDIN_SCRIPT], - { stdin: jumboShapedPrompt }, - ); - - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(jumboShapedPrompt); - // Specifically the lines that used to be silently dropped: - expect(result.stdout).toContain('Active goal: eaf11d9e'); - expect(result.stdout).toContain('Build a hello-world TypeScript module'); - await executor.cleanup(workDir); - }); - }); - }); - - it('cleanup removes the directory', async () => { - const workDir = await executor.createWorkDir('test-cleanup-'); - expect(existsSync(workDir)).toBe(true); - - await executor.cleanup(workDir); - expect(existsSync(workDir)).toBe(false); - }); - - it('cleanup is safe to call on non-existent directory', async () => { - await expect(executor.cleanup('/tmp/nonexistent-dir-xyz')).resolves.not.toThrow(); - }); - - describe('captureWorkspaceSnapshot', () => { - it('includes normal files in snapshot', async () => { - const workDir = await executor.createWorkDir('test-snap-'); - await writeFile(join(workDir, 'main.ts'), 'export const x = 1;'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - expect(snapshot.files.some((f) => f.path === 'main.ts')).toBe(true); - await executor.cleanup(workDir); - }); - - it('excludes .env files', async () => { - const workDir = await executor.createWorkDir('test-snap-env-'); - await writeFile(join(workDir, '.env'), 'API_KEY=secret123'); - await writeFile(join(workDir, '.env.local'), 'DB_PASSWORD=hunter2'); - await writeFile(join(workDir, '.env.production'), 'TOKEN=abc'); - await writeFile(join(workDir, 'safe.txt'), 'safe content'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - const paths = snapshot.files.map((f) => f.path); - expect(paths).not.toContain('.env'); - expect(paths).not.toContain('.env.local'); - expect(paths).not.toContain('.env.production'); - expect(paths).toContain('safe.txt'); - await executor.cleanup(workDir); - }); - - it('excludes cert and key files', async () => { - const workDir = await executor.createWorkDir('test-snap-keys-'); - await writeFile(join(workDir, 'server.key'), 'PRIVATE KEY CONTENT'); - await writeFile(join(workDir, 'server.pem'), 'CERT CONTENT'); - await writeFile(join(workDir, 'id_rsa'), 'RSA KEY'); - await writeFile(join(workDir, 'config.json'), '{"port":3000}'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - const paths = snapshot.files.map((f) => f.path); - expect(paths).not.toContain('server.key'); - expect(paths).not.toContain('server.pem'); - expect(paths).not.toContain('id_rsa'); - expect(paths).toContain('config.json'); - await executor.cleanup(workDir); - }); - - it('excludes secret-named credential files', async () => { - const workDir = await executor.createWorkDir('test-snap-creds-'); - await writeFile(join(workDir, 'credentials.json'), '{"token":"abc"}'); - await writeFile(join(workDir, 'secrets.yaml'), 'key: value'); - await writeFile(join(workDir, 'api_keys.txt'), 'key=123'); - await writeFile(join(workDir, 'readme.md'), '# Project'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - const paths = snapshot.files.map((f) => f.path); - expect(paths).not.toContain('credentials.json'); - expect(paths).not.toContain('secrets.yaml'); - expect(paths).not.toContain('api_keys.txt'); - expect(paths).toContain('readme.md'); - await executor.cleanup(workDir); - }); - - it('excludes secret-prone files in subdirectories', async () => { - const workDir = await executor.createWorkDir('test-snap-sub-'); - await mkdir(join(workDir, 'config')); - await writeFile(join(workDir, 'config', '.env'), 'SECRET=x'); - await writeFile(join(workDir, 'config', 'app.json'), '{}'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - const paths = snapshot.files.map((f) => f.path); - expect(paths).not.toContain('config/.env'); - expect(paths).toContain('config/app.json'); - await executor.cleanup(workDir); - }); - - it('summarizes .jumbo/events into jumboEvents without capturing event content as files', async () => { - const workDir = await executor.createWorkDir('test-snap-events-'); - const eventsDir = join(workDir, '.jumbo', 'events'); - await mkdir(join(eventsDir, 'project'), { recursive: true }); - await mkdir(join(eventsDir, 'agg-1'), { recursive: true }); - await writeFile(join(eventsDir, 'project', '000001.ProjectInitializedEvent.json'), '{"big":"content"}'); - await writeFile(join(eventsDir, 'agg-1', '000001.DecisionAddedEvent.json'), '{"big":"content"}'); - await writeFile(join(workDir, 'main.ts'), 'export const x = 1;'); - - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - - // user-authored files still captured - expect(snapshot.files.some((f) => f.path === 'main.ts')).toBe(true); - // .jumbo content NOT captured as workspace files - expect(snapshot.files.some((f) => f.path.startsWith('.jumbo/'))).toBe(false); - // event log summarized - expect(snapshot.jumboEvents).toBeDefined(); - expect(snapshot.jumboEvents!.eventCount).toBe(2); - expect(snapshot.jumboEvents!.aggregateCount).toBe(2); - expect(snapshot.jumboEvents!.countsByType).toEqual({ - ProjectInitializedEvent: 1, - DecisionAddedEvent: 1, - }); - await executor.cleanup(workDir); - }); - - it('omits jumboEvents when there is no .jumbo directory (baseline arm)', async () => { - const workDir = await executor.createWorkDir('test-snap-no-jumbo-'); - await writeFile(join(workDir, 'main.ts'), 'export const x = 1;'); - const snapshot = await executor.captureWorkspaceSnapshot(workDir); - expect(snapshot.jumboEvents).toBeUndefined(); - await executor.cleanup(workDir); - }); - }); - - describe('diffWorkspaceSnapshots', () => { - const snap = (...files: ReadonlyArray<readonly [string, string]>) => ({ - capturedAt: '2026-05-01T00:00:00.000Z', - files: files.map(([path, content]) => ({ path, content })), - }); - - it('returns empty when snapshots match', () => { - const before = snap(['a.ts', 'x'], ['b.ts', 'y']); - const after = snap(['a.ts', 'x'], ['b.ts', 'y']); - expect(LocalExecutor.diffWorkspaceSnapshots(before, after)).toEqual([]); - }); - - it('reports added files', () => { - const before = snap(['a.ts', 'x']); - const after = snap(['a.ts', 'x'], ['new.ts', 'hello']); - expect(LocalExecutor.diffWorkspaceSnapshots(before, after)).toEqual(['new.ts']); - }); - - it('reports files with changed content', () => { - const before = snap(['a.ts', 'old']); - const after = snap(['a.ts', 'new']); - expect(LocalExecutor.diffWorkspaceSnapshots(before, after)).toEqual(['a.ts']); - }); - - it('treats missing before-snapshot as all files added', () => { - const after = snap(['b.ts', 'y'], ['a.ts', 'x']); - expect(LocalExecutor.diffWorkspaceSnapshots(undefined, after)).toEqual(['a.ts', 'b.ts']); - }); - - it('ignores files removed in after', () => { - const before = snap(['a.ts', 'x'], ['b.ts', 'y']); - const after = snap(['a.ts', 'x']); - expect(LocalExecutor.diffWorkspaceSnapshots(before, after)).toEqual([]); - }); - - it('returns sorted paths', () => { - const before = snap(); - const after = snap(['z.ts', '1'], ['a.ts', '1'], ['m.ts', '1']); - expect(LocalExecutor.diffWorkspaceSnapshots(before, after)).toEqual(['a.ts', 'm.ts', 'z.ts']); - }); - }); -}); diff --git a/evals/tests/unit/output/comparison-display.test.ts b/evals/tests/unit/output/comparison-display.test.ts deleted file mode 100644 index 8c7b5e0c..00000000 --- a/evals/tests/unit/output/comparison-display.test.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { formatComparisonOutput } from '../../../src/output/comparison-display.js'; -import { - createComparisonResult, - createTestResult, - createSessionRecord, -} from '../../../src/domain/types.js'; -import type { DimensionScore, PerSessionScore } from '../../../src/domain/types.js'; - -function makeRecord(sessionNumber: number, suffix: string) { - return createSessionRecord({ - id: `r-${suffix}-${sessionNumber}`, - scenarioId: 's1', - sessionNumber, - harness: 'h', - agentOutput: '', - filesModified: [], - transcript: '', - startedAt: '', - completedAt: '', - }); -} - -describe('formatComparisonOutput — token efficiency rendering', () => { - it('renders a single token-efficiency row with the comparative ratio, (ref) baseline, and raw totals', () => { - const tokenEfficiency: DimensionScore = { - dimension: 'token-efficiency', - score: 0.87, - maxScore: 1, - details: 'jumbo: 1300 tokens (130 tpq); baseline: 10000 tokens (1000 tpq)', - }; - - const comparison = createComparisonResult({ - id: 'cmp', - scenarioId: 's1', - harness: 'h', - jumboResult: createTestResult({ - id: 'jr', - scenarioId: 's1', - harness: 'h', - sessionRecords: [makeRecord(1, 'j')], - }), - baselineResult: createTestResult({ - id: 'br', - scenarioId: 's1', - harness: 'h', - sessionRecords: [makeRecord(1, 'b')], - }), - jumboScores: [tokenEfficiency], - baselineScores: [tokenEfficiency], - deltas: [{ dimension: 'token-efficiency', score: 0, maxScore: 1 }], - }); - - const output = formatComparisonOutput(comparison); - - expect(output).toContain('token-efficiency'); - expect(output).toContain('+0.87'); - expect(output).toContain('(ref)'); - expect(output).toContain('jumbo: 1300 tokens'); - expect(output).toContain('baseline: 10000 tokens'); - // No self-ratio rendering - expect(output).not.toMatch(/0\.87\/1\.00\s+0\.87\/1\.00/); - }); - - it('renders negative comparative ratio without sign prefix', () => { - const tokenEfficiency: DimensionScore = { - dimension: 'token-efficiency', - score: -0.42, - maxScore: 1, - details: 'jumbo: 5000 tokens (500 tpq); baseline: 3500 tokens (350 tpq)', - }; - - const comparison = createComparisonResult({ - id: 'cmp', - scenarioId: 's1', - harness: 'h', - jumboResult: createTestResult({ - id: 'jr', - scenarioId: 's1', - harness: 'h', - sessionRecords: [makeRecord(1, 'j')], - }), - baselineResult: createTestResult({ - id: 'br', - scenarioId: 's1', - harness: 'h', - sessionRecords: [makeRecord(1, 'b')], - }), - jumboScores: [tokenEfficiency], - baselineScores: [tokenEfficiency], - deltas: [{ dimension: 'token-efficiency', score: 0, maxScore: 1 }], - }); - - const output = formatComparisonOutput(comparison); - expect(output).toContain('-0.42'); - }); - - it('renders token-usage timeline rows as raw totals, not score/maxScore', () => { - const jumboRecords = [makeRecord(1, 'j'), makeRecord(2, 'j')]; - const baselineRecords = [makeRecord(1, 'b'), makeRecord(2, 'b')]; - - const tokenEfficiency: DimensionScore = { - dimension: 'token-efficiency', - score: 0.5, - maxScore: 1, - details: 'jumbo: 100 tokens; baseline: 200 tokens', - }; - - const jumboTimeline: PerSessionScore[] = [ - { - sessionNumber: 1, - scores: [{ dimension: 'token-usage', score: 1200, maxScore: 1200, details: '' }], - }, - { - sessionNumber: 2, - scores: [{ dimension: 'token-usage', score: 800, maxScore: 800, details: '' }], - }, - ]; - const baselineTimeline: PerSessionScore[] = [ - { - sessionNumber: 1, - scores: [{ dimension: 'token-usage', score: 3000, maxScore: 3000, details: '' }], - }, - { - sessionNumber: 2, - scores: [{ dimension: 'token-usage', score: 2500, maxScore: 2500, details: '' }], - }, - ]; - - const comparison = createComparisonResult({ - id: 'cmp', - scenarioId: 's1', - harness: 'h', - jumboResult: createTestResult({ - id: 'jr', - scenarioId: 's1', - harness: 'h', - sessionRecords: jumboRecords, - }), - baselineResult: createTestResult({ - id: 'br', - scenarioId: 's1', - harness: 'h', - sessionRecords: baselineRecords, - }), - jumboScores: [tokenEfficiency], - baselineScores: [tokenEfficiency], - deltas: [{ dimension: 'token-efficiency', score: 0, maxScore: 1 }], - jumboTimeline, - baselineTimeline, - }); - - const output = formatComparisonOutput(comparison); - - expect(output).toContain('TIMELINE'); - expect(output).toContain('token-usage'); - // Raw totals appear - expect(output).toMatch(/\b1200\b/); - expect(output).toMatch(/\b3000\b/); - // Signed raw difference - expect(output).toContain('-1800'); - expect(output).toContain('-1700'); - // No bogus self-ratio rendering for token-usage rows - expect(output).not.toContain('1200.00/1200.00'); - expect(output).not.toContain('3000.00/3000.00'); - }); -}); diff --git a/evals/tests/unit/phase-timings.test.ts b/evals/tests/unit/phase-timings.test.ts deleted file mode 100644 index e2d3d6c6..00000000 --- a/evals/tests/unit/phase-timings.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { runABComparison } from '../../src/ab-runner.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { EvalRunRecord, RunHeartbeat, SessionRecord, TestResult } from '../../src/domain/types.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; - -describe('phase timings', () => { - it('stamps successful session records with phaseTimings', async () => { - const scenario = createTestScenario({ - id: 'scenario-timing', - name: 'Timing', - initialPrompt: 'Build', - sessionCount: 1, - }); - const store = createStore(); - - const comparison = await runABComparison({ - scenario, - adapter: createAdapter(), - executor: createExecutor(), - store, - }); - - const jumbo = comparison.jumboResult.sessionRecords[0]; - const baseline = comparison.baselineResult.sessionRecords[0]; - expect(jumbo.phaseTimings?.harnessExec.elapsedMs).toBeGreaterThan(0); - expect(jumbo.phaseTimings?.lifecycleAudit?.elapsedMs).toBeGreaterThan(0); - expect(baseline.phaseTimings?.harnessExec.elapsedMs).toBeGreaterThan(0); - expect(baseline.phaseTimings?.lifecycleAudit).toBeUndefined(); - }); -}); - -function createExecutor(): LocalExecutor { - return { - createWorkDir: async (prefix?: string) => `/tmp/${prefix ?? 'eval'}${Math.random()}`, - installJumboShim: async (workDir: string) => ({ env: { PATH: `${workDir}/.eval-bin:/usr/bin` } }), - exec: async (_workDir: string, command: string[], options?: { env?: Record<string, string | undefined> }) => { - if (command[0] === 'jumbo' && command[1] === '--version') { - const shimmed = options?.env?.PATH?.includes('.eval-bin') ?? false; - return shimmed - ? { stdout: '', stderr: 'shim', exitCode: 127 } - : { stdout: 'jumbo 1.2.3', stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'show') { - return { stdout: JSON.stringify({ goal: { status: 'submitted' } }), stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[1] === 'sessions' && command[2] === 'list') { - return { stdout: '[]', stderr: '', exitCode: 0 }; - } - if (command[0] === 'jumbo' && command[3] === '--format') { - return { stdout: '[]', stderr: '', exitCode: 0 }; - } - if (command[0] === 'mock') { - return { stdout: '{"result":"ok","files_modified":[]}', stderr: '', exitCode: 0 }; - } - return { stdout: 'ok', stderr: '', exitCode: 0 }; - }, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - } as LocalExecutor; -} - -function createAdapter(): HarnessAdapter { - return { - name: 'mock', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ agentOutput: result.stdout, filesModified: [], transcript: result.stdout }), - seedToolPermissions: async () => {}, - }; -} - -function createStore(): ResultStore { - return { - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async () => {}, - getSessionRecords: async () => [], - saveTestResult: async () => {}, - getTestResult: async () => null, - listTestResults: async () => [], - saveRunRecord: async (_record: EvalRunRecord) => {}, - getRunRecord: async () => null, - listRunRecords: async () => [], - writeHeartbeat: async (_runId: string, _heartbeat: RunHeartbeat) => {}, - readHeartbeat: async () => null, - }; -} diff --git a/evals/tests/unit/protocol-adherence-scorer.test.ts b/evals/tests/unit/protocol-adherence-scorer.test.ts deleted file mode 100644 index b726fc90..00000000 --- a/evals/tests/unit/protocol-adherence-scorer.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { describe, expect, it } from '@jest/globals'; -import { - adherenceForSession, - baselineProtocolAdherenceScore, - scoreProtocolAdherence, - scoreProtocolAdherenceTimeline, -} from '../../src/scoring/protocol-adherence-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; -import type { JumboLifecycleAudit, SessionRecord } from '../../src/domain/types.js'; - -function audit(overrides: Partial<JumboLifecycleAudit> = {}): JumboLifecycleAudit { - return { - sessionStartExecuted: true, - goalStartExecuted: true, - inSessionCapturesExecuted: true, - progressUpdatesExecuted: true, - goalSubmitExecuted: true, - sessionEndExecuted: true, - sessionsTotalDelta: 1, - sessionsEndedDelta: 1, - newEntityCount: 0, - evidence: {}, - ...overrides, - }; -} - -function record(sessionNumber: number, a: JumboLifecycleAudit | undefined): SessionRecord { - return createSessionRecord({ - id: `r-${sessionNumber}`, - scenarioId: 's', - sessionNumber, - harness: 'mock', - variant: 'jumbo', - agentOutput: '', - filesModified: [], - transcript: '', - jumboLifecycleAudit: a, - startedAt: '', - completedAt: '', - }); -} - -describe('adherenceForSession', () => { - it('returns score=1 when every prescribed step is recorded as executed', () => { - const result = adherenceForSession(record(1, audit())); - expect(result.score).toBe(1); - expect(result.steps).toHaveLength(6); - expect(result.steps.every((s) => s.passed)).toBe(true); - }); - - it('returns score=0 when no audit exists (e.g. baseline session)', () => { - const result = adherenceForSession(record(1, undefined)); - expect(result.score).toBe(0); - expect(result.steps).toHaveLength(0); - }); - - it('emits per-step pass/fail when only some steps executed', () => { - const result = adherenceForSession( - record( - 1, - audit({ - inSessionCapturesExecuted: false, - progressUpdatesExecuted: false, - }), - ), - ); - expect(result.steps.find((s) => s.step === 'session-start')?.passed).toBe(true); - expect(result.steps.find((s) => s.step === 'in-session-captures')?.passed).toBe(false); - expect(result.steps.find((s) => s.step === 'progress-updates')?.passed).toBe(false); - expect(result.score).toBeCloseTo(4 / 6, 4); - }); -}); - -describe('scoreProtocolAdherence', () => { - it('averages per-session scores across audited sessions', () => { - const score = scoreProtocolAdherence([ - record(1, audit()), - record( - 2, - audit({ progressUpdatesExecuted: false, goalSubmitExecuted: false }), - ), - ]); - - expect(score.dimension).toBe('protocol-adherence'); - // Session 1: 6/6 = 1.00. Session 2: 4/6 ≈ 0.667. Mean ≈ 0.833 → rounds to 0.83. - expect(score.score).toBe(0.83); - expect(score.maxScore).toBe(1); - expect(score.details).toContain('sessions-audited=2'); - expect(score.details).toContain('steps-passed=10/12'); - expect(score.details).toContain('progress-updates-missed-in-sessions:2'); - expect(score.details).toContain('goal-submit-missed-in-sessions:2'); - }); - - it('reports zero with explanatory details when no sessions have an audit', () => { - const score = scoreProtocolAdherence([record(1, undefined)]); - expect(score.score).toBe(0); - expect(score.maxScore).toBe(1); - expect(score.details).toContain('cannot be measured'); - }); - - it('reports all-steps-passed when every session passed every step', () => { - const score = scoreProtocolAdherence([record(1, audit()), record(2, audit())]); - expect(score.details).toContain('all-steps-passed'); - expect(score.score).toBe(1); - }); - - it('keeps the baseline arm at maxScore=0 (N/A)', () => { - const score = baselineProtocolAdherenceScore(); - expect(score.dimension).toBe('protocol-adherence'); - expect(score.score).toBe(0); - expect(score.maxScore).toBe(0); - expect(score.details).toContain('Not applicable'); - }); -}); - -describe('scoreProtocolAdherenceTimeline', () => { - it('emits a per-session DimensionScore with passed/failed step lists', () => { - const timeline = scoreProtocolAdherenceTimeline([ - record(1, audit()), - record(2, audit({ goalSubmitExecuted: false, sessionEndExecuted: false })), - ]); - - expect(timeline).toHaveLength(2); - expect(timeline[0].score).toBe(1); - expect(timeline[0].details).toContain('passed:'); - expect(timeline[0].details).toContain('failed: none'); - expect(timeline[1].score).toBeCloseTo(4 / 6, 1); - expect(timeline[1].details).toContain('failed: goal-submit,session-end'); - }); - - it('emits a sentinel entry for sessions without an audit', () => { - const timeline = scoreProtocolAdherenceTimeline([record(1, undefined)]); - expect(timeline[0].score).toBe(0); - expect(timeline[0].details).toContain('No lifecycle audit'); - }); -}); diff --git a/evals/tests/unit/replication-stats.test.ts b/evals/tests/unit/replication-stats.test.ts deleted file mode 100644 index 28b1cb29..00000000 --- a/evals/tests/unit/replication-stats.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { aggregateReplications } from '../../src/analysis/replication-stats.js'; -import type { ComparisonResult, DimensionScore } from '../../src/domain/index.js'; - -/** Builds a minimal ComparisonResult carrying only the per-dimension scores the aggregator reads. */ -function comparison( - dims: Record<string, { jumbo: number; baseline: number; maxScore?: number }>, -): ComparisonResult { - const score = (dimension: string, value: number, maxScore: number): DimensionScore => ({ - dimension, - score: value, - maxScore, - details: '', - }); - const jumboScores = Object.entries(dims).map(([d, v]) => score(d, v.jumbo, v.maxScore ?? 1)); - const baselineScores = Object.entries(dims).map(([d, v]) => score(d, v.baseline, v.maxScore ?? 1)); - const deltas = Object.entries(dims).map(([d, v]) => score(d, v.jumbo - v.baseline, v.maxScore ?? 1)); - return { - id: 'c', - scenarioId: 'scenario-1', - harness: 'claude-code', - jumboResult: { id: 'j', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], createdAt: 't', tampered: false, tamperLog: [] }, - baselineResult: { id: 'b', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], createdAt: 't', tampered: false, tamperLog: [] }, - jumboScores, - baselineScores, - deltas, - createdAt: 't', - tampered: false, - tamperLog: [], - }; -} - -function dim(report: ReturnType<typeof aggregateReplications>, name: string) { - const d = report.dimensions.find((x) => x.dimension === name); - if (!d) throw new Error(`dimension ${name} not in report`); - return d; -} - -describe('aggregateReplications', () => { - it('computes mean lift, sample (n-1) SD, and arm means across replications', () => { - const report = aggregateReplications([ - comparison({ 'file-accuracy': { jumbo: 0.8, baseline: 0.6 } }), - comparison({ 'file-accuracy': { jumbo: 0.9, baseline: 0.5 } }), - comparison({ 'file-accuracy': { jumbo: 1.0, baseline: 0.4 } }), - ]); - expect(report.k).toBe(3); - expect(report.scenarioId).toBe('scenario-1'); - expect(report.harness).toBe('claude-code'); - - const fa = dim(report, 'file-accuracy'); - expect(fa.meanJumbo).toBeCloseTo(0.9, 6); - expect(fa.meanBaseline).toBeCloseTo(0.5, 6); - expect(fa.meanLift).toBeCloseTo(0.4, 6); // lifts [0.2, 0.4, 0.6] - expect(fa.sdLift).toBeCloseTo(0.2, 6); // sample SD of [0.2,0.4,0.6] - expect(fa.applicableReplications).toBe(3); - expect(fa.tStatistic).toBeCloseTo(0.4 / (0.2 / Math.sqrt(3)), 4); - }); - - it('flags a signal only when |mean lift| exceeds one SD', () => { - // lifts [0.3, 0.5] -> mean 0.4, sd 0.1414 -> signal - const signal = aggregateReplications([ - comparison({ d: { jumbo: 0.3, baseline: 0 } }), - comparison({ d: { jumbo: 0.5, baseline: 0 } }), - ]); - expect(dim(signal, 'd').isSignal).toBe(true); - - // lifts [0, 0.4] -> mean 0.2, sd 0.2828 -> not a signal - const noise = aggregateReplications([ - comparison({ d: { jumbo: 0, baseline: 0 } }), - comparison({ d: { jumbo: 0.4, baseline: 0 } }), - ]); - expect(dim(noise, 'd').isSignal).toBe(false); - }); - - it('treats K=1 as no SD and never a signal', () => { - const report = aggregateReplications([comparison({ d: { jumbo: 1, baseline: 0 } })]); - const d = dim(report, 'd'); - expect(report.k).toBe(1); - expect(d.sdLift).toBe(0); - expect(d.tStatistic).toBe(0); - expect(d.isSignal).toBe(false); - expect(d.applicableReplications).toBe(1); - }); - - it('excludes N/A (maxScore 0) token-efficiency replications and records the applicable count', () => { - const report = aggregateReplications([ - comparison({ 'token-efficiency': { jumbo: 0.5, baseline: 0, maxScore: 1 } }), - comparison({ 'token-efficiency': { jumbo: 0.3, baseline: 0, maxScore: 1 } }), - comparison({ 'token-efficiency': { jumbo: 0, baseline: 0, maxScore: 0 } }), // N/A - ]); - const te = dim(report, 'token-efficiency'); - expect(te.k).toBe(3); - expect(te.applicableReplications).toBe(2); - expect(te.meanLift).toBeCloseTo(0.4, 6); // mean of [0.5, 0.3] - }); - - it('aggregates multiple dimensions independently', () => { - const report = aggregateReplications([ - comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 0.2, baseline: 0.2 } }), - comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 0.4, baseline: 0.4 } }), - ]); - expect(dim(report, 'a').meanLift).toBeCloseTo(1, 6); - expect(dim(report, 'b').meanLift).toBeCloseTo(0, 6); - expect(dim(report, 'a').isSignal).toBe(true); // lift 1, sd 0 - expect(dim(report, 'b').isSignal).toBe(false); // lift 0 - }); - - it('only includes dimensions present in every replication', () => { - const report = aggregateReplications([ - comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 1, baseline: 0 } }), - comparison({ a: { jumbo: 1, baseline: 0 } }), // no b - ]); - expect(report.dimensions.map((d) => d.dimension)).toEqual(['a']); - }); - - it('records the K=5 significance threshold note', () => { - const report = aggregateReplications([ - comparison({ a: { jumbo: 1, baseline: 0 } }), - comparison({ a: { jumbo: 1, baseline: 0 } }), - comparison({ a: { jumbo: 1, baseline: 0 } }), - comparison({ a: { jumbo: 1, baseline: 0 } }), - comparison({ a: { jumbo: 1, baseline: 0 } }), - ]); - expect(report.significance.tCriticalOneTailed05).toBeCloseTo(2.132, 2); // df = 4 - expect(report.significance.note).toContain('2.13'); - }); -}); diff --git a/evals/tests/unit/report-generator.test.ts b/evals/tests/unit/report-generator.test.ts deleted file mode 100644 index 89e524a2..00000000 --- a/evals/tests/unit/report-generator.test.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { - computeDivergenceCurve, - computeLiftPercentages, - detectDivergenceOnset, - analyzeDisruptionImpact, - aggregateHarnessLifts, - generateFullReport, - formatFullReport, -} from '../../src/output/report-generator.js'; -import { createComparisonResult, createTestResult, createSessionRecord } from '../../src/domain/types.js'; -import type { ComparisonResult, Disruption } from '../../src/domain/types.js'; - -function makeComparison(opts: { - harness?: string; - jumboScores?: Array<{ dimension: string; score: number }>; - baselineScores?: Array<{ dimension: string; score: number }>; - sessionCount?: number; -}): ComparisonResult { - const harness = opts.harness ?? 'claude-code'; - const sessionCount = opts.sessionCount ?? 3; - const jumboScoreValues = opts.jumboScores ?? [ - { dimension: 'file-accuracy', score: 0.9 }, - { dimension: 'knowledge-retention', score: 0.8 }, - ]; - const baselineScoreValues = opts.baselineScores ?? [ - { dimension: 'file-accuracy', score: 0.7 }, - { dimension: 'knowledge-retention', score: 0.4 }, - ]; - - const records = Array.from({ length: sessionCount }, (_, i) => - createSessionRecord({ - id: `rec-${i}`, - scenarioId: 'scenario-1', - sessionNumber: i + 1, - harness, - agentOutput: 'output', - filesModified: ['src/index.ts'], - transcript: 'transcript', - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }), - ); - - const jumboResult = createTestResult({ id: 'jr', scenarioId: 'scenario-1', harness, sessionRecords: records }); - const baselineResult = createTestResult({ id: 'br', scenarioId: 'scenario-1', harness, sessionRecords: records }); - - const jumboScores = jumboScoreValues.map((s) => ({ ...s, maxScore: 1 })); - const baselineScores = baselineScoreValues.map((s) => ({ ...s, maxScore: 1 })); - const deltas = jumboScores.map((js, i) => ({ - dimension: js.dimension, - score: Math.round((js.score - baselineScores[i].score) * 1000) / 1000, - maxScore: 1, - })); - - // Build timelines with increasing divergence - const jumboTimeline = records.map((r, i) => ({ - sessionNumber: r.sessionNumber, - scores: jumboScoreValues.map((s) => ({ - dimension: s.dimension, - score: Math.round((s.score - (sessionCount - 1 - i) * 0.05) * 100) / 100, - maxScore: 1, - })), - })); - - const baselineTimeline = records.map((r, i) => ({ - sessionNumber: r.sessionNumber, - scores: baselineScoreValues.map((s) => ({ - dimension: s.dimension, - score: Math.round((s.score - i * 0.1) * 100) / 100, - maxScore: 1, - })), - })); - - return createComparisonResult({ - id: `comp-${harness}`, - scenarioId: 'scenario-1', - harness, - jumboResult, - baselineResult, - jumboScores, - baselineScores, - deltas, - jumboTimeline, - baselineTimeline, - }); -} - -describe('computeDivergenceCurve', () => { - it('returns per-session per-dimension deltas', () => { - const comp = makeComparison({}); - const curve = computeDivergenceCurve(comp); - - expect(curve.length).toBeGreaterThan(0); - expect(curve[0]).toHaveProperty('sessionNumber'); - expect(curve[0]).toHaveProperty('dimension'); - expect(curve[0]).toHaveProperty('delta'); - }); - - it('returns empty for comparisons without timelines', () => { - const comp: ComparisonResult = { - ...makeComparison({}), - jumboTimeline: undefined, - baselineTimeline: undefined, - }; - expect(computeDivergenceCurve(comp)).toEqual([]); - }); -}); - -describe('computeLiftPercentages', () => { - it('computes absolute and percentage lift', () => { - const comp = makeComparison({ - jumboScores: [{ dimension: 'file-accuracy', score: 0.9 }], - baselineScores: [{ dimension: 'file-accuracy', score: 0.6 }], - }); - - const lifts = computeLiftPercentages(comp); - - expect(lifts).toHaveLength(1); - expect(lifts[0].absoluteLift).toBeCloseTo(0.3, 2); - expect(lifts[0].percentageLift).toBeCloseTo(50, 0); - }); - - it('returns null percentage when baseline is zero', () => { - const comp = makeComparison({ - jumboScores: [{ dimension: 'test', score: 0.5 }], - baselineScores: [{ dimension: 'test', score: 0 }], - }); - - const lifts = computeLiftPercentages(comp); - expect(lifts[0].percentageLift).toBeNull(); - }); -}); - -describe('detectDivergenceOnset', () => { - it('detects onset when delta exceeds threshold', () => { - const curve = [ - { sessionNumber: 1, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.78, delta: 0.02 }, - { sessionNumber: 2, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.7, delta: 0.1 }, - { sessionNumber: 3, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.5, delta: 0.3 }, - ]; - - const onsets = detectDivergenceOnset(curve, 0.1); - expect(onsets).toHaveLength(1); - expect(onsets[0].onsetSession).toBe(2); - expect(onsets[0].deltaAtOnset).toBe(0.1); - }); - - it('returns null onset when no divergence exceeds threshold', () => { - const curve = [ - { sessionNumber: 1, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.79, delta: 0.01 }, - { sessionNumber: 2, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.78, delta: 0.02 }, - ]; - - const onsets = detectDivergenceOnset(curve, 0.1); - expect(onsets[0].onsetSession).toBeNull(); - }); -}); - -describe('analyzeDisruptionImpact', () => { - it('ranks disruptions by impact magnitude', () => { - const curve = [ - { sessionNumber: 1, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.8, delta: 0 }, - { sessionNumber: 2, dimension: 'retention', jumboScore: 0.9, baselineScore: 0.5, delta: 0.4 }, - { sessionNumber: 3, dimension: 'retention', jumboScore: 0.85, baselineScore: 0.6, delta: 0.25 }, - ]; - - const disruptions: Disruption[] = [ - { type: 'correction', sessionNumber: 2, content: 'Fix naming convention', recoveryPatterns: ['camelCase'] }, - ]; - - const impacts = analyzeDisruptionImpact(curve, disruptions); - expect(impacts.length).toBeGreaterThan(0); - expect(impacts[0].impactMagnitude).toBeCloseTo(0.4, 2); - }); - - it('returns empty for no disruptions', () => { - const curve = [ - { sessionNumber: 1, dimension: 'retention', jumboScore: 0.8, baselineScore: 0.8, delta: 0 }, - ]; - - expect(analyzeDisruptionImpact(curve, [])).toEqual([]); - }); -}); - -describe('aggregateHarnessLifts', () => { - it('computes average lift per harness', () => { - const comp1 = makeComparison({ - harness: 'claude-code', - jumboScores: [{ dimension: 'test', score: 0.9 }], - baselineScores: [{ dimension: 'test', score: 0.6 }], - }); - const comp2 = makeComparison({ - harness: 'codex-cli', - jumboScores: [{ dimension: 'test', score: 0.8 }], - baselineScores: [{ dimension: 'test', score: 0.4 }], - }); - - const agg = aggregateHarnessLifts([comp1, comp2]); - - expect(agg).toHaveLength(2); - expect(agg[0].harness).toBe('claude-code'); - expect(agg[0].avgLift).toBeCloseTo(0.3, 2); - expect(agg[1].harness).toBe('codex-cli'); - expect(agg[1].avgLift).toBeCloseTo(0.4, 2); - }); -}); - -describe('generateFullReport', () => { - it('produces a complete report structure', () => { - const comp = makeComparison({}); - const report = generateFullReport([comp]); - - expect(report.scenarioId).toBe('scenario-1'); - expect(report.harnesses).toEqual(['claude-code']); - expect(report.divergenceCurve.length).toBeGreaterThan(0); - expect(report.liftResults.length).toBeGreaterThan(0); - expect(report.divergenceOnsets.length).toBeGreaterThan(0); - expect(report.generatedAt).toBeTruthy(); - }); -}); - -describe('formatFullReport', () => { - it('produces readable terminal output', () => { - const comp = makeComparison({}); - const report = generateFullReport([comp]); - const output = formatFullReport(report); - - expect(output).toContain('JUMBO EVALUATION REPORT'); - expect(output).toContain('LIFT BY DIMENSION'); - expect(output).toContain('DIVERGENCE ONSET'); - expect(output).toContain('DIVERGENCE CURVE'); - expect(output).toContain('scenario-1'); - }); - - it('includes cross-harness aggregation when multiple harnesses', () => { - const comp1 = makeComparison({ harness: 'claude-code' }); - const comp2 = makeComparison({ harness: 'codex-cli' }); - const report = generateFullReport([comp1, comp2]); - const output = formatFullReport(report); - - expect(output).toContain('CROSS-HARNESS AGGREGATION'); - }); -}); diff --git a/evals/tests/unit/run-record.test.ts b/evals/tests/unit/run-record.test.ts deleted file mode 100644 index a418c52a..00000000 --- a/evals/tests/unit/run-record.test.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals'; -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; -import * as os from 'node:os'; -import { JsonResultStore } from '../../src/storage/json-result-store.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import { createProgram } from '../../src/cli/index.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { EvalRunRecord, RunHeartbeat, SessionRecord, TestResult, TestScenario } from '../../src/domain/types.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; - -describe('run records', () => { - let tmpDir: string; - let store: JsonResultStore; - - beforeEach(async () => { - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'jumbo-run-record-')); - store = new JsonResultStore(tmpDir); - await store.initialize(); - }); - - afterEach(async () => { - await fs.rm(tmpDir, { recursive: true, force: true }); - }); - - it('persists run records under runs/<runId>/run.json', async () => { - const record: EvalRunRecord = { - runId: 'run-1', - scenarioId: 'scenario-1', - harnesses: ['claude-code', 'codex-cli'], - sessionCount: 2, - startedAt: '2026-04-29T10:00:00.000Z', - status: 'running', - }; - - await store.saveRunRecord(record); - - await expect(fs.readFile(path.join(tmpDir, 'runs', 'run-1', 'run.json'), 'utf-8')).resolves.toContain('"runId": "run-1"'); - await expect(store.getRunRecord('run-1')).resolves.toEqual(record); - await expect(store.listRunRecords('scenario-1')).resolves.toEqual([record]); - }); - - it('writes heartbeat state atomically and leaves only parseable JSON visible', async () => { - const first = makeHeartbeat('run-atomic', 'pending'); - const second = makeHeartbeat('run-atomic', 'completed'); - - await Promise.all(Array.from({ length: 20 }, async (_, index) => { - await store.writeHeartbeat('run-atomic', index % 2 === 0 ? first : second); - const observed = await store.readHeartbeat('run-atomic'); - expect(observed?.runId).toBe('run-atomic'); - })); - - const files = await fs.readdir(path.join(tmpDir, 'runs', 'run-atomic')); - expect(files).toContain('state.json'); - expect(files.some((file) => file.endsWith('.tmp'))).toBe(false); - await expect(store.readHeartbeat('run-atomic')).resolves.toMatchObject({ runId: 'run-atomic' }); - }); -}); - -describe('run command run records', () => { - it('transitions the run record to failed when execution fails', async () => { - const store = new RunStateStore(); - store.scenarios.push(createTestScenario({ - id: 'scenario-cli-fail', - name: 'CLI failure', - initialPrompt: 'Build', - sessionCount: 1, - })); - const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - const program = createProgram({ - storeProvider: async () => store, - executorProvider: () => createFailingHarnessExecutor(), - adapterProvider: () => createAdapter(), - }); - - await expect(program.parseAsync([ - 'node', 'eval', 'run', '--scenario', 'scenario-cli-fail', - ])).rejects.toThrow('harness mock session 1 failed'); - - expect(store.runRecords.map((record) => record.status)).toEqual(['running', 'failed']); - expect(store.runRecords[1].completedAt).toBeDefined(); - const failed = store.heartbeats - .flatMap((heartbeat) => heartbeat.harnesses) - .filter((harness) => harness.variant === 'jumbo') - .flatMap((harness) => harness.sessions) - .findLast((session) => session.status === 'failed'); - expect(failed?.phase).toBe('harness-exec'); - expect(failed?.errorMessage).toContain('harness mock session 1 failed'); - logSpy.mockRestore(); - }); -}); - -function makeHeartbeat(runId: string, status: 'pending' | 'completed'): RunHeartbeat { - return { - runId, - scenarioId: 'scenario-1', - updatedAt: new Date().toISOString(), - harnesses: [{ - harness: 'claude-code', - variant: 'jumbo', - sessions: [{ sessionNumber: 1, status }], - }], - }; -} - -class RunStateStore implements ResultStore { - readonly scenarios: TestScenario[] = []; - readonly runRecords: EvalRunRecord[] = []; - readonly heartbeats: RunHeartbeat[] = []; - - async saveScenario(scenario: TestScenario): Promise<void> { - this.scenarios.push(scenario); - } - - async getScenario(id: string): Promise<TestScenario | null> { - return this.scenarios.find((scenario) => scenario.id === id) ?? null; - } - - async listScenarios(): Promise<TestScenario[]> { - return [...this.scenarios]; - } - - async saveSessionRecord(_record: SessionRecord): Promise<void> {} - - async getSessionRecords(_scenarioId: string): Promise<SessionRecord[]> { - return []; - } - - async saveTestResult(_result: TestResult): Promise<void> {} - - async getTestResult(_id: string): Promise<TestResult | null> { - return null; - } - - async listTestResults(_scenarioId?: string): Promise<TestResult[]> { - return []; - } - - async saveRunRecord(record: EvalRunRecord): Promise<void> { - this.runRecords.push(record); - } - - async getRunRecord(_runId: string): Promise<EvalRunRecord | null> { - return null; - } - - async listRunRecords(_scenarioId?: string): Promise<EvalRunRecord[]> { - return [...this.runRecords]; - } - - async writeHeartbeat(_runId: string, heartbeat: RunHeartbeat): Promise<void> { - this.heartbeats.push(heartbeat); - } - - async readHeartbeat(_runId: string): Promise<RunHeartbeat | null> { - return null; - } -} - -function createFailingHarnessExecutor(): LocalExecutor { - let dir = 0; - return { - createWorkDir: async (prefix?: string) => `/tmp/${prefix ?? 'eval'}${++dir}`, - installJumboShim: async (workDir: string) => ({ env: { PATH: `${workDir}/.eval-bin:/usr/bin` } }), - exec: async (_workDir: string, command: string[], options?: { env?: Record<string, string | undefined> }) => { - if (command[0] === 'jumbo' && command[1] === '--version') { - const shimmed = options?.env?.PATH?.includes('.eval-bin') ?? false; - return shimmed - ? { stdout: '', stderr: 'shim', exitCode: 127 } - : { stdout: 'jumbo 1.2.3', stderr: '', exitCode: 0 }; - } - if (command[0] === 'mock') { - return { stdout: '', stderr: 'harness crashed', exitCode: 2 }; - } - if (command[0] === 'jumbo' && command[3] === '--format') { - return { stdout: '[]', stderr: '', exitCode: 0 }; - } - return { stdout: 'ok', stderr: '', exitCode: 0 }; - }, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - } as LocalExecutor; -} - -function createAdapter(): HarnessAdapter { - return { - name: 'mock', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ agentOutput: result.stdout, filesModified: [], transcript: result.stdout }), - seedToolPermissions: async () => {}, - }; -} diff --git a/evals/tests/unit/run-session.test.ts b/evals/tests/unit/run-session.test.ts deleted file mode 100644 index d28e826a..00000000 --- a/evals/tests/unit/run-session.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { createTestScenario } from '../../src/domain/types.js'; -import { runSession } from '../../src/run-session.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; -import type { SessionRecord } from '../../src/domain/types.js'; - -function createMockExecutor(execResult: ExecResult): LocalExecutor { - return { - createWorkDir: async () => '/tmp/mock-workdir', - exec: async () => execResult, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - } as LocalExecutor; -} - -function createMockAdapter(): HarnessAdapter { - return { - name: 'mock-harness', - buildCommand: () => ['mock'], - parseOutput: (result: ExecResult) => ({ - agentOutput: result.stdout, - filesModified: ['created-file.ts'], - transcript: result.stdout, - }), - seedToolPermissions: async () => {}, - }; -} - -function createMockStore(): ResultStore & { savedRecords: SessionRecord[] } { - const savedRecords: SessionRecord[] = []; - return { - savedRecords, - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async (record: SessionRecord) => { savedRecords.push(record); }, - getSessionRecords: async () => [], - saveTestResult: async () => {}, - getTestResult: async () => null, - listTestResults: async () => [], - }; -} - -describe('runSession', () => { - it('orchestrates end-to-end: scenario -> harness exec -> store session record', async () => { - const scenario = createTestScenario({ - id: 'scenario-1', - name: 'Smoke test', - initialPrompt: 'Create a hello world project', - sessionCount: 1, - }); - - const executor = createMockExecutor({ - stdout: 'Hello world project created successfully', - stderr: '', - exitCode: 0, - }); - - const adapter = createMockAdapter(); - const store = createMockStore(); - - const record = await runSession({ - scenario, - sessionNumber: 1, - variant: 'jumbo', - prompt: 'Jumbo context\n\nCreate a hello world project', - scenarioPrompt: 'Create a hello world project', - deliveredContext: 'Jumbo context', - workDir: '/tmp/mock-workdir', - executor, - adapter, - store, - }); - - expect(record.scenarioId).toBe('scenario-1'); - expect(record.sessionNumber).toBe(1); - expect(record.harness).toBe('mock-harness'); - expect(record.variant).toBe('jumbo'); - expect(record.scenarioPrompt).toBe('Create a hello world project'); - expect(record.effectivePrompt).toBe('Jumbo context\n\nCreate a hello world project'); - expect(record.deliveredContext).toBe('Jumbo context'); - expect(record.agentOutput).toBe('Hello world project created successfully'); - expect(record.filesModified).toEqual(['created-file.ts']); - expect(record.startedAt).toBeDefined(); - expect(record.completedAt).toBeDefined(); - - // Verify it was persisted - expect(store.savedRecords).toHaveLength(1); - expect(store.savedRecords[0].id).toBe(record.id); - }); -}); diff --git a/evals/tests/unit/scenarios-structural-assertions.test.ts b/evals/tests/unit/scenarios-structural-assertions.test.ts deleted file mode 100644 index 354fe63a..00000000 --- a/evals/tests/unit/scenarios-structural-assertions.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { scoreStructuralAssertions } from '../../src/scoring/structural-assertion-scorer.js'; -import { createSessionRecord } from '../../src/domain/index.js'; -import type { SessionRecord, StructuralAssertion, TestScenario } from '../../src/domain/index.js'; - -const scenario = JSON.parse( - readFileSync(join(process.cwd(), 'scenarios', 'event-sourced-inventory.json'), 'utf-8'), -) as TestScenario; - -function record(sessionNumber: number, files: Record<string, string>): SessionRecord { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'event-sourced-inventory', - sessionNumber, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - workspaceSnapshot: { - capturedAt: '2026-03-21T10:05:00Z', - files: Object.entries(files).map(([path, content]) => ({ path, content })), - }, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -describe('event-sourced-inventory authored structural assertions', () => { - it('declares structural assertions with unique ids and in-range session numbers', () => { - const assertions = scenario.structuralAssertions ?? []; - expect(assertions.length).toBeGreaterThan(0); - - const ids = new Set<string>(); - for (const a of assertions) { - expect(a.id).toBeTruthy(); - expect(ids.has(a.id)).toBe(false); - ids.add(a.id); - expect(a.sessionNumber).toBeGreaterThanOrEqual(1); - expect(a.sessionNumber).toBeLessThanOrEqual(scenario.sessionCount); - } - }); - - it('is fully satisfied by a correct cumulative implementation', () => { - const assertions = scenario.structuralAssertions as readonly StructuralAssertion[]; - - // A correct event-sourced implementation as it would look by each due session. - // Workspaces are cumulative: later sessions retain earlier artefacts plus the - // disruption-driven additions (metadata at s3, concurrency control at s5). - const typesWithMetadata = ` - export interface EventMetadata { correlationId: string; causationId: string; timestamp: number; } - export type DomainEvent = - | { type: 'ProductAdded'; metadata: EventMetadata } - | { type: 'StockReceived'; metadata: EventMetadata } - | { type: 'StockReserved'; metadata: EventMetadata }; - `; - const store = `export class EventStore { append(e: DomainEvent) { this.events.push(e); } }`; - const projection = `export function project(events: DomainEvent[]) { return {}; }`; - const handlersWithConcurrency = ` - export class ConcurrencyError extends Error {} - export function handle(cmd: { expectedVersion: number }) { - if (cmd.expectedVersion !== current) throw new ConcurrencyError(); - } - `; - - const records = [ - record(1, { - 'src/events/types.ts': typesWithMetadata, - 'src/events/store.ts': store, - 'src/projections/inventory.ts': projection, - }), - record(3, { - 'src/events/types.ts': typesWithMetadata, - 'src/events/store.ts': store, - 'src/projections/inventory.ts': projection, - }), - record(5, { - 'src/events/types.ts': typesWithMetadata, - 'src/events/store.ts': store, - 'src/projections/inventory.ts': projection, - 'src/commands/handlers.ts': handlersWithConcurrency, - }), - record(7, { - 'src/events/types.ts': typesWithMetadata, - 'src/events/store.ts': store, - 'src/projections/inventory.ts': projection, - 'src/commands/handlers.ts': handlersWithConcurrency, - }), - ]; - - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(1); - }); - - it('is not satisfied when the implementation is empty', () => { - const assertions = scenario.structuralAssertions as readonly StructuralAssertion[]; - const records = [record(1, {}), record(3, {}), record(5, {}), record(7, {})]; - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(0); - }); -}); diff --git a/evals/tests/unit/scenarios-validity.test.ts b/evals/tests/unit/scenarios-validity.test.ts deleted file mode 100644 index 394f5354..00000000 --- a/evals/tests/unit/scenarios-validity.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { scoreStructuralAssertions } from '../../src/scoring/structural-assertion-scorer.js'; -import { createSessionRecord, createTestScenario } from '../../src/domain/index.js'; -import type { SessionRecord, StructuralAssertion, TestScenario } from '../../src/domain/index.js'; - -function loadScenario(fileName: string): TestScenario { - return JSON.parse( - readFileSync(join(process.cwd(), 'scenarios', fileName), 'utf-8'), - ) as TestScenario; -} - -function record(scenarioId: string, sessionNumber: number, files: Record<string, string>): SessionRecord { - return createSessionRecord({ - id: `rec-${scenarioId}-${sessionNumber}`, - scenarioId, - sessionNumber, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - workspaceSnapshot: { - capturedAt: '2026-03-21T10:05:00Z', - files: Object.entries(files).map(([path, content]) => ({ path, content })), - }, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -/** Constructor-level validation (unique ids, session ranges) must accept both scenarios. */ -function validate(scenario: TestScenario): void { - createTestScenario({ - id: 'validate', - name: scenario.name, - initialPrompt: scenario.initialPrompt, - continuationPrompt: scenario.continuationPrompt, - sessionCount: scenario.sessionCount, - expectedFiles: scenario.expectedFiles, - retentionPatterns: scenario.retentionPatterns, - structuralAssertions: scenario.structuralAssertions, - disruptions: scenario.disruptions, - expectedJumboMemoryCaptures: scenario.expectedJumboMemoryCaptures, - jumboPlan: scenario.jumboPlan, - }); -} - -describe('null-hypothesis-rate-limiter scenario', () => { - const scenario = loadScenario('null-hypothesis-rate-limiter.json'); - - it('is single-session — the property that makes it a null-hypothesis validity check', () => { - expect(scenario.sessionCount).toBe(1); - expect(scenario.disruptions ?? []).toHaveLength(0); - // Every assertion is due in the one and only session. - for (const a of scenario.structuralAssertions ?? []) { - expect(a.sessionNumber).toBe(1); - } - }); - - it('carries the Jumbo prompt asymmetry (a goal) but no pre-seeded memory', () => { - // The goal handoff keeps the structural prompt difference under test; - // pre-seeded memory would add an information asymmetry and contaminate the null. - expect(scenario.jumboPlan?.goals).toHaveLength(1); - expect(scenario.jumboPlan?.preSeededMemory ?? []).toHaveLength(0); - }); - - it('passes constructor validation', () => { - expect(() => validate(scenario)).not.toThrow(); - }); - - it('is fully satisfied by a correct single-session implementation', () => { - const files = { - 'src/rate-limiter.ts': 'export interface RateLimiter { tryAcquire(key: string): boolean; }', - 'src/token-bucket.ts': 'export class TokenBucket implements RateLimiter { constructor(private capacity: number, private refillPerSecond: number) {} tryAcquire(key: string) { return this.refill(key); } private refill(k: string) { return true; } }', - 'src/fixed-window.ts': 'export class FixedWindow implements RateLimiter { tryAcquire(key: string) { return true; } }', - 'src/index.ts': "export { TokenBucket } from './token-bucket.js';\nexport { FixedWindow } from './fixed-window.js';\nexport type { RateLimiter } from './rate-limiter.js';", - }; - const score = scoreStructuralAssertions( - [record('null-hypothesis', 1, files)], - scenario.structuralAssertions as readonly StructuralAssertion[], - ); - expect(score.score).toBe(1); - }); - - it('is unsatisfied by an empty implementation', () => { - const score = scoreStructuralAssertions( - [record('null-hypothesis', 1, {})], - scenario.structuralAssertions as readonly StructuralAssertion[], - ); - expect(score.score).toBe(0); - }); -}); - -describe('file-reconstruction-expense-tracker scenario', () => { - const scenario = loadScenario('file-reconstruction-expense-tracker.json'); - - it('is multi-session with decisions legible in files (DECISIONS.md assertions)', () => { - expect(scenario.sessionCount).toBeGreaterThan(1); - const decisionLogAssertions = (scenario.structuralAssertions ?? []).filter( - (a) => a.file === 'DECISIONS.md', - ); - // The decision log is asserted at the start, after a disruption, and at the end — - // the in-file recovery path that distinguishes this scenario. - expect(decisionLogAssertions.length).toBeGreaterThanOrEqual(3); - }); - - it('pre-seeded memory only restates prompt content (no information asymmetry beyond the treatment)', () => { - // Guard the scenario's core property: both entries must reference what the - // shared prompt already mandates (layering / decision log), so session 1 - // starts information-symmetric across arms. - const entries = scenario.jumboPlan?.preSeededMemory ?? []; - expect(entries.length).toBeGreaterThan(0); - for (const entry of entries) { - const text = JSON.stringify(entry).toLowerCase(); - expect(text.includes('prompt')).toBe(true); - } - }); - - it('passes constructor validation', () => { - expect(() => validate(scenario)).not.toThrow(); - }); - - it('is fully satisfied by a correct cumulative implementation', () => { - const domainV1 = 'export interface Expense { id: string; description: string; amount: number; category: string; date: string; }'; - const domainMinorUnits = 'export interface Expense { id: string; description: string; amountMinorUnits: number; category: string; date: string; } // amounts in cents'; - const store = 'export interface ExpenseStore { add(e: Expense): void; list(): Expense[]; }\nexport class MemoryStore implements ExpenseStore { add(e: Expense) {} list() { return []; } }'; - const serviceV1 = 'export class ExpenseService { constructor(private store: ExpenseStore) {} }'; - const serviceAudited = 'export interface AuditEntry { at: string; operation: string; expenseId: string; }\nexport class ExpenseService { private auditLog: AuditEntry[] = []; constructor(private store: ExpenseStore) {} }'; - const decisionsV1 = '# Decisions\n\n2026-01-01: Layered architecture; service depends on the ExpenseStore interface.'; - const decisionsV3 = decisionsV1 + '\n2026-01-02: Store amounts as integer minor units (cents), never floats.'; - const decisionsV5 = decisionsV3 + '\n2026-01-03: Every mutation appends an AuditEntry to the audit log.'; - - const records = [ - record('file-recon', 1, { - 'src/domain/expense.ts': domainV1, - 'src/storage/memory-store.ts': store, - 'src/services/expense-service.ts': serviceV1, - 'DECISIONS.md': decisionsV1, - }), - record('file-recon', 3, { - 'src/domain/expense.ts': domainMinorUnits, - 'src/storage/memory-store.ts': store, - 'src/services/expense-service.ts': serviceV1, - 'DECISIONS.md': decisionsV3, - }), - record('file-recon', 5, { - 'src/domain/expense.ts': domainMinorUnits, - 'src/storage/memory-store.ts': store, - 'src/services/expense-service.ts': serviceAudited, - 'DECISIONS.md': decisionsV5, - }), - ]; - const score = scoreStructuralAssertions( - records, - scenario.structuralAssertions as readonly StructuralAssertion[], - ); - expect(score.score).toBe(1); - }); - - it('is unsatisfied by an empty implementation', () => { - const records = [1, 3, 5].map((s) => record('file-recon', s, {})); - const score = scoreStructuralAssertions( - records, - scenario.structuralAssertions as readonly StructuralAssertion[], - ); - expect(score.score).toBe(0); - }); -}); diff --git a/evals/tests/unit/structural-assertion-scorer.test.ts b/evals/tests/unit/structural-assertion-scorer.test.ts deleted file mode 100644 index 274f554c..00000000 --- a/evals/tests/unit/structural-assertion-scorer.test.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { - scoreStructuralAssertions, - scoreStructuralAssertionsTimeline, -} from '../../src/scoring/structural-assertion-scorer.js'; -import { createSessionRecord } from '../../src/domain/index.js'; -import type { SessionRecord, StructuralAssertion, WorkspaceSnapshot } from '../../src/domain/index.js'; - -function makeSnapshot(files: Record<string, string>): WorkspaceSnapshot { - return { - capturedAt: '2026-03-21T10:05:00Z', - files: Object.entries(files).map(([path, content]) => ({ path, content })), - }; -} - -function makeRecord(sessionNumber: number, files?: Record<string, string>): SessionRecord { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'scenario-1', - sessionNumber, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - workspaceSnapshot: files ? makeSnapshot(files) : undefined, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -describe('scoreStructuralAssertions', () => { - it('returns trivial score when no assertions are defined', () => { - const score = scoreStructuralAssertions([makeRecord(1)], []); - expect(score.dimension).toBe('structural-retention'); - expect(score.score).toBe(1); - expect(score.maxScore).toBe(1); - expect(score.details).toContain('No structural assertions'); - }); - - it('passes a fileExists assertion when a matching file is present in the due session', () => { - const assertions: StructuralAssertion[] = [ - { id: 'a1', file: 'src/events/types.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ]; - const records = [makeRecord(1, { 'src/events/types.ts': 'export type X = 1;' })]; - - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(1); - }); - - it('fails a fileExists assertion when no file matches the glob', () => { - const assertions: StructuralAssertion[] = [ - { id: 'a1', file: 'src/events/types.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ]; - const records = [makeRecord(1, { 'src/other.ts': 'noop' })]; - - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(0); - expect(score.details).toContain('a1'); - }); - - it('evaluates matchesRegex against the matched file content', () => { - const assertions: StructuralAssertion[] = [ - { - id: 'discriminated-union', - file: 'src/events/types.ts', - sessionNumber: 1, - matcher: { kind: 'matchesRegex', pattern: "type\\s+DomainEvent\\s*=" }, - }, - ]; - const pass = scoreStructuralAssertions( - [makeRecord(1, { 'src/events/types.ts': 'export type DomainEvent = A | B;' })], - assertions, - ); - expect(pass.score).toBe(1); - - const fail = scoreStructuralAssertions( - [makeRecord(1, { 'src/events/types.ts': 'export interface DomainEvent {}' })], - assertions, - ); - expect(fail.score).toBe(0); - }); - - it('honours regex flags', () => { - const assertions: StructuralAssertion[] = [ - { - id: 'a1', - file: 'src/a.ts', - sessionNumber: 1, - matcher: { kind: 'matchesRegex', pattern: 'CONCURRENCYERROR', flags: 'i' }, - }, - ]; - const score = scoreStructuralAssertions( - [makeRecord(1, { 'src/a.ts': 'class ConcurrencyError extends Error {}' })], - assertions, - ); - expect(score.score).toBe(1); - }); - - it('containsAll passes only when every substring is present across matched files', () => { - const assertions: StructuralAssertion[] = [ - { - id: 'metadata', - file: 'src/**/*.ts', - sessionNumber: 1, - matcher: { kind: 'containsAll', substrings: ['correlationId', 'causationId', 'timestamp'] }, - }, - ]; - const pass = scoreStructuralAssertions( - [makeRecord(1, { - 'src/events/types.ts': 'interface Meta { correlationId: string; causationId: string; }', - 'src/events/clock.ts': 'const timestamp = Date.now();', - })], - assertions, - ); - expect(pass.score).toBe(1); - - const fail = scoreStructuralAssertions( - [makeRecord(1, { 'src/events/types.ts': 'interface Meta { correlationId: string; }' })], - assertions, - ); - expect(fail.score).toBe(0); - expect(fail.details).toContain('causationId'); - }); - - it('containsAny passes when at least one substring is present', () => { - const assertions: StructuralAssertion[] = [ - { - id: 'a1', - file: 'src/a.ts', - sessionNumber: 1, - matcher: { kind: 'containsAny', substrings: ['ReservationConfirmed', 'ReservationCancelled'] }, - }, - ]; - const score = scoreStructuralAssertions( - [makeRecord(1, { 'src/a.ts': 'case "ReservationCancelled": return;' })], - assertions, - ); - expect(score.score).toBe(1); - }); - - it('notContains fails when a forbidden substring is present', () => { - const assertions: StructuralAssertion[] = [ - { - id: 'no-any', - file: 'src/a.ts', - sessionNumber: 1, - matcher: { kind: 'notContains', substrings: [': any'] }, - }, - ]; - const fail = scoreStructuralAssertions( - [makeRecord(1, { 'src/a.ts': 'const x: any = 1;' })], - assertions, - ); - expect(fail.score).toBe(0); - - const pass = scoreStructuralAssertions( - [makeRecord(1, { 'src/a.ts': 'const x: number = 1;' })], - assertions, - ); - expect(pass.score).toBe(1); - }); - - it('exportsSymbol detects an exported declaration', () => { - const assertions: StructuralAssertion[] = [ - { id: 'a1', file: 'src/store.ts', sessionNumber: 1, matcher: { kind: 'exportsSymbol', symbol: 'EventStore' } }, - ]; - const cls = scoreStructuralAssertions( - [makeRecord(1, { 'src/store.ts': 'export class EventStore {}' })], - assertions, - ); - expect(cls.score).toBe(1); - - const named = scoreStructuralAssertions( - [makeRecord(1, { 'src/store.ts': 'class EventStore {}\nexport { EventStore };' })], - assertions, - ); - expect(named.score).toBe(1); - - const missing = scoreStructuralAssertions( - [makeRecord(1, { 'src/store.ts': 'class Foo {}' })], - assertions, - ); - expect(missing.score).toBe(0); - }); - - it('evaluates each assertion against the session it is due in, not the latest', () => { - const assertions: StructuralAssertion[] = [ - { id: 's1', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'matchesRegex', pattern: 'foundation' } }, - { id: 's3', file: 'src/a.ts', sessionNumber: 3, matcher: { kind: 'matchesRegex', pattern: 'metadata' } }, - ]; - const records = [ - makeRecord(1, { 'src/a.ts': 'foundation only' }), - makeRecord(2, { 'src/a.ts': 'foundation only' }), - makeRecord(3, { 'src/a.ts': 'foundation with metadata' }), - ]; - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(1); // both pass in their respective due sessions - }); - - it('fails an assertion whose due session has no record', () => { - const assertions: StructuralAssertion[] = [ - { id: 'a5', file: 'src/a.ts', sessionNumber: 5, matcher: { kind: 'fileExists' } }, - ]; - const records = [makeRecord(1, { 'src/a.ts': 'x' })]; - const score = scoreStructuralAssertions(records, assertions); - expect(score.score).toBe(0); - expect(score.details).toContain('a5'); - }); - - it('matches nested paths with ** glob', () => { - const assertions: StructuralAssertion[] = [ - { id: 'a1', file: 'src/**/handlers.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ]; - const score = scoreStructuralAssertions( - [makeRecord(1, { 'src/commands/handlers.ts': 'x' })], - assertions, - ); - expect(score.score).toBe(1); - }); - - it('aggregate score is the fraction of assertions that pass', () => { - const assertions: StructuralAssertion[] = [ - { id: 'p', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 'f', file: 'src/missing.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - ]; - const score = scoreStructuralAssertions( - [makeRecord(1, { 'src/a.ts': 'x' })], - assertions, - ); - expect(score.score).toBe(0.5); - }); -}); - -describe('scoreStructuralAssertionsTimeline', () => { - it('returns empty array when no assertions defined', () => { - expect(scoreStructuralAssertionsTimeline([makeRecord(1)], [])).toEqual([]); - }); - - it('produces a per-session score equal to the fraction of assertions due that session', () => { - const assertions: StructuralAssertion[] = [ - { id: 's1a', file: 'src/a.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 's1b', file: 'src/b.ts', sessionNumber: 1, matcher: { kind: 'fileExists' } }, - { id: 's2', file: 'src/c.ts', sessionNumber: 2, matcher: { kind: 'fileExists' } }, - ]; - const records = [ - makeRecord(1, { 'src/a.ts': 'x' }), // s1a passes, s1b fails -> 0.5 - makeRecord(2, { 'src/c.ts': 'x' }), // s2 passes -> 1 - ]; - const timeline = scoreStructuralAssertionsTimeline(records, assertions); - expect(timeline).toHaveLength(2); - expect(timeline[0].score).toBe(0.5); - expect(timeline[0].details).toContain('session 1'); - expect(timeline[1].score).toBe(1); - }); - - it('omits sessions that have no assertions due', () => { - const assertions: StructuralAssertion[] = [ - { id: 's2', file: 'src/c.ts', sessionNumber: 2, matcher: { kind: 'fileExists' } }, - ]; - const records = [makeRecord(1, {}), makeRecord(2, { 'src/c.ts': 'x' })]; - const timeline = scoreStructuralAssertionsTimeline(records, assertions); - expect(timeline).toHaveLength(1); - expect(timeline[0].details).toContain('session 2'); - }); -}); diff --git a/evals/tests/unit/tamper-provenance.test.ts b/evals/tests/unit/tamper-provenance.test.ts deleted file mode 100644 index b72326b7..00000000 --- a/evals/tests/unit/tamper-provenance.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { mergeTamperProvenance } from '../../src/domain/tamper-provenance.js'; -import { createTestResult, createComparisonResult } from '../../src/domain/index.js'; -import type { SessionRecord, TamperEvent } from '../../src/domain/index.js'; - -function event(action: TamperEvent['action'], operator: string): TamperEvent { - return { occurredAt: '2026-03-21T10:00:00Z', action, operator }; -} - -describe('mergeTamperProvenance', () => { - it('is untampered with empty seed and no sources', () => { - const p = mergeTamperProvenance({}, []); - expect(p.tampered).toBe(false); - expect(p.tamperLog).toEqual([]); - }); - - it('is tampered when the seed says so', () => { - const p = mergeTamperProvenance({ tampered: true, tamperLog: [event('pause', 'op')] }, []); - expect(p.tampered).toBe(true); - expect(p.tamperLog).toHaveLength(1); - }); - - it('is tampered when any source is tampered', () => { - const p = mergeTamperProvenance({}, [ - { tampered: false, tamperLog: [] }, - { tampered: true, tamperLog: [event('abort', 'op')] }, - ]); - expect(p.tampered).toBe(true); - }); - - it('concatenates seed log then each source log in order', () => { - const p = mergeTamperProvenance({ tamperLog: [event('pause', 'seed')] }, [ - { tampered: true, tamperLog: [event('resume', 'a')] }, - { tampered: false, tamperLog: [event('inject-context', 'b')] }, - ]); - expect(p.tamperLog.map((e) => e.operator)).toEqual(['seed', 'a', 'b']); - }); -}); - -describe('result factories with an injectable clock', () => { - const fixedClock = () => '2000-01-01T00:00:00Z'; - - it('createTestResult stamps createdAt from the provided clock', () => { - const result = createTestResult( - { id: 'r1', scenarioId: 's1', harness: 'claude-code', sessionRecords: [] }, - fixedClock, - ); - expect(result.createdAt).toBe('2000-01-01T00:00:00Z'); - expect(result.tampered).toBe(false); - }); - - it('createTestResult folds session tamper provenance into the result', () => { - const tamperedSession = { - tampered: true, - tamperLog: [event('abort', 'op')], - } as unknown as SessionRecord; - const result = createTestResult( - { id: 'r1', scenarioId: 's1', harness: 'claude-code', sessionRecords: [tamperedSession] }, - fixedClock, - ); - expect(result.tampered).toBe(true); - expect(result.tamperLog).toHaveLength(1); - }); - - it('createComparisonResult stamps createdAt from the provided clock', () => { - const arm = createTestResult( - { id: 'a', scenarioId: 's1', harness: 'claude-code', sessionRecords: [] }, - fixedClock, - ); - const cmp = createComparisonResult( - { - id: 'c1', - scenarioId: 's1', - harness: 'claude-code', - jumboResult: arm, - baselineResult: arm, - jumboScores: [], - baselineScores: [], - deltas: [], - }, - fixedClock, - ); - expect(cmp.createdAt).toBe('2000-01-01T00:00:00Z'); - }); -}); diff --git a/evals/tests/unit/tamper-quarantine.test.ts b/evals/tests/unit/tamper-quarantine.test.ts deleted file mode 100644 index 987447b1..00000000 --- a/evals/tests/unit/tamper-quarantine.test.ts +++ /dev/null @@ -1,364 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { runABComparison, TamperAbortError } from '../../src/ab-runner.js'; -import { createTestScenario } from '../../src/domain/types.js'; -import type { - ResultStore, -} from '../../src/storage/result-store.js'; -import type { - RunControlFile, - SessionRecord, - TestResult, - EvalRunRecord, - RunHeartbeat, - TestScenario, - TamperEvent, -} from '../../src/domain/types.js'; -import type { LocalExecutor } from '../../src/infrastructure/local-executor.js'; -import type { ExecResult } from '../../src/infrastructure/container-manager.js'; -import type { HarnessAdapter } from '../../src/harness/harness-adapter.js'; - -interface MemoryStore extends ResultStore { - readonly sessionRecords: SessionRecord[]; - readonly testResults: TestResult[]; - readonly control: Map<string, RunControlFile>; -} - -function memoryStore(initialControl?: RunControlFile): MemoryStore { - const sessionRecords: SessionRecord[] = []; - const testResults: TestResult[] = []; - const control = new Map<string, RunControlFile>(); - if (initialControl) control.set(initialControl.runId, initialControl); - const runRecords: EvalRunRecord[] = []; - const heartbeats = new Map<string, RunHeartbeat>(); - return { - sessionRecords, - testResults, - control, - saveScenario: async () => {}, - getScenario: async () => null, - listScenarios: async () => [], - saveSessionRecord: async (r) => { - const idx = sessionRecords.findIndex((x) => x.id === r.id); - if (idx >= 0) sessionRecords[idx] = r; - else sessionRecords.push(r); - }, - getSessionRecords: async () => sessionRecords, - saveTestResult: async (r) => { testResults.push(r); }, - getTestResult: async () => null, - listTestResults: async () => testResults, - saveRunRecord: async (r) => { runRecords.push(r); }, - getRunRecord: async () => runRecords.at(-1) ?? null, - listRunRecords: async () => runRecords, - writeHeartbeat: async (runId, hb) => { heartbeats.set(runId, hb); }, - readHeartbeat: async (runId) => heartbeats.get(runId) ?? null, - writeRunControl: async (runId, file) => { control.set(runId, file); }, - readRunControl: async (runId) => control.get(runId) ?? null, - }; -} - -function execMock(): LocalExecutor & { calls: { workDir: string; command: string[] }[] } { - const calls: { workDir: string; command: string[] }[] = []; - let dir = 0; - return { - calls, - createWorkDir: async (p?: string) => { dir++; return `/tmp/${p ?? 'd-'}${dir}`; }, - cleanup: async () => {}, - captureWorkspaceSnapshot: async () => ({ capturedAt: new Date().toISOString(), files: [] }), - installJumboShim: async (workDir: string) => ({ env: { PATH: `${workDir}/.eval-bin:/usr/bin` } }), - exec: async (workDir: string, command: string[], options?: { env?: Record<string, string | undefined> }) => { - calls.push({ workDir, command }); - if (command[0] === 'jumbo' && command[1] === '--version') { - const shimmed = options?.env?.PATH?.includes('.eval-bin') ?? false; - return shimmed - ? { stdout: '', stderr: 'shim', exitCode: 127 } - : ok('jumbo 1.2.3'); - } - if (command[0] === 'jumbo' && command[1] === 'init') return ok('initialized'); - if (command[0] === 'jumbo' && command[1] === 'goal' && command[2] === 'show') { - return ok(JSON.stringify({ goal: { status: 'submitted' } })); - } - if (command[0] === 'jumbo' && command[1] === 'sessions' && command[2] === 'list') { - return ok('[]'); - } - if (command[0] === 'jumbo' && command[3] === '--format' && command[4] === 'json') { - return ok('[]'); - } - return ok(JSON.stringify({ result: 'done', files_modified: ['src/x.ts'] })); - }, - } as unknown as LocalExecutor & { calls: { workDir: string; command: string[] }[] }; -} - -function ok(stdout: string): ExecResult { - return { stdout, stderr: '', exitCode: 0 }; -} - -function adapterMock(): HarnessAdapter { - return { - name: 'mock-h', - buildCommand: () => ['mock'], - parseOutput: (r: ExecResult) => { - try { - const p = JSON.parse(r.stdout); - return { agentOutput: p.result, filesModified: p.files_modified ?? [], transcript: r.stdout }; - } catch { - return { agentOutput: r.stdout, filesModified: [], transcript: r.stdout }; - } - }, - seedToolPermissions: async () => {}, - }; -} - -function scenario(sessionCount: number, opts: Partial<TestScenario> = {}): TestScenario { - return createTestScenario({ - id: 's1', - name: 'tamper test', - initialPrompt: 'do work', - continuationPrompt: 'continue', - sessionCount, - expectedFiles: ['src/x.ts'], - ...opts, - }); -} - -function controlFile(runId: string, partial: Partial<RunControlFile>): RunControlFile { - return { - runId, - updatedAt: new Date().toISOString(), - pendingActions: [], - pauseRequested: false, - abortRequested: false, - ...partial, - }; -} - -function tamperEvent(action: TamperEvent['action'], extra: Partial<TamperEvent> = {}): TamperEvent { - return { occurredAt: new Date().toISOString(), action, ...extra }; -} - -describe('tamper quarantine', () => { - it('abort halts before scoring and never writes a ComparisonResult', async () => { - const runId = 'run-abort'; - const store = memoryStore(controlFile(runId, { - pendingActions: [tamperEvent('abort', { operator: 'alice' })], - abortRequested: true, - })); - await expect(runABComparison({ - scenario: scenario(3), - adapter: adapterMock(), - executor: execMock(), - store, - runId, - })).rejects.toBeInstanceOf(TamperAbortError); - expect(store.testResults).toHaveLength(0); - }); - - it('abort taints already-saved SessionRecords with abort event in the last record', async () => { - const runId = 'run-abort-mid'; - const store = memoryStore(); - let firstSessionDone = false; - const executor = execMock(); - const originalExec = executor.exec.bind(executor); - executor.exec = async (workDir, command, options) => { - const r = await originalExec(workDir, command, options); - // After first jumbo session ends, plant an abort - if (!firstSessionDone && command[0] === 'mock' && workDir.includes('jumbo-eval-jumbo-')) { - firstSessionDone = true; - await store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('abort')], - abortRequested: true, - })); - } - return r; - }; - - await expect(runABComparison({ - scenario: scenario(3), - adapter: adapterMock(), - executor, - store, - runId, - })).rejects.toBeInstanceOf(TamperAbortError); - - const jumbo = store.sessionRecords.filter((r) => r.variant === 'jumbo'); - expect(jumbo.length).toBeGreaterThanOrEqual(1); - expect(jumbo.every((r) => r.tampered)).toBe(true); - const lastJumbo = jumbo[jumbo.length - 1]; - expect(lastJumbo.tamperLog.some((e) => e.action === 'abort')).toBe(true); - expect(store.testResults).toHaveLength(0); - }); - - it('inject-context appends only to the next jumbo session deliveredContext', async () => { - const runId = 'run-inject'; - const store = memoryStore(); - const executor = execMock(); - let firstSessionEnded = false; - const originalExec = executor.exec.bind(executor); - executor.exec = async (workDir, command, options) => { - const r = await originalExec(workDir, command, options); - if (!firstSessionEnded && command[0] === 'mock' && workDir.includes('jumbo-eval-jumbo-')) { - firstSessionEnded = true; - await store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('inject-context', { - payload: 'remember edge case Q', - variant: 'jumbo', - })], - })); - } - return r; - }; - - await runABComparison({ - scenario: scenario(2), - adapter: adapterMock(), - executor, - store, - runId, - }); - - const jumbo = store.sessionRecords.filter((r) => r.variant === 'jumbo').sort((a, b) => a.sessionNumber - b.sessionNumber); - const baseline = store.sessionRecords.filter((r) => r.variant === 'baseline').sort((a, b) => a.sessionNumber - b.sessionNumber); - - expect(jumbo[0].deliveredContext ?? '').not.toContain('edge case Q'); - expect(jumbo[1].deliveredContext ?? '').toContain('OPERATOR-INJECTED CONTEXT'); - expect(jumbo[1].deliveredContext ?? '').toContain('remember edge case Q'); - // scenarioPrompt unchanged - expect(jumbo[1].scenarioPrompt).toBe('continue'); - // baseline never sees the injection - for (const b of baseline) { - expect(b.deliveredContext).toBeUndefined(); - } - }); - - it('tamper taint propagates forward to all later sessions in the variant', async () => { - const runId = 'run-taint'; - const store = memoryStore(); - const executor = execMock(); - let firstEnded = false; - const originalExec = executor.exec.bind(executor); - executor.exec = async (workDir, command, options) => { - const r = await originalExec(workDir, command, options); - if (!firstEnded && command[0] === 'mock' && workDir.includes('jumbo-eval-jumbo-')) { - firstEnded = true; - await store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('inject-context', { - payload: 'taint', - variant: 'jumbo', - })], - })); - } - return r; - }; - - await runABComparison({ - scenario: scenario(3), - adapter: adapterMock(), - executor, - store, - runId, - }); - - const jumbo = store.sessionRecords - .filter((r) => r.variant === 'jumbo') - .sort((a, b) => a.sessionNumber - b.sessionNumber); - const baseline = store.sessionRecords - .filter((r) => r.variant === 'baseline') - .sort((a, b) => a.sessionNumber - b.sessionNumber); - - expect(jumbo[0].tampered).toBe(false); - expect(jumbo[1].tampered).toBe(true); - expect(jumbo[2].tampered).toBe(true); - expect(jumbo[1].tamperLog).toHaveLength(1); - expect(jumbo[2].tamperLog).toHaveLength(0); // event recorded only on the immediate next; taint flag is sticky - // baseline is untouched - expect(baseline.every((r) => !r.tampered)).toBe(true); - }); - - it('TestResult and ComparisonResult are tampered when any session is tampered', async () => { - const runId = 'run-cmp'; - const store = memoryStore(); - const executor = execMock(); - let firstEnded = false; - const originalExec = executor.exec.bind(executor); - executor.exec = async (workDir, command, options) => { - const r = await originalExec(workDir, command, options); - if (!firstEnded && command[0] === 'mock' && workDir.includes('jumbo-eval-jumbo-')) { - firstEnded = true; - await store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('inject-context', { - payload: 'q', - variant: 'jumbo', - })], - })); - } - return r; - }; - - const cmp = await runABComparison({ - scenario: scenario(2), - adapter: adapterMock(), - executor, - store, - runId, - }); - expect(cmp.tampered).toBe(true); - expect(cmp.tamperLog.length).toBeGreaterThan(0); - expect(cmp.jumboResult.tampered).toBe(true); - expect(cmp.baselineResult.tampered).toBe(false); - }); - - it('pause blocks at the session boundary until resume clears pauseRequested', async () => { - const runId = 'run-pause'; - const store = memoryStore(controlFile(runId, { - pendingActions: [tamperEvent('pause')], - pauseRequested: true, - })); - setTimeout(() => { - void store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('resume')], - pauseRequested: false, - })); - }, 1200); - - const start = Date.now(); - await runABComparison({ - scenario: scenario(1), - adapter: adapterMock(), - executor: execMock(), - store, - runId, - }); - const elapsed = Date.now() - start; - expect(elapsed).toBeGreaterThanOrEqual(1000); - const jumbo = store.sessionRecords.filter((r) => r.variant === 'jumbo'); - expect(jumbo[0].tampered).toBe(true); - expect(jumbo[0].tamperLog.some((e) => e.action === 'pause')).toBe(true); - expect(jumbo[0].tamperLog.some((e) => e.action === 'resume')).toBe(true); - }, 15000); - - it('aggregate scoring excludes tampered jumbo records by default', async () => { - const runId = 'run-score'; - const store = memoryStore(); - const executor = execMock(); - // Pre-plant inject-context so session 2 will be tainted - await store.writeRunControl(runId, controlFile(runId, { - pendingActions: [tamperEvent('inject-context', { - payload: 'q', - variant: 'jumbo', - })], - })); - // The control fires before session 1; that taints session 1+. Use 1-session scenario: - const cmp = await runABComparison({ - scenario: scenario(1), - adapter: adapterMock(), - executor, - store, - runId, - }); - // Only jumbo session is tainted; baseline is not. file-accuracy: baseline modified expected file -> 1.0; jumbo records are filtered out -> 0 (no records vs expected). - const fileAcc = cmp.deltas.find((d) => d.dimension === 'file-accuracy'); - expect(fileAcc).toBeDefined(); - // jumbo aggregate excluded -> jumbo score 0; baseline score 1 - expect(cmp.jumboScores.find((s) => s.dimension === 'file-accuracy')?.score).toBe(0); - expect(cmp.baselineScores.find((s) => s.dimension === 'file-accuracy')?.score).toBe(1); - }); -}); diff --git a/evals/tests/unit/token-efficiency-scorer.test.ts b/evals/tests/unit/token-efficiency-scorer.test.ts deleted file mode 100644 index 6380ff21..00000000 --- a/evals/tests/unit/token-efficiency-scorer.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { scoreTokenEfficiency, compareTokenEfficiency, tokenUsageTimeline } from '../../src/scoring/token-efficiency-scorer.js'; -import { createSessionRecord } from '../../src/domain/types.js'; - -function makeRecord(sessionNumber: number, inputTokens?: number, outputTokens?: number) { - return createSessionRecord({ - id: `rec-${sessionNumber}`, - scenarioId: 'scenario-1', - sessionNumber, - harness: 'claude-code', - agentOutput: '', - filesModified: [], - transcript: '', - inputTokens, - outputTokens, - startedAt: '2026-03-21T10:00:00Z', - completedAt: '2026-03-21T10:05:00Z', - }); -} - -describe('scoreTokenEfficiency', () => { - it('computes total tokens and tokens per quality point', () => { - const records = [ - makeRecord(1, 1000, 500), - makeRecord(2, 800, 400), - ]; - - const score = scoreTokenEfficiency(records, 0.8); - expect(score.dimension).toBe('token-efficiency'); - expect(score.score).toBe(2700); // 1000+500+800+400 - expect(score.details).toContain('2700 total tokens'); - expect(score.details).toContain('1800 in'); - expect(score.details).toContain('900 out'); - expect(score.details).toContain('3375 tokens/quality-point'); // 2700/0.8 - }); - - it('returns zero score when no token data', () => { - const records = [makeRecord(1)]; - const score = scoreTokenEfficiency(records, 0.8); - expect(score.score).toBe(0); - expect(score.details).toContain('No token data'); - }); -}); - -describe('compareTokenEfficiency', () => { - it('returns positive when Jumbo uses fewer tokens per quality point', () => { - const jumboRecords = [makeRecord(1, 500, 200)]; // 700 tokens, quality 0.8 → 875 tpq - const baselineRecords = [makeRecord(1, 1000, 500)]; // 1500 tokens, quality 0.8 → 1875 tpq - - const score = compareTokenEfficiency(jumboRecords, baselineRecords, 0.8, 0.8); - expect(score.score).toBeGreaterThan(0); - expect(score.details).toContain('jumbo: 700 tokens'); - expect(score.details).toContain('baseline: 1500 tokens'); - }); - - it('returns negative when baseline is more efficient', () => { - const jumboRecords = [makeRecord(1, 2000, 1000)]; - const baselineRecords = [makeRecord(1, 500, 200)]; - - const score = compareTokenEfficiency(jumboRecords, baselineRecords, 0.8, 0.8); - expect(score.score).toBeLessThan(0); - }); - - it('returns zero when both have no token data', () => { - const score = compareTokenEfficiency([makeRecord(1)], [makeRecord(1)], 0.5, 0.5); - expect(score.score).toBe(0); - }); - - it('accounts for quality differences', () => { - // Jumbo: 1000 tokens, quality 1.0 → 1000 tpq - // Baseline: 1000 tokens, quality 0.5 → 2000 tpq - // Jumbo is more efficient per quality point - const jumboRecords = [makeRecord(1, 600, 400)]; - const baselineRecords = [makeRecord(1, 600, 400)]; - - const score = compareTokenEfficiency(jumboRecords, baselineRecords, 1.0, 0.5); - expect(score.score).toBeGreaterThan(0); - }); -}); - -describe('tokenUsageTimeline', () => { - it('produces per-session token usage', () => { - const records = [ - makeRecord(1, 1000, 500), - makeRecord(2, 800, 400), - makeRecord(3, 600, 300), - ]; - - const timeline = tokenUsageTimeline(records); - expect(timeline).toHaveLength(3); - expect(timeline[0].score).toBe(1500); - expect(timeline[1].score).toBe(1200); - expect(timeline[2].score).toBe(900); - expect(timeline[0].dimension).toBe('token-usage'); - }); - - it('handles missing token data gracefully', () => { - const records = [makeRecord(1), makeRecord(2, 100, 50)]; - const timeline = tokenUsageTimeline(records); - expect(timeline[0].score).toBe(0); - expect(timeline[1].score).toBe(150); - }); -}); diff --git a/evals/tests/unit/watch-status.test.ts b/evals/tests/unit/watch-status.test.ts deleted file mode 100644 index 1f90c2a2..00000000 --- a/evals/tests/unit/watch-status.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { describe, it, expect } from '@jest/globals'; -import { watchRunStatus, createWatchAppElement } from '../../src/cli/commands/status.js'; -import type { ResultStore } from '../../src/storage/result-store.js'; -import type { EvalRunRecord, RunHeartbeat, SessionRecord, TestResult, TestScenario } from '../../src/domain/types.js'; - -function buildHeartbeat(overrides: Partial<RunHeartbeat> = {}): RunHeartbeat { - return { - runId: 'run-1', - scenarioId: 'scen-1', - updatedAt: '2026-04-29T10:00:00.000Z', - harnesses: [{ - harness: 'claude-code', - variant: 'jumbo', - sessions: [{ - sessionNumber: 1, - status: 'completed', - startedAt: '2026-04-29T10:00:00.000Z', - completedAt: '2026-04-29T10:00:01.000Z', - }], - }], - ...overrides, - }; -} - -function buildRunningHeartbeat(): RunHeartbeat { - return buildHeartbeat({ - harnesses: [{ - harness: 'claude-code', - variant: 'jumbo', - sessions: [{ sessionNumber: 1, status: 'running', phase: 'harness-exec', startedAt: new Date().toISOString() }], - }], - }); -} - -class FakeStore implements ResultStore { - heartbeatQueue: Array<RunHeartbeat | null> = []; - runRecord: EvalRunRecord | null = null; - readCount = 0; - - async saveScenario(): Promise<void> {} - async getScenario(): Promise<TestScenario | null> { return null; } - async listScenarios(): Promise<TestScenario[]> { return []; } - async saveSessionRecord(): Promise<void> {} - async getSessionRecords(): Promise<SessionRecord[]> { return []; } - async saveTestResult(): Promise<void> {} - async getTestResult(): Promise<TestResult | null> { return null; } - async listTestResults(): Promise<TestResult[]> { return []; } - async saveRunRecord(): Promise<void> {} - async getRunRecord(): Promise<EvalRunRecord | null> { return this.runRecord; } - async listRunRecords(): Promise<EvalRunRecord[]> { return []; } - async writeHeartbeat(): Promise<void> {} - async readHeartbeat(): Promise<RunHeartbeat | null> { - this.readCount += 1; - if (this.heartbeatQueue.length === 0) return null; - if (this.heartbeatQueue.length === 1) return this.heartbeatQueue[0]; - return this.heartbeatQueue.shift() ?? null; - } -} - -async function captureLog(fn: () => Promise<void>): Promise<string[]> { - const messages: string[] = []; - const original = console.log; - console.log = (...args: unknown[]): void => { messages.push(args.map(String).join(' ')); }; - try { - await fn(); - } finally { - console.log = original; - } - return messages; -} - -describe('watchRunStatus non-interactive', () => { - it('prints a single final snapshot when the run is already complete', async () => { - const store = new FakeStore(); - store.heartbeatQueue = [buildHeartbeat()]; - const messages = await captureLog(() => watchRunStatus(store, 'run-1', { isInteractive: false, pollIntervalMs: 1 })); - expect(messages).toHaveLength(1); - expect(messages[0]).toContain('Run: run-1'); - }); - - it('prints "no heartbeat" exactly once when none exists', async () => { - const store = new FakeStore(); - const messages = await captureLog(() => watchRunStatus(store, 'missing-run', { isInteractive: false, pollIntervalMs: 1 })); - expect(messages).toHaveLength(1); - expect(messages[0]).toContain('No heartbeat found for run: missing-run'); - }); - - it('polls until completion then prints exactly one snapshot', async () => { - const store = new FakeStore(); - store.heartbeatQueue = [buildRunningHeartbeat(), buildRunningHeartbeat(), buildHeartbeat()]; - const messages = await captureLog(() => watchRunStatus(store, 'run-1', { isInteractive: false, pollIntervalMs: 1 })); - expect(store.readCount).toBeGreaterThanOrEqual(3); - expect(messages).toHaveLength(1); - }); -}); - -describe('watchRunStatus interactive (ink)', () => { - it('renders the heartbeat snapshot in-place via ink', async () => { - const ink = await import('ink'); - const React = await import('react'); - const { render } = await import('ink-testing-library'); - - const store = new FakeStore(); - store.heartbeatQueue = [buildHeartbeat()]; - - const element = createWatchAppElement(React, ink, { store, runId: 'run-1', pollIntervalMs: 1 }); - const instance = render(element); - - await waitFor(() => instance.frames.some((f) => f.includes('Run: run-1'))); - - const matched = instance.frames.find((f) => f.includes('Run: run-1')) ?? ''; - expect(matched).toContain('Run: run-1'); - expect(matched).toContain('Sessions'); - instance.unmount(); - }); - - it('renders the missing-heartbeat message when no heartbeat is found', async () => { - const ink = await import('ink'); - const React = await import('react'); - const { render } = await import('ink-testing-library'); - - const store = new FakeStore(); - const element = createWatchAppElement(React, ink, { store, runId: 'missing-run', pollIntervalMs: 1 }); - const instance = render(element); - - await waitFor(() => instance.frames.some((f) => f.includes('No heartbeat found for run: missing-run'))); - - const matched = instance.frames.find((f) => f.includes('No heartbeat found for run: missing-run')) ?? ''; - expect(matched).toContain('No heartbeat found for run: missing-run'); - instance.unmount(); - }); - - it('updates frames in place across polls and unmounts on completion', async () => { - const ink = await import('ink'); - const React = await import('react'); - const { render } = await import('ink-testing-library'); - - const store = new FakeStore(); - store.heartbeatQueue = [buildRunningHeartbeat(), buildRunningHeartbeat(), buildHeartbeat()]; - - const element = createWatchAppElement(React, ink, { store, runId: 'run-1', pollIntervalMs: 1 }); - const instance = render(element); - - await waitFor(() => instance.frames.some((f) => f.includes('completed') && f.includes('Run: run-1'))); - - for (const frame of instance.frames) { - const occurrences = frame.split('Run: run-1').length - 1; - expect(occurrences).toBeLessThanOrEqual(1); - } - instance.unmount(); - }); -}); - -async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> { - const start = Date.now(); - while (!predicate()) { - if (Date.now() - start > timeoutMs) throw new Error('waitFor timed out'); - await new Promise((resolve) => setTimeout(resolve, 5)); - } -} diff --git a/evals/tsconfig.json b/evals/tsconfig.json deleted file mode 100644 index ed085f0d..00000000 --- a/evals/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "Node16", - "moduleResolution": "Node16", - "lib": ["ES2022"], - "outDir": "dist", - "rootDir": "src", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "tests"] -} diff --git a/src/application/context/project/init/AgentSelection.ts b/src/application/context/project/init/AgentSelection.ts index 254869a0..4d3e2d10 100644 --- a/src/application/context/project/init/AgentSelection.ts +++ b/src/application/context/project/init/AgentSelection.ts @@ -1,4 +1,4 @@ -export type AgentId = "claude" | "gemini" | "copilot" | "vibe" | "codex" | "cursor"; +export type AgentId = "claude" | "antigravity" | "copilot" | "vibe" | "codex" | "cursor"; export interface AvailableAgent { readonly id: AgentId; diff --git a/src/infrastructure/agents/AgentCliGateway.ts b/src/infrastructure/agents/AgentCliGateway.ts index 92a80630..7df9172d 100644 --- a/src/infrastructure/agents/AgentCliGateway.ts +++ b/src/infrastructure/agents/AgentCliGateway.ts @@ -3,7 +3,7 @@ import { IAgentGateway } from "../../application/agents/IAgentGateway.js"; import { AgentInvocation, AgentInvocationResult } from "../../application/agents/AgentInvocation.js"; import { ITelemetryClient } from "../../application/telemetry/ITelemetryClient.js"; -export const SUPPORTED_AGENT_IDS = ["claude", "gemini", "copilot", "codex", "cursor", "vibe"] as const; +export const SUPPORTED_AGENT_IDS = ["claude", "antigravity", "copilot", "codex", "cursor", "vibe"] as const; export type SupportedAgentId = typeof SUPPORTED_AGENT_IDS[number]; interface AgentCommand { @@ -17,7 +17,7 @@ const AGENT_OUTPUT_TAIL_MAX_LENGTH = 16_384; const AGENT_COMMANDS: Record<SupportedAgentId, AgentCommand> = { claude: { executable: "claude", promptFlag: "-p" }, - gemini: { executable: "gemini", promptFlag: "-p" }, + antigravity: { executable: "agy", promptFlag: "-p" }, copilot: { executable: "gh copilot", promptFlag: "-p" }, codex: { executable: "codex", args: CODEX_NON_INTERACTIVE_EXEC_ARGS }, cursor: { executable: "cursor", promptFlag: "-p" }, diff --git a/src/infrastructure/context/project/init/AgentFileProtocol.ts b/src/infrastructure/context/project/init/AgentFileProtocol.ts index b4cbdfbd..626e787f 100644 --- a/src/infrastructure/context/project/init/AgentFileProtocol.ts +++ b/src/infrastructure/context/project/init/AgentFileProtocol.ts @@ -23,7 +23,7 @@ import { JumboMdContent } from "../../../../domain/project/JumboMdContent.js"; import { AgentsMdContent } from "../../../../domain/project/AgentsMdContent.js"; import { IConfigurer } from "./IConfigurer.js"; import { ClaudeConfigurer } from "./ClaudeConfigurer.js"; -import { GeminiConfigurer } from "./GeminiConfigurer.js"; +import { AntigravityConfigurer } from "./AntigravityConfigurer.js"; import { CopilotConfigurer } from "./CopilotConfigurer.js"; import { VibeConfigurer } from "./VibeConfigurer.js"; import { CodexConfigurer } from "./CodexConfigurer.js"; @@ -48,7 +48,7 @@ export class AgentFileProtocol implements IAgentFileProtocol { constructor(private readonly templateSkillsRoot: string = DEFAULT_TEMPLATE_SKILLS_ROOT) { this.configurers = [ new ClaudeConfigurer(), - new GeminiConfigurer(), + new AntigravityConfigurer(), new CopilotConfigurer(), new VibeConfigurer(), new CodexConfigurer(templateSkillsRoot), diff --git a/src/infrastructure/context/project/init/AntigravityConfigurer.ts b/src/infrastructure/context/project/init/AntigravityConfigurer.ts new file mode 100644 index 00000000..6fbb97ae --- /dev/null +++ b/src/infrastructure/context/project/init/AntigravityConfigurer.ts @@ -0,0 +1,326 @@ +/** + * Infrastructure: Antigravity CLI Configurer + * + * Encapsulates Antigravity workspace configuration: + * - GEMINI.md compatibility reference to JUMBO.md + * - workspace skills distributed through .agents/skills + * + * Operations are idempotent and gracefully handle errors. + */ + +import path from "path"; +import fs from "fs-extra"; +import { AgentFileReferenceContent } from "../../../../domain/project/AgentFileReferenceContent.js"; +import { IConfigurer } from "./IConfigurer.js"; +import { PlannedFileChange } from "../../../../application/context/project/init/PlannedFileChange.js"; + +export class AntigravityConfigurer implements IConfigurer { + readonly agent = { + id: "antigravity", + name: "Antigravity", + } as const; + + readonly skillPlatforms = [".agents/skills"] as const; + + /** + * Configure all Antigravity CLI requirements for Jumbo. + * + * @param projectRoot Absolute path to project root directory + */ + async configure(projectRoot: string): Promise<void> { + await this.ensureGeminiMdCompatibility(projectRoot); + await this.removeLegacyGeminiConfigs(projectRoot); + await this.removeLegacyAntigravityHooks(projectRoot); + } + + /** + * Ensure GEMINI.md exists because Antigravity documents compatibility with it. + */ + private async ensureGeminiMdCompatibility(projectRoot: string): Promise<void> { + const geminiMdPath = path.join(projectRoot, "GEMINI.md"); + const reference = AgentFileReferenceContent.getAgentFileReference("GEMINI.md"); + + try { + const exists = await fs.pathExists(geminiMdPath); + + if (!exists) { + await fs.writeFile(geminiMdPath, reference, "utf-8"); + return; + } + + const content = await fs.readFile(geminiMdPath, "utf-8"); + + if (!content.includes("JUMBO.md")) { + const updatedContent = content.trimEnd() + "\n\n" + reference; + await fs.writeFile(geminiMdPath, updatedContent, "utf-8"); + } + } catch (error) { + console.warn( + `Warning: Failed to update Antigravity GEMINI.md compatibility file: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + /** + * Repair Antigravity configuration by replacing stale managed content. + */ + async repair(projectRoot: string): Promise<void> { + await this.repairGeminiMdCompatibility(projectRoot); + await this.removeLegacyGeminiConfigs(projectRoot); + await this.removeLegacyAntigravityHooks(projectRoot); + } + + /** + * Repair GEMINI.md by replacing legacy Jumbo reference content. + */ + private async repairGeminiMdCompatibility(projectRoot: string): Promise<void> { + const geminiMdPath = path.join(projectRoot, "GEMINI.md"); + const reference = AgentFileReferenceContent.getAgentFileReference("GEMINI.md"); + + try { + const exists = await fs.pathExists(geminiMdPath); + + if (!exists) { + await fs.writeFile(geminiMdPath, reference, "utf-8"); + return; + } + + const content = await fs.readFile(geminiMdPath, "utf-8"); + const replaced = AgentFileReferenceContent.replaceAgentFileReference(content, "GEMINI.md"); + + if (replaced !== null) { + await fs.writeFile(geminiMdPath, replaced, "utf-8"); + } else if (!content.includes("JUMBO.md")) { + const updatedContent = content.trimEnd() + "\n\n" + reference; + await fs.writeFile(geminiMdPath, updatedContent, "utf-8"); + } + } catch (error) { + console.warn( + `Warning: Failed to repair Antigravity GEMINI.md compatibility file: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + /** + * Return what changes this configurer will make without executing. + */ + async getPlannedFileChanges(projectRoot: string): Promise<PlannedFileChange[]> { + const changes: PlannedFileChange[] = []; + + const geminiMdPath = path.join(projectRoot, "GEMINI.md"); + changes.push({ + path: "GEMINI.md", + action: (await fs.pathExists(geminiMdPath)) ? "modify" : "create", + description: "Add Antigravity-compatible Jumbo instructions", + }); + + const hooksPath = path.join(projectRoot, ".agents", "hooks.json"); + if (await fs.pathExists(hooksPath)) { + changes.push({ + path: ".agents/hooks.json", + action: "modify", + description: "Remove legacy Antigravity hook configuration", + }); + } + + const hookRunnerPath = this.getHookRunnerPath(projectRoot); + if (await fs.pathExists(hookRunnerPath)) { + changes.push({ + path: ".agents/jumbo/antigravity-hook.mjs", + action: "modify", + description: "Remove legacy Antigravity hook runner", + }); + } + + const geminiSettingsPath = path.join(projectRoot, ".gemini", "settings.json"); + if (await fs.pathExists(geminiSettingsPath)) { + changes.push({ + path: ".gemini/settings.json", + action: "modify", + description: "Remove legacy Gemini settings and Jumbo hook configs", + }); + } + + const geminiSkillsPath = path.join(projectRoot, ".gemini", "skills"); + if (await fs.pathExists(geminiSkillsPath)) { + changes.push({ + path: ".gemini/skills", + action: "modify", + description: "Remove legacy Gemini skills", + }); + } + + return changes; + } + + private async removeLegacyAntigravityHooks(projectRoot: string): Promise<void> { + const agentsDir = path.join(projectRoot, ".agents"); + if (!(await fs.pathExists(agentsDir))) { + return; + } + + // 1. Delete hook runner + const runnerPath = this.getHookRunnerPath(projectRoot); + if (await fs.pathExists(runnerPath)) { + await fs.remove(runnerPath); + } + const jumboDir = path.dirname(runnerPath); + if (await fs.pathExists(jumboDir)) { + try { + const files = await fs.readdir(jumboDir); + if (files.length === 0) { + await fs.remove(jumboDir); + } + } catch { + // Ignore + } + } + + // 2. Clean jumbo-session-bootstrap hook from .agents/hooks.json + const hooksPath = path.join(projectRoot, ".agents", "hooks.json"); + if (await fs.pathExists(hooksPath)) { + try { + const content = await fs.readFile(hooksPath, "utf-8"); + if (content.trim()) { + const hooks = JSON.parse(content); + if (hooks["jumbo-session-bootstrap"]) { + delete hooks["jumbo-session-bootstrap"]; + + if (Object.keys(hooks).length === 0) { + await fs.remove(hooksPath); + } else { + await fs.writeFile(hooksPath, JSON.stringify(hooks, null, 2) + "\n", "utf-8"); + } + } + } + } catch (error) { + console.warn( + `Warning: Failed to clean legacy hooks in .agents/hooks.json: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + } + + private async removeLegacyGeminiConfigs(projectRoot: string): Promise<void> { + const geminiDir = path.join(projectRoot, ".gemini"); + if (!(await fs.pathExists(geminiDir))) { + return; + } + + // 1. Clean up .gemini/settings.json + const settingsPath = path.join(geminiDir, "settings.json"); + if (await fs.pathExists(settingsPath)) { + try { + const content = await fs.readFile(settingsPath, "utf-8"); + if (content.trim()) { + const settings = JSON.parse(content); + + // Remove jumbo-specific hooks + if (settings.hooks) { + for (const eventType of Object.keys(settings.hooks)) { + const matchers = settings.hooks[eventType]; + if (Array.isArray(matchers)) { + settings.hooks[eventType] = matchers + .map((matcher: any) => { + if (matcher && Array.isArray(matcher.hooks)) { + matcher.hooks = matcher.hooks.filter((hook: any) => { + return hook && !(typeof hook.command === "string" && hook.command.includes("jumbo")); + }); + } + return matcher; + }) + .filter((matcher: any) => matcher && Array.isArray(matcher.hooks) && matcher.hooks.length > 0); + + if (settings.hooks[eventType].length === 0) { + delete settings.hooks[eventType]; + } + } + } + if (Object.keys(settings.hooks).length === 0) { + delete settings.hooks; + } + } + + // Remove jumbo-specific allowed tools + if (settings.tools && Array.isArray(settings.tools.allowed)) { + settings.tools.allowed = settings.tools.allowed.filter((tool: string) => { + return typeof tool === "string" && !tool.includes("jumbo") && !tool.includes("antigravity-hook.mjs"); + }); + if (settings.tools.allowed.length === 0) { + delete settings.tools.allowed; + } + } + if (settings.tools && Object.keys(settings.tools).length === 0) { + delete settings.tools; + } + + // If settings is now empty, delete settings.json (and backups) + if (Object.keys(settings).length === 0) { + await fs.remove(settingsPath); + + const files = await fs.readdir(geminiDir); + for (const file of files) { + if (file.startsWith("settings.json.backup.")) { + await fs.remove(path.join(geminiDir, file)); + } + } + } else { + await fs.writeFile(settingsPath, JSON.stringify(settings, null, 2) + "\n", "utf-8"); + } + } + } catch (error) { + console.warn( + `Warning: Failed to clean legacy settings in .gemini/settings.json: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + // 2. Clean up obsolete Jumbo-managed skills in .gemini/skills + const skillsDir = path.join(geminiDir, "skills"); + if (await fs.pathExists(skillsDir)) { + try { + const skillEntries = await fs.readdir(skillsDir); + for (const skillName of skillEntries) { + const skillPath = path.join(skillsDir, skillName); + const stat = await fs.stat(skillPath); + if (stat.isDirectory()) { + await fs.remove(skillPath); + } + } + + const remainingSkills = await fs.readdir(skillsDir); + if (remainingSkills.length === 0) { + await fs.remove(skillsDir); + } + } catch (error) { + console.warn( + `Warning: Failed to clean legacy skills in .gemini/skills: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + // 3. Remove .gemini directory if it is now empty + try { + const remainingFiles = await fs.readdir(geminiDir); + if (remainingFiles.length === 0) { + await fs.remove(geminiDir); + } + } catch (error) { + // Ignore + } + } + + private getHookRunnerPath(projectRoot: string): string { + return path.join(projectRoot, ".agents", "jumbo", "antigravity-hook.mjs"); + } +} diff --git a/src/infrastructure/context/project/init/GeminiConfigurer.ts b/src/infrastructure/context/project/init/GeminiConfigurer.ts deleted file mode 100644 index 955d4517..00000000 --- a/src/infrastructure/context/project/init/GeminiConfigurer.ts +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Infrastructure: Gemini CLI Configurer - * - * Encapsulates all knowledge about Gemini CLI configuration: - * - GEMINI.md with thin reference to JUMBO.md - * - .gemini/settings.json with SessionStart hooks - * - * Operations are idempotent and gracefully handle errors. - */ - -import path from "path"; -import fs from "fs-extra"; -import { AgentFileReferenceContent } from "../../../../domain/project/AgentFileReferenceContent.js"; -import { AgentFileAssetContent } from "../../../../domain/project/AgentFileAssetContent.js"; -import { SafeGeminiSettingsMerger, GeminiSettings } from "./SafeGeminiSettingsMerger.js"; -import { IConfigurer } from "./IConfigurer.js"; -import { PlannedFileChange } from "../../../../application/context/project/init/PlannedFileChange.js"; - -export class GeminiConfigurer implements IConfigurer { - readonly agent = { - id: "gemini", - name: "Gemini", - } as const; - - readonly skillPlatforms = [".gemini/skills"] as const; - - /** - * Configure all Gemini CLI requirements for Jumbo - * - * @param projectRoot Absolute path to project root directory - */ - async configure(projectRoot: string): Promise<void> { - await this.ensureGeminiMd(projectRoot); - await this.ensureGeminiSettings(projectRoot); - } - - /** - * Ensure GEMINI.md exists with thin reference to JUMBO.md - */ - private async ensureGeminiMd(projectRoot: string): Promise<void> { - const geminiMdPath = path.join(projectRoot, "GEMINI.md"); - const reference = AgentFileReferenceContent.getAgentFileReference("GEMINI.md"); - - try { - const exists = await fs.pathExists(geminiMdPath); - - if (!exists) { - await fs.writeFile(geminiMdPath, reference, "utf-8"); - return; - } - - // File exists - check if reference is present - const content = await fs.readFile(geminiMdPath, "utf-8"); - - if (!content.includes("JUMBO.md")) { - // Reference missing - append it - const updatedContent = content.trimEnd() + "\n\n" + reference; - await fs.writeFile(geminiMdPath, updatedContent, "utf-8"); - } - // else: Reference already present - no-op - } catch (error) { - // Graceful degradation - log but don't throw - console.warn( - `Warning: Failed to update GEMINI.md: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - /** - * Ensure Gemini CLI hooks are configured in .gemini/settings.json - */ - private async ensureGeminiSettings(projectRoot: string): Promise<void> { - try { - const jumboSettings = AgentFileAssetContent.readJson<GeminiSettings>("gemini-settings.fragment.json"); - - await SafeGeminiSettingsMerger.mergeSettings(projectRoot, jumboSettings); - } catch (error) { - // Graceful degradation - log but don't throw - console.warn( - `Warning: Failed to configure Gemini CLI hooks: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - /** - * Repair Gemini CLI configuration by replacing stale content - */ - async repair(projectRoot: string): Promise<void> { - await this.repairGeminiMd(projectRoot); - await this.ensureGeminiSettings(projectRoot); - } - - /** - * Repair GEMINI.md by replacing the reference with the current thin reference - */ - private async repairGeminiMd(projectRoot: string): Promise<void> { - const geminiMdPath = path.join(projectRoot, "GEMINI.md"); - const reference = AgentFileReferenceContent.getAgentFileReference("GEMINI.md"); - - try { - const exists = await fs.pathExists(geminiMdPath); - - if (!exists) { - await fs.writeFile(geminiMdPath, reference, "utf-8"); - return; - } - - const content = await fs.readFile(geminiMdPath, "utf-8"); - const replaced = AgentFileReferenceContent.replaceAgentFileReference(content, "GEMINI.md"); - - if (replaced !== null) { - // Legacy reference block found - replace entire file with new thin reference - await fs.writeFile(geminiMdPath, replaced, "utf-8"); - } else if (!content.includes("JUMBO.md")) { - // No reference at all - append - const updatedContent = content.trimEnd() + "\n\n" + reference; - await fs.writeFile(geminiMdPath, updatedContent, "utf-8"); - } - } catch (error) { - console.warn( - `Warning: Failed to repair GEMINI.md: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - /** - * Return what changes this configurer will make without executing. - */ - async getPlannedFileChanges(projectRoot: string): Promise<PlannedFileChange[]> { - const changes: PlannedFileChange[] = []; - - const geminiMdPath = path.join(projectRoot, "GEMINI.md"); - changes.push({ - path: "GEMINI.md", - action: (await fs.pathExists(geminiMdPath)) ? "modify" : "create", - description: "Add Jumbo instructions", - }); - - const settingsPath = path.join(projectRoot, ".gemini/settings.json"); - changes.push({ - path: ".gemini/settings.json", - action: (await fs.pathExists(settingsPath)) ? "modify" : "create", - description: "Add hook configuration and permissions", - }); - - return changes; - } -} diff --git a/src/infrastructure/context/project/init/SafeAntigravityHooksMerger.ts b/src/infrastructure/context/project/init/SafeAntigravityHooksMerger.ts new file mode 100644 index 00000000..91e72e2f --- /dev/null +++ b/src/infrastructure/context/project/init/SafeAntigravityHooksMerger.ts @@ -0,0 +1,176 @@ +import fs from "fs-extra"; +import path from "path"; + +export interface AntigravityHookHandler { + type?: "command"; + command: string; + timeout?: number; +} + +export interface AntigravityToolHookMatcher { + matcher: string; + hooks: AntigravityHookHandler[]; +} + +export interface AntigravityHookDefinition { + enabled?: boolean; + PreToolUse?: AntigravityToolHookMatcher[]; + PostToolUse?: AntigravityToolHookMatcher[]; + PreInvocation?: AntigravityHookHandler[]; + PostInvocation?: AntigravityHookHandler[]; + Stop?: AntigravityHookHandler[]; + [eventName: string]: + | boolean + | AntigravityToolHookMatcher[] + | AntigravityHookHandler[] + | undefined; +} + +export type AntigravityHooksDocument = Record<string, AntigravityHookDefinition>; + +export class SafeAntigravityHooksMerger { + /** + * Merges Jumbo-managed hook definitions into .agents/hooks.json. + * + * Existing user hook definitions and unknown fields are preserved. Hook handlers + * are deduplicated by command content, so repeated init or repair remains idempotent. + */ + static async mergeHooks( + projectRoot: string, + newHooks: AntigravityHooksDocument + ): Promise<void> { + const hooksPath = path.join(projectRoot, ".agents", "hooks.json"); + const backupPath = `${hooksPath}.backup.${Date.now()}`; + + await fs.ensureDir(path.join(projectRoot, ".agents")); + + if (await fs.pathExists(hooksPath)) { + await fs.copy(hooksPath, backupPath); + } + + try { + let existing: AntigravityHooksDocument = {}; + if (await fs.pathExists(hooksPath)) { + const content = await fs.readFile(hooksPath, "utf-8"); + if (content.trim()) { + existing = JSON.parse(content); + } + } + + const merged = this.mergeDocuments(existing, newHooks); + await fs.writeFile(hooksPath, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + } catch (error) { + if (await fs.pathExists(backupPath)) { + await fs.copy(backupPath, hooksPath, { overwrite: true }); + } + throw error; + } + } + + private static mergeDocuments( + existing: AntigravityHooksDocument, + additions: AntigravityHooksDocument + ): AntigravityHooksDocument { + const merged: AntigravityHooksDocument = { ...existing }; + + for (const [hookName, hookDefinition] of Object.entries(additions)) { + merged[hookName] = this.mergeHookDefinition(merged[hookName], hookDefinition); + } + + return merged; + } + + private static mergeHookDefinition( + existing: AntigravityHookDefinition | undefined, + addition: AntigravityHookDefinition + ): AntigravityHookDefinition { + if (existing === undefined) { + return addition; + } + + const merged: AntigravityHookDefinition = { ...existing }; + + if (addition.enabled !== undefined) { + merged.enabled = existing.enabled ?? addition.enabled; + } + + for (const eventName of Object.keys(addition)) { + if (eventName === "enabled") { + continue; + } + + const additionValue = addition[eventName]; + if (!Array.isArray(additionValue)) { + continue; + } + + const existingValue = existing[eventName]; + if (this.isToolMatcherEvent(eventName)) { + merged[eventName] = this.mergeToolHookMatchers( + Array.isArray(existingValue) + ? (existingValue as readonly AntigravityToolHookMatcher[]) + : [], + additionValue as readonly AntigravityToolHookMatcher[] + ); + } else { + merged[eventName] = this.mergeHookHandlers( + Array.isArray(existingValue) + ? (existingValue as readonly AntigravityHookHandler[]) + : [], + additionValue as readonly AntigravityHookHandler[] + ); + } + } + + return merged; + } + + private static mergeToolHookMatchers( + existing: readonly AntigravityToolHookMatcher[], + additions: readonly AntigravityToolHookMatcher[] + ): AntigravityToolHookMatcher[] { + const merged = [...existing]; + + for (const addition of additions) { + const existingIndex = merged.findIndex((matcher) => matcher.matcher === addition.matcher); + + if (existingIndex === -1) { + merged.push(addition); + continue; + } + + const existingMatcher = merged[existingIndex]; + merged[existingIndex] = { + ...existingMatcher, + hooks: this.mergeHookHandlers(existingMatcher.hooks, addition.hooks), + }; + } + + return merged; + } + + private static mergeHookHandlers( + existing: readonly AntigravityHookHandler[], + additions: readonly AntigravityHookHandler[] + ): AntigravityHookHandler[] { + const handlersByKey = new Map<string, AntigravityHookHandler>(); + + for (const handler of existing) { + handlersByKey.set(this.getHandlerKey(handler), handler); + } + + for (const handler of additions) { + handlersByKey.set(this.getHandlerKey(handler), handler); + } + + return Array.from(handlersByKey.values()); + } + + private static isToolMatcherEvent(eventName: string): boolean { + return eventName === "PreToolUse" || eventName === "PostToolUse"; + } + + private static getHandlerKey(handler: AntigravityHookHandler): string { + return `${handler.type ?? "command"}:${handler.command}`; + } +} diff --git a/src/infrastructure/context/project/init/SafeGeminiSettingsMerger.ts b/src/infrastructure/context/project/init/SafeGeminiSettingsMerger.ts deleted file mode 100644 index 8ceba523..00000000 --- a/src/infrastructure/context/project/init/SafeGeminiSettingsMerger.ts +++ /dev/null @@ -1,217 +0,0 @@ -import fs from "fs-extra"; -import * as path from "path"; - -/** - * Gemini CLI settings structure - * Based on Gemini CLI hooks configuration - */ -export interface GeminiCommandHook { - type: "command"; - command: string; -} - -export interface GeminiPromptHook { - type: "prompt"; - prompt: string; -} - -export type GeminiHook = GeminiCommandHook | GeminiPromptHook; - -export interface GeminiHookMatcher { - matcher: string; - hooks: GeminiHook[]; -} - -export interface GeminiSettings { - hooks?: { - SessionStart?: GeminiHookMatcher[]; - PreCompress?: GeminiHookMatcher[]; - SessionEnd?: GeminiHookMatcher[]; - [key: string]: GeminiHookMatcher[] | undefined; - }; - tools?: { - allowed?: string[]; - }; -} - -/** - * Safe merger for Gemini CLI settings.json files - * - * Strategy: - * - Creates backup before modification - * - Deep merges objects, deduplicates arrays - * - Lenient with existing user config (may have extensions or typos) - * - Rolls back on error - * - Idempotent (safe to run multiple times) - * - * @example - * await SafeGeminiSettingsMerger.mergeSettings(projectRoot, { - * hooks: { - * SessionStart: [{ - * matcher: "startup", - * hooks: [{ type: "command", command: "jumbo session start" }] - * }] - * } - * }); - */ -export class SafeGeminiSettingsMerger { - /** - * Merges new settings into existing .gemini/settings.json - * - * @param projectRoot - Root directory of the project - * @param newSettings - Settings to merge in - * @throws Error if JSON is malformed - */ - static async mergeSettings( - projectRoot: string, - newSettings: GeminiSettings - ): Promise<void> { - const settingsPath = path.join(projectRoot, ".gemini", "settings.json"); - const backupPath = `${settingsPath}.backup.${Date.now()}`; - - // Ensure .gemini directory exists - await fs.ensureDir(path.join(projectRoot, ".gemini")); - - // STEP 1: Create backup if file exists - if (await fs.pathExists(settingsPath)) { - await fs.copy(settingsPath, backupPath); - } - - try { - // STEP 2: Read existing or use empty object - // Be lenient with existing user content - only parse JSON, don't validate structure - // User's config may have extensions or typos we shouldn't reject - let existing: GeminiSettings = {}; - if (await fs.pathExists(settingsPath)) { - const content = await fs.readFile(settingsPath, "utf-8"); - // Handle empty file case - treat as empty config - if (content.trim()) { - existing = JSON.parse(content); - } - } - - // STEP 3: Merge safely - const merged = this.deepMerge(existing, newSettings); - - // STEP 4: Write back with formatting - // Skip validation - user's config may have extensions or typos we should preserve - await fs.writeFile( - settingsPath, - JSON.stringify(merged, null, 2) + "\n", - "utf-8" - ); - - // STEP 5: Cleanup backup on success (optional - keep for audit trail) - // await fs.remove(backupPath); - } catch (error) { - // STEP 6: Rollback on error - if (await fs.pathExists(backupPath)) { - await fs.copy(backupPath, settingsPath, { overwrite: true }); - } - throw error; - } - } - - /** - * Deep merges two settings objects - * - Objects are merged recursively - * - Arrays are deduplicated based on content - * - Existing user settings are preserved - */ - private static deepMerge( - existing: GeminiSettings, - newSettings: GeminiSettings - ): GeminiSettings { - const result: GeminiSettings = { ...existing }; - - // Merge hooks - handle all event types generically - if (newSettings.hooks) { - result.hooks = result.hooks ?? {}; - - for (const eventType of Object.keys(newSettings.hooks)) { - const newHooks = newSettings.hooks[eventType]; - if (newHooks) { - const existingHooks = existing.hooks?.[eventType] ?? []; - result.hooks[eventType] = this.mergeHookMatchers( - existingHooks, - newHooks - ); - } - } - } - - // Merge tools - if (newSettings.tools) { - result.tools = result.tools ?? {}; - - // Merge allowed array (unique values) - if (newSettings.tools.allowed) { - const existingAllowed = existing.tools?.allowed ?? []; - const newAllowed = newSettings.tools.allowed; - result.tools.allowed = Array.from(new Set([...existingAllowed, ...newAllowed])); - } - } - - return result; - } - - /** - * Merges hook matcher arrays, deduplicating by hook content - */ - private static mergeHookMatchers( - existing: GeminiHookMatcher[], - additions: GeminiHookMatcher[] - ): GeminiHookMatcher[] { - const merged = [...existing]; - - for (const newMatcher of additions) { - // Find existing matcher with same matcher type - const existingIndex = merged.findIndex( - (m) => m.matcher === newMatcher.matcher - ); - - if (existingIndex >= 0) { - // Merge hooks within the same matcher, deduplicating by content - const existingMatcher = merged[existingIndex]; - const hookMap = new Map<string, GeminiHook>(); - - // Add existing hooks - key by unique content - for (const hook of existingMatcher.hooks) { - hookMap.set(this.getHookKey(hook), hook); - } - - // Add new hooks (overwrites if same key) - for (const hook of newMatcher.hooks) { - hookMap.set(this.getHookKey(hook), hook); - } - - merged[existingIndex] = { - ...existingMatcher, - hooks: Array.from(hookMap.values()), - }; - } else { - // No existing matcher with this type, add it - merged.push(newMatcher); - } - } - - return merged; - } - - /** - * Generate a unique key for a hook based on its type and content - * Handles malformed hooks gracefully by including all available properties - */ - private static getHookKey(hook: GeminiHook): string { - // Handle malformed hooks that might have mismatched type/content - const anyHook = hook as unknown as Record<string, unknown>; - if (hook.type === "command" && typeof anyHook.command === "string") { - return `command:${anyHook.command}`; - } else if (hook.type === "prompt" && typeof anyHook.prompt === "string") { - return `prompt:${anyHook.prompt}`; - } else { - // Fallback for malformed hooks - use JSON serialization for uniqueness - return `unknown:${JSON.stringify(hook)}`; - } - } -} diff --git a/src/presentation/cli/commands/work/shared/AgentSpawner.ts b/src/presentation/cli/commands/work/shared/AgentSpawner.ts index 905adbb1..4419d13c 100644 --- a/src/presentation/cli/commands/work/shared/AgentSpawner.ts +++ b/src/presentation/cli/commands/work/shared/AgentSpawner.ts @@ -7,7 +7,7 @@ import { spawn } from "node:child_process"; -export const SUPPORTED_AGENTS = ["claude", "gemini", "copilot", "codex", "cursor", "vibe"] as const; +export const SUPPORTED_AGENTS = ["claude", "antigravity", "copilot", "codex", "cursor", "vibe"] as const; export type AgentId = typeof SUPPORTED_AGENTS[number]; export interface AgentCommand { @@ -18,7 +18,7 @@ export interface AgentCommand { export const AGENT_COMMANDS: Record<AgentId, AgentCommand> = { claude: { executable: "claude", promptFlag: "-p" }, - gemini: { executable: "gemini", promptFlag: "-p" }, + antigravity: { executable: "agy", promptFlag: "-p" }, copilot: { executable: "gh copilot", promptFlag: "-p" }, codex: { executable: "codex", args: ["exec"] }, cursor: { executable: "cursor", promptFlag: "-p" }, diff --git a/src/presentation/cli/output/HookOutput.ts b/src/presentation/cli/output/HookOutput.ts index bcd9628e..fa4c461f 100644 --- a/src/presentation/cli/output/HookOutput.ts +++ b/src/presentation/cli/output/HookOutput.ts @@ -1,6 +1,6 @@ /** * JSON output structure for hook integration. - * Must match Gemini CLI hook specification. + * Must match the hook contract expected by the invoking agent harness. */ export interface HookOutput { decision: 'allow' | 'deny' | 'modify'; diff --git a/src/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.ts b/src/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.ts index d091dce7..f7e21e63 100644 --- a/src/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.ts +++ b/src/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.ts @@ -1,6 +1,6 @@ import type { DaemonConfig } from "../daemon-subprocesses/ISubprocessManager.js"; -const COCKPIT_DAEMON_AGENT_OPTIONS = ["codex", "claude", "gemini", "copilot", "cursor", "vibe"] as const; +const COCKPIT_DAEMON_AGENT_OPTIONS = ["codex", "claude", "antigravity", "copilot", "cursor", "vibe"] as const; export function getNextCockpitDaemonAgentConfig(config: DaemonConfig): DaemonConfig { const currentIndex = COCKPIT_DAEMON_AGENT_OPTIONS.indexOf( diff --git a/src/presentation/tui/project-initialization/Constants.ts b/src/presentation/tui/project-initialization/Constants.ts index 01bab284..5f4f7bf0 100644 --- a/src/presentation/tui/project-initialization/Constants.ts +++ b/src/presentation/tui/project-initialization/Constants.ts @@ -146,7 +146,7 @@ export const InitFlowConfirmationGroupLabel = { shared: "Shared", claude: "Claude", codex: "Codex", - gemini: "Gemini", + antigravity: "Antigravity", copilot: "Copilot", cursor: "Cursor", vibe: "Vibe", diff --git a/src/presentation/tui/project-initialization/InitFlow.tsx b/src/presentation/tui/project-initialization/InitFlow.tsx index e27fa906..f4eda973 100644 --- a/src/presentation/tui/project-initialization/InitFlow.tsx +++ b/src/presentation/tui/project-initialization/InitFlow.tsx @@ -773,7 +773,7 @@ function summarizePlannedChangeGroups( InitFlowConfirmationGroupLabel.shared, InitFlowConfirmationGroupLabel.claude, InitFlowConfirmationGroupLabel.codex, - InitFlowConfirmationGroupLabel.gemini, + InitFlowConfirmationGroupLabel.antigravity, InitFlowConfirmationGroupLabel.copilot, InitFlowConfirmationGroupLabel.cursor, InitFlowConfirmationGroupLabel.vibe, @@ -787,8 +787,8 @@ function classifyPlannedChangeGroup(path: string): string { return InitFlowConfirmationGroupLabel.claude; } if (path.startsWith(".codex/")) return InitFlowConfirmationGroupLabel.codex; - if (path === "GEMINI.md" || path.startsWith(".gemini/")) { - return InitFlowConfirmationGroupLabel.gemini; + if (path === "GEMINI.md" || path === ".agents/hooks.json" || path.startsWith(".agents/jumbo/")) { + return InitFlowConfirmationGroupLabel.antigravity; } if (path.startsWith(".github/")) { return InitFlowConfirmationGroupLabel.copilot; diff --git a/tests/application/context/project/init/InitializeProjectCommandHandler.test.ts b/tests/application/context/project/init/InitializeProjectCommandHandler.test.ts index 4251bd74..681b1353 100644 --- a/tests/application/context/project/init/InitializeProjectCommandHandler.test.ts +++ b/tests/application/context/project/init/InitializeProjectCommandHandler.test.ts @@ -120,7 +120,7 @@ describe("InitializeProjectCommandHandler", () => { purpose: "LLM context management", }; - const result = await handler.execute(command, "/repo", ["claude", "gemini"]); + const result = await handler.execute(command, "/repo", ["claude", "antigravity"]); expect(result.projectId).toBe(generatedProjectId); expect(projectIdentityResolver.generateProjectId).toHaveBeenCalledTimes(1); @@ -137,7 +137,7 @@ describe("InitializeProjectCommandHandler", () => { expect(settingsInitializer.ensureSettingsFileExists).toHaveBeenCalledTimes(1); expect(projectIdentityResolver.persistProjectId).toHaveBeenCalledWith(generatedProjectId); expect(agentFileProtocol.ensureAgentsMd).toHaveBeenCalledWith("/repo"); - expect(agentFileProtocol.ensureAgentConfigurations).toHaveBeenCalledWith("/repo", ["claude", "gemini"]); + expect(agentFileProtocol.ensureAgentConfigurations).toHaveBeenCalledWith("/repo", ["claude", "antigravity"]); expect(gitignoreProtocol.ensureExclusions).toHaveBeenCalledWith("/repo"); }); }); diff --git a/tests/application/context/project/init/LocalInitializeProjectGateway.test.ts b/tests/application/context/project/init/LocalInitializeProjectGateway.test.ts index bbdc42b9..d4414335 100644 --- a/tests/application/context/project/init/LocalInitializeProjectGateway.test.ts +++ b/tests/application/context/project/init/LocalInitializeProjectGateway.test.ts @@ -93,7 +93,7 @@ describe("LocalInitializeProjectGateway", () => { name: "my-project", purpose: undefined, projectRoot: "/home/user/project", - selectedAgentIds: ["gemini"], + selectedAgentIds: ["antigravity"], }; mockPlanGateway.planProjectInit.mockResolvedValue({ availableAgents: [], plannedChanges: [] }); @@ -103,7 +103,7 @@ describe("LocalInitializeProjectGateway", () => { expect(mockPlanGateway.planProjectInit).toHaveBeenCalledWith({ projectRoot: "/home/user/project", - selectedAgentIds: ["gemini"], + selectedAgentIds: ["antigravity"], }); }); diff --git a/tests/application/context/project/init/LocalPlanProjectInitGateway.test.ts b/tests/application/context/project/init/LocalPlanProjectInitGateway.test.ts index d29f6d52..a180717e 100644 --- a/tests/application/context/project/init/LocalPlanProjectInitGateway.test.ts +++ b/tests/application/context/project/init/LocalPlanProjectInitGateway.test.ts @@ -96,12 +96,12 @@ describe("LocalPlanProjectInitGateway", () => { it("should pass selected agent ids to the agent file protocol", async () => { await gateway.planProjectInit({ projectRoot: "/test/project", - selectedAgentIds: ["gemini", "copilot"], + selectedAgentIds: ["antigravity", "copilot"], }); expect(mockAgentFileProtocol.getPlannedFileChanges).toHaveBeenCalledWith( "/test/project", - ["gemini", "copilot"] + ["antigravity", "copilot"] ); }); }); diff --git a/tests/infrastructure/agents/AgentCliGateway.test.ts b/tests/infrastructure/agents/AgentCliGateway.test.ts index 7fc74ce1..7a7ee224 100644 --- a/tests/infrastructure/agents/AgentCliGateway.test.ts +++ b/tests/infrastructure/agents/AgentCliGateway.test.ts @@ -93,6 +93,28 @@ describe("AgentCliGateway", () => { ); }); + it("spawns Antigravity through the documented agy executable", async () => { + const child = childProcess(); + spawnMock.mockReturnValue(child); + + const promise = new AgentCliGateway(telemetryClient).invoke({ + agentId: "antigravity", + prompt: "run review", + }); + child.emit("close", 0); + + await expect(promise).resolves.toEqual({ exitCode: 0, stdout: "", stderr: "" }); + expect(spawnMock).toHaveBeenCalledWith( + 'agy -p "run review"', + [], + expect.objectContaining({ shell: true }), + ); + expect(telemetryClient.track).toHaveBeenCalledWith( + "agent_invocation_completed", + expect.objectContaining({ agentId: "antigravity", exitCode: 0, success: true }), + ); + }); + it("captures bounded agent stdout and stderr tails without mirroring raw output to daemon stderr", async () => { const child = childProcess(); spawnMock.mockReturnValue(child); @@ -150,4 +172,14 @@ describe("AgentCliGateway", () => { ).rejects.toThrow("Unsupported agent: unknown"); expect(spawnMock).not.toHaveBeenCalled(); }); + + it("rejects the retired Gemini agent before spawning", async () => { + await expect( + new AgentCliGateway(telemetryClient).invoke({ + agentId: "gemini", + prompt: "run codify", + }), + ).rejects.toThrow("Unsupported agent: gemini"); + expect(spawnMock).not.toHaveBeenCalled(); + }); }); diff --git a/tests/infrastructure/context/project/init/AgentFileProtocol.test.ts b/tests/infrastructure/context/project/init/AgentFileProtocol.test.ts index a4b6c03a..3a3023e3 100644 --- a/tests/infrastructure/context/project/init/AgentFileProtocol.test.ts +++ b/tests/infrastructure/context/project/init/AgentFileProtocol.test.ts @@ -4,6 +4,7 @@ import fs from "fs-extra"; import * as path from "path"; +import { spawnSync } from "node:child_process"; import { AgentFileProtocol } from "../../../../../src/infrastructure/context/project/init/AgentFileProtocol"; import { JumboMdContent } from "../../../../../src/domain/project/JumboMdContent"; import { AgentsMdContent } from "../../../../../src/domain/project/AgentsMdContent"; @@ -18,7 +19,7 @@ jest.setTimeout(30_000); describe("AgentFileProtocol", () => { let tmpDir: string; let protocol: AgentFileProtocol; - const skillPlatforms = [".agents/skills", ".claude/skills", ".gemini/skills", ".vibe/skills"]; + const skillPlatforms = [".agents/skills", ".claude/skills", ".vibe/skills"]; beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "test-agent-files-")); @@ -33,7 +34,7 @@ describe("AgentFileProtocol", () => { it("should return the configured agent list derived from configurers", () => { expect(protocol.getAvailableAgents()).toEqual([ { id: "claude", name: "Claude" }, - { id: "gemini", name: "Gemini" }, + { id: "antigravity", name: "Antigravity" }, { id: "copilot", name: "Copilot" }, { id: "vibe", name: "Vibe" }, { id: "codex", name: "Codex" }, @@ -186,6 +187,7 @@ describe("AgentFileProtocol", () => { expect(await fs.pathExists(path.join(tmpDir, ".github", "hooks", "hooks.json"))).toBe(true); expect(await fs.pathExists(path.join(tmpDir, "GEMINI.md"))).toBe(false); + expect(await fs.pathExists(path.join(tmpDir, ".agents", "hooks.json"))).toBe(false); expect(await fs.pathExists(path.join(tmpDir, ".gemini", "settings.json"))).toBe(false); expect(await fs.pathExists(path.join(tmpDir, ".claude", "skills", "my-skill", "SKILL.md"))).toBe(true); @@ -305,8 +307,8 @@ describe("AgentFileProtocol", () => { expect(content).toContain("# CLAUDE.md"); }); - it("should create GEMINI.md with thin reference to JUMBO.md", async () => { - await protocol.ensureAgentConfigurations(tmpDir); + it("should create GEMINI.md with thin reference to JUMBO.md for Antigravity compatibility", async () => { + await protocol.ensureAgentConfigurations(tmpDir, ["antigravity"]); const geminiMdPath = path.join(tmpDir, "GEMINI.md"); const exists = await fs.pathExists(geminiMdPath); @@ -342,29 +344,35 @@ describe("AgentFileProtocol", () => { expect(settings.permissions.allow).toContain("Bash(jumbo --help)"); }); - it("should create .gemini/settings.json with jumbo --help permission", async () => { - await protocol.ensureAgentConfigurations(tmpDir); + it("should clean up legacy Antigravity hooks", async () => { + // Pre-create legacy hook runner and hooks + const hookRunnerPath = path.join(tmpDir, ".agents", "jumbo", "antigravity-hook.mjs"); + await fs.ensureDir(path.dirname(hookRunnerPath)); + await fs.writeFile(hookRunnerPath, "JUMBO_MANAGED_ANTIGRAVITY_HOOK_RUNNER_V1", "utf-8"); - const settingsPath = path.join(tmpDir, ".gemini", "settings.json"); - const content = await fs.readFile(settingsPath, "utf-8"); - const settings = JSON.parse(content); - expect(settings.tools?.allowed).toBeDefined(); - expect(settings.tools.allowed).toContain("run_shell_command(jumbo --help)"); - }); + const hooksPath = path.join(tmpDir, ".agents", "hooks.json"); + await fs.writeFile( + hooksPath, + JSON.stringify({ + "jumbo-session-bootstrap": { + PreInvocation: [{ type: "command", command: "node ..." }] + }, + "user-linter": { + enabled: false + } + }, null, 2), + "utf-8" + ); - it("should create .gemini/settings.json with SessionStart hook", async () => { - await protocol.ensureAgentConfigurations(tmpDir); + await protocol.ensureAgentConfigurations(tmpDir, ["antigravity"]); - const settingsPath = path.join(tmpDir, ".gemini", "settings.json"); - const exists = await fs.pathExists(settingsPath); - expect(exists).toBe(true); + expect(await fs.pathExists(hookRunnerPath)).toBe(false); + expect(await fs.pathExists(path.join(tmpDir, ".agents", "jumbo"))).toBe(false); - const content = await fs.readFile(settingsPath, "utf-8"); - const settings = JSON.parse(content); - expect(settings.hooks?.SessionStart).toBeDefined(); - expect(settings.hooks.SessionStart[0].matcher).toBe("startup"); - expect(settings.hooks.SessionStart[0].hooks[0].command).toBe("jumbo session start"); - expect(settings.hooks?.SessionEnd).toBeUndefined(); + const content = await fs.readFile(hooksPath, "utf-8"); + const hooks = JSON.parse(content); + expect(hooks["jumbo-session-bootstrap"]).toBeUndefined(); + expect(hooks["user-linter"].enabled).toBe(false); }); it("should create .github/copilot-instructions.md with thin reference", async () => { @@ -422,9 +430,10 @@ describe("AgentFileProtocol", () => { const claudeSettingsExists = await fs.pathExists(claudeSettingsPath); expect(claudeSettingsExists).toBe(true); - const geminiSettingsPath = path.join(tmpDir, ".gemini", "settings.json"); - const geminiSettingsExists = await fs.pathExists(geminiSettingsPath); - expect(geminiSettingsExists).toBe(true); + const antigravityHooksPath = path.join(tmpDir, ".agents", "hooks.json"); + const antigravityHooksExists = await fs.pathExists(antigravityHooksPath); + expect(antigravityHooksExists).toBe(false); + expect(await fs.pathExists(path.join(tmpDir, ".gemini", "settings.json"))).toBe(false); const copilotPath = path.join(tmpDir, ".github", "copilot-instructions.md"); const copilotExists = await fs.pathExists(copilotPath); @@ -643,6 +652,8 @@ describe("AgentFileProtocol", () => { const geminiMdPath = path.join(tmpDir, "GEMINI.md"); expect(await fs.pathExists(geminiMdPath)).toBe(true); + expect(await fs.pathExists(path.join(tmpDir, ".agents", "hooks.json"))).toBe(false); + expect(await fs.pathExists(path.join(tmpDir, ".agents", "jumbo", "antigravity-hook.mjs"))).toBe(false); const copilotPath = path.join(tmpDir, ".github", "copilot-instructions.md"); expect(await fs.pathExists(copilotPath)).toBe(true); @@ -760,6 +771,8 @@ describe("AgentFileProtocol", () => { const geminiOccurrences = (geminiContent.match(/See JUMBO\.md and follow all instructions/g) || []).length; expect(geminiOccurrences).toBe(1); + expect(await fs.pathExists(path.join(tmpDir, ".agents", "hooks.json"))).toBe(false); + const claudeSettingsPath = path.join(tmpDir, ".claude", "settings.json"); const claudeSettings = JSON.parse(await fs.readFile(claudeSettingsPath, "utf-8")); expect(claudeSettings.hooks.SessionStart.length).toBe(2); @@ -812,10 +825,6 @@ describe("AgentFileProtocol", () => { path: ".claude/skills/my-skill", description: expect.stringContaining("assets/skills"), }), - expect.objectContaining({ - path: ".gemini/skills/my-skill", - description: expect.stringContaining("assets/skills"), - }), expect.objectContaining({ path: ".vibe/skills/my-skill", description: expect.stringContaining("assets/skills"), @@ -829,17 +838,15 @@ describe("AgentFileProtocol", () => { await fs.ensureDir(templateSkillPath); await fs.writeFile(path.join(templateSkillPath, "SKILL.md"), "# My Skill\n", "utf-8"); - const changes = await protocol.getPlannedFileChanges(tmpDir, ["gemini", "copilot"]); + const changes = await protocol.getPlannedFileChanges(tmpDir, ["antigravity", "copilot"]); expect(changes).toEqual( expect.arrayContaining([ expect.objectContaining({ path: "JUMBO.md" }), expect.objectContaining({ path: "AGENTS.md" }), expect.objectContaining({ path: "GEMINI.md" }), - expect.objectContaining({ path: ".gemini/settings.json" }), expect.objectContaining({ path: ".github/copilot-instructions.md" }), expect.objectContaining({ path: ".github/hooks/hooks.json" }), - expect.objectContaining({ path: ".gemini/skills/my-skill" }), expect.objectContaining({ path: ".agents/skills/my-skill" }), ]) ); diff --git a/tests/presentation/cli/commands/project/init/project.init.test.ts b/tests/presentation/cli/commands/project/init/project.init.test.ts index acff212e..36f1532d 100644 --- a/tests/presentation/cli/commands/project/init/project.init.test.ts +++ b/tests/presentation/cli/commands/project/init/project.init.test.ts @@ -39,7 +39,7 @@ import type { AddValuePropositionController } from "../../../../../../src/applic describe("project.init command", () => { const availableAgents = [ { id: "claude", name: "Claude" }, - { id: "gemini", name: "Gemini" }, + { id: "antigravity", name: "Antigravity" }, { id: "copilot", name: "Copilot" }, ] as const; let mockContainer: Partial<IApplicationContainer>; @@ -560,7 +560,7 @@ describe("project.init command", () => { mockInteractiveSkipAllPrimitives( {}, { enabled: true }, - { selectedAgentIds: ["gemini", "copilot"] }, + { selectedAgentIds: ["antigravity", "copilot"] }, ); await projectInit({ yolo: true }, mockContainer as IApplicationContainer); @@ -574,7 +574,7 @@ describe("project.init command", () => { mockContainer.planProjectInitController!.handle, ).toHaveBeenNthCalledWith(2, { projectRoot: process.cwd(), - selectedAgentIds: ["gemini", "copilot"], + selectedAgentIds: ["antigravity", "copilot"], }); expect( mockContainer.initializeProjectController!.handle, @@ -582,7 +582,7 @@ describe("project.init command", () => { name: "TestProject", purpose: undefined, projectRoot: process.cwd(), - selectedAgentIds: ["gemini", "copilot"], + selectedAgentIds: ["antigravity", "copilot"], }); }); diff --git a/tests/presentation/cli/commands/work/review/work.review.test.ts b/tests/presentation/cli/commands/work/review/work.review.test.ts index 5fb1a1e5..ce76d39f 100644 --- a/tests/presentation/cli/commands/work/review/work.review.test.ts +++ b/tests/presentation/cli/commands/work/review/work.review.test.ts @@ -14,7 +14,7 @@ jest.unstable_mockModule("../../../../../../src/presentation/cli/commands/work/s })); jest.unstable_mockModule("../../../../../../src/presentation/cli/commands/work/shared/AgentSpawner.js", () => ({ - SUPPORTED_AGENTS: ["claude", "gemini"], + SUPPORTED_AGENTS: ["claude", "antigravity"], spawnAgent: jest.fn().mockResolvedValue(0), })); diff --git a/tests/presentation/cli/commands/work/shared/AgentSpawner.test.ts b/tests/presentation/cli/commands/work/shared/AgentSpawner.test.ts index 282b34f8..8db1f448 100644 --- a/tests/presentation/cli/commands/work/shared/AgentSpawner.test.ts +++ b/tests/presentation/cli/commands/work/shared/AgentSpawner.test.ts @@ -9,12 +9,13 @@ describe("AgentSpawner", () => { describe("SUPPORTED_AGENTS", () => { it("includes all expected agent identifiers", () => { expect(SUPPORTED_AGENTS).toContain("claude"); - expect(SUPPORTED_AGENTS).toContain("gemini"); + expect(SUPPORTED_AGENTS).toContain("antigravity"); expect(SUPPORTED_AGENTS).toContain("copilot"); expect(SUPPORTED_AGENTS).toContain("codex"); expect(SUPPORTED_AGENTS).toContain("cursor"); expect(SUPPORTED_AGENTS).toContain("vibe"); expect(SUPPORTED_AGENTS).toHaveLength(6); + expect(SUPPORTED_AGENTS).not.toContain("gemini"); }); }); @@ -47,8 +48,8 @@ describe("AgentSpawner", () => { expect(AGENT_COMMANDS.claude.executable).toBe("claude"); }); - it("maps gemini to gemini executable", () => { - expect(AGENT_COMMANDS.gemini.executable).toBe("gemini"); + it("maps antigravity to the documented agy executable", () => { + expect(AGENT_COMMANDS.antigravity.executable).toBe("agy"); }); }); }); diff --git a/tests/presentation/cli/commands/work/shared/DaemonDisplay.test.ts b/tests/presentation/cli/commands/work/shared/DaemonDisplay.test.ts index 6fa0dc52..88830587 100644 --- a/tests/presentation/cli/commands/work/shared/DaemonDisplay.test.ts +++ b/tests/presentation/cli/commands/work/shared/DaemonDisplay.test.ts @@ -128,9 +128,9 @@ describe("DaemonDisplay", () => { describe("renderUnknownAgent", () => { it("outputs the unknown agent and supported list", () => { - display.renderUnknownAgent("badagent", ["claude", "gemini"]); + display.renderUnknownAgent("badagent", ["claude", "antigravity"]); expect(captured).toContain("badagent"); - expect(captured).toContain("claude, gemini"); + expect(captured).toContain("claude, antigravity"); }); }); diff --git a/tests/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.test.ts b/tests/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.test.ts index b2bdbeee..f695291c 100644 --- a/tests/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.test.ts +++ b/tests/presentation/tui/cockpit/CockpitDaemonAgentConfigCycler.test.ts @@ -12,6 +12,16 @@ describe("CockpitDaemonAgentConfigCycler", () => { }); }); + it("cycles from Claude to Antigravity instead of retired Gemini", () => { + const config = { agentId: "claude", pollIntervalMs: 30_000, maxRetries: 3 }; + + expect(getNextCockpitDaemonAgentConfig(config)).toEqual({ + agentId: "antigravity", + pollIntervalMs: 30_000, + maxRetries: 3, + }); + }); + it("starts at the first agent when the current agent is not configured", () => { expect(getNextCockpitDaemonAgentConfig({ agentId: "unknown-agent",