From 7505fecc33923033a147800b9fec1257017d3af3 Mon Sep 17 00:00:00 2001 From: joshwheelock Date: Thu, 9 Jul 2026 10:00:40 +0200 Subject: [PATCH] fix: codex-cli-adapter cmd updated --- evals/README.md | 2 +- ...n-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt | 274 ++++++++++++++++++ evals/src/harness/codex-cli-adapter.ts | 195 +++++++++++-- evals/tests/unit/codex-cli-adapter.test.ts | 40 ++- 4 files changed, 479 insertions(+), 32 deletions(-) create mode 100644 evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt diff --git a/evals/README.md b/evals/README.md index 5db6c625..d78a1f61 100644 --- a/evals/README.md +++ b/evals/README.md @@ -150,7 +150,7 @@ All commands support `--help` for full usage details. | Harness | CLI | Adapter | |---------|-----|---------| | Claude Code | `claude -p --output-format json` | `ClaudeCodeAdapter` | -| Codex CLI | `codex --quiet --json` | `CodexCliAdapter` | +| Codex CLI | `codex --ask-for-approval never --sandbox workspace-write exec --json --skip-git-repo-check` | `CodexCliAdapter` | | Gemini CLI | `gemini --json` | `GeminiCliAdapter` | ## Report Output diff --git a/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt b/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt new file mode 100644 index 00000000..4fd043bb --- /dev/null +++ b/evals/ab-comparison-7ac1ef79-0105-411e-a604-62c85ab3c8a4.txt @@ -0,0 +1,274 @@ +════════════════════════════════════════════════════════════ + A/B Comparison: 7ac1ef79-0105-411e-a604-62c85ab3c8a4 + Harness: codex-cli +════════════════════════════════════════════════════════════ + + JUMBO RUN (JUMBO_ENABLED=true) +──────────────────────────────────────────────────────────── + Session 1: 3 files modified + Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + Session 2: 2 files modified + Files: src/events/types.ts, src/projections/inventory.ts + Session 3: 4 files modified + Files: src/events/factory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + Session 4: 1 files modified + Files: src/queries/inventoryAvailability.ts + Session 5: 3 files modified + Files: src/commands/inventory.ts, src/projections/inventory.ts, src/queries/inventoryAvailability.ts + Session 6: 4 files modified + Files: src/commands/inventory.ts, src/events/factory.ts, src/events/types.ts, src/projections/inventory.ts + Session 7: 1 files modified + Files: src/queries/inventoryAvailability.ts + + BASELINE RUN (JUMBO_ENABLED=false) +──────────────────────────────────────────────────────────── + Session 1: 3 files modified + Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + Session 2: 2 files modified + Files: src/events/types.ts, src/projections/inventory.ts + Session 3: 3 files modified + Files: src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + Session 4: 2 files modified + Files: src/events/types.ts, src/projections/inventory.ts + Session 5: 4 files modified + Files: src/commands/inventory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + Session 6: 3 files modified + Files: src/commands/inventory.ts, src/events/types.ts, src/projections/inventory.ts + Session 7: 3 files modified + Files: src/commands/inventory.ts, src/events/types.ts, src/projections/inventory.ts + + SCORES +════════════════════════════════════════════════════════════ + Dimension Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + file-accuracy 0.55/1.00 0.67/1.00 -0.12 + knowledge-retention 0.57/1.00 0.57/1.00 0.00 + structural-retention 1.00/1.00 1.00/1.00 0.00 + disruption-recovery 1.00/1.00 1.00/1.00 0.00 + jumbo-memory-capture 0.25/1.00 0.00/0.00 +0.25 + jumbo-event-capture 1.00/1.00 0.00/0.00 +1.00 + protocol-adherence 0.33/1.00 0.00/0.00 +0.33 + token-efficiency 0.00 (ref) — + └─ N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). + + JUMBO MEMORY CAPTURE EVIDENCE +──────────────────────────────────────────────────────────── + Jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} + Baseline: Not applicable: baseline runs do not use Jumbo project memory. + + PROTOCOL ADHERENCE EVIDENCE +──────────────────────────────────────────────────────────── + Jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) + Baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. + + TIMELINE (per-session scores) +════════════════════════════════════════════════════════════ + file-accuracy + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 0.75/1.00 0.75/1.00 0.00 + 2 0.57/1.00 0.57/1.00 0.00 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 0.67/1.00 0.75/1.00 -0.08 + 4 0.00/1.00 0.57/1.00 -0.57 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 0.25/1.00 0.67/1.00 -0.42 + 6 0.44/1.00 0.50/1.00 -0.06 + 7 0.00/1.00 0.50/1.00 -0.50 + + knowledge-retention + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 0.57/1.00 0.57/1.00 0.00 + 2 0.57/1.00 0.57/1.00 0.00 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 0.57/1.00 0.57/1.00 0.00 + 4 0.57/1.00 0.57/1.00 0.00 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 0.57/1.00 0.57/1.00 0.00 + 6 0.57/1.00 0.57/1.00 0.00 + 7 0.57/1.00 0.57/1.00 0.00 + + structural-retention + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 1.00/1.00 1.00/1.00 0.00 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 1.00/1.00 1.00/1.00 0.00 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 1.00/1.00 1.00/1.00 0.00 + 7 1.00/1.00 1.00/1.00 0.00 + + disruption-recovery + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 1.00/1.00 1.00/1.00 0.00 + 2 1.00/1.00 1.00/1.00 0.00 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 1.00/1.00 1.00/1.00 0.00 + 4 1.00/1.00 1.00/1.00 0.00 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 1.00/1.00 1.00/1.00 0.00 + 6 1.00/1.00 1.00/1.00 0.00 + 7 1.00/1.00 1.00/1.00 0.00 + + jumbo-memory-capture + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 0.22/1.00 0.00/0.00 +0.22 + 2 0.22/1.00 0.00/0.00 +0.22 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 0.17/1.00 0.00/0.00 +0.17 + 4 0.15/1.00 0.00/0.00 +0.15 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 0.25/1.00 0.00/0.00 +0.25 + 6 0.25/1.00 0.00/0.00 +0.25 + 7 0.25/1.00 0.00/0.00 +0.25 + + jumbo-event-capture + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 1.00/1.00 0.00/0.00 +1.00 + 2 1.00/1.00 0.00/0.00 +1.00 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 1.00/1.00 0.00/0.00 +1.00 + 4 1.00/1.00 0.00/0.00 +1.00 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 1.00/1.00 0.00/0.00 +1.00 + 6 1.00/1.00 0.00/0.00 +1.00 + 7 1.00/1.00 0.00/0.00 +1.00 + + protocol-adherence + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 0.33/1.00 0.00/0.00 +0.33 + 2 0.33/1.00 0.00/0.00 +0.33 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 0.33/1.00 0.00/0.00 +0.33 + 4 0.33/1.00 0.00/0.00 +0.33 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 0.33/1.00 0.00/0.00 +0.33 + 6 0.33/1.00 0.00/0.00 +0.33 + 7 0.33/1.00 0.00/0.00 +0.33 + + token-usage + Session Jumbo Baseline Delta +──────────────────────────────────────────────────────────── + 1 744030 250088 +493942 + 2 1017547 232222 +785325 + >>> [CORRECTION] The event types need a 'metadata' field containing... + 3 2049125 453763 +1595362 + 4 1128233 464308 +663925 + >>> [NEW-CONSTRAINT] Add optimistic concurrency control. Each aggregate... + 5 2703659 752580 +1951079 + 6 1868112 536532 +1331580 + 7 1019923 443985 +575938 + +════════════════════════════════════════════════════════════ +════════════════════════════════════════════════════════════════════════ + JUMBO EVALUATION REPORT + Scenario: 7ac1ef79-0105-411e-a604-62c85ab3c8a4 + Harnesses: codex-cli + Treatment: whole Jumbo system vs Jumbo-unreachable baseline + Methodology: framework-as-developer orchestration, agent-driven lifecycle + Lift attribution: init + pre-seeded memory + lifecycle protocol + agent capture + audit +════════════════════════════════════════════════════════════════════════ + + LIFT BY DIMENSION +──────────────────────────────────────────────────────────────────────── + Dimension Jumbo Base Lift Lift % +──────────────────────────────────────────────────────────────────────── + file-accuracy 0.55 0.67 -0.120 -17.9% + knowledge-retention 0.57 0.57 0.000 0.0% + structural-retention 1.00 1.00 0.000 0.0% + disruption-recovery 1.00 1.00 0.000 0.0% + jumbo-memory-capture 0.25 0.00 +0.250 n/a + jumbo-event-capture 1.00 0.00 +1.000 n/a + protocol-adherence 0.33 0.00 +0.330 n/a + token-efficiency 0.00 0.00 0.000 n/a + + DIVERGENCE ONSET (session where amnesia impact exceeds threshold) +──────────────────────────────────────────────────────────────────────── + file-accuracy session 4 (delta: -0.570, threshold: 0.1) + knowledge-retention no significant divergence detected + structural-retention no significant divergence detected + disruption-recovery no significant divergence detected + jumbo-memory-capture session 1 (delta: +0.220, threshold: 0.1) + jumbo-event-capture session 1 (delta: +1.000, threshold: 0.1) + protocol-adherence session 1 (delta: +0.330, threshold: 0.1) + token-usage session 1 (delta: +493942.000, threshold: 0.1) + + DIVERGENCE CURVE (session-by-session delta) +──────────────────────────────────────────────────────────────────────── + file-accuracy S1:0.00 S2:0.00 S3:-0.08 S4:-0.57 S5:-0.42 S6:-0.06 S7:-0.50 + knowledge-retention S1:0.00 S2:0.00 S3:0.00 S4:0.00 S5:0.00 S6:0.00 S7:0.00 + structural-retention S1:0.00 S3:0.00 S5:0.00 S7:0.00 + disruption-recovery S1:0.00 S2:0.00 S3:0.00 S4:0.00 S5:0.00 S6:0.00 S7:0.00 + jumbo-memory-capture S1:+0.22 S2:+0.22 S3:+0.17 S4:+0.15 S5:+0.25 S6:+0.25 S7:+0.25 + jumbo-event-capture S1:+1.00 S2:+1.00 S3:+1.00 S4:+1.00 S5:+1.00 S6:+1.00 S7:+1.00 + protocol-adherence S1:+0.33 S2:+0.33 S3:+0.33 S4:+0.33 S5:+0.33 S6:+0.33 S7:+0.33 + token-usage S1:+493942.00 S2:+785325.00 S3:+1595362.00 S4:+663925.00 S5:+1951079.00 S6:+1331580.00 S7:+575938.00 + + JUMBO MEMORY CAPTURE EVIDENCE +──────────────────────────────────────────────────────────────────────── + codex-cli score: 0.25 + Jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} + Baseline: Not applicable: baseline runs do not use Jumbo project memory. + + PROTOCOL ADHERENCE EVIDENCE (per-step lifecycle execution) +──────────────────────────────────────────────────────────────────────── + codex-cli score: 0.33 + Jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) + Baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. + S1: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S2: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S3: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S4: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S5: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S6: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + S7: 0.33 (passed: session-start,in-session-captures; failed: goal-start,progress-updates,goal-submit,session-end) + + DISRUPTION IMPACT (ranked by magnitude) +──────────────────────────────────────────────────────────────────────── + [new-constraint] S5 token-usage before:+663925.000 after:+1951079.000 magnitude:1287154.000 + [correction] S3 token-usage before:+785325.000 after:+1595362.000 magnitude:810037.000 + [new-constraint] S5 file-accuracy before:-0.570 after:-0.420 magnitude:0.150 + [new-constraint] S5 jumbo-memory-capture before:+0.150 after:+0.250 magnitude:0.100 + [correction] S3 file-accuracy before:+0.000 after:-0.080 magnitude:0.080 + [correction] S3 jumbo-memory-capture before:+0.220 after:+0.170 magnitude:0.050 + [correction] S3 knowledge-retention before:+0.000 after:+0.000 magnitude:0.000 + [correction] S3 structural-retention before:+0.000 after:+0.000 magnitude:0.000 + [correction] S3 disruption-recovery before:+0.000 after:+0.000 magnitude:0.000 + [correction] S3 jumbo-event-capture before:+1.000 after:+1.000 magnitude:0.000 + + AUDIT TRAIL (effective context, final snapshot, scoring evidence) +──────────────────────────────────────────────────────────────────────── + codex-cli + Jumbo final session 7 delivered context: none + Baseline final session 7 effective prompt: Continue building the inventory system. Review the existing code and event ty... + Jumbo final snapshot files: .agents/skills/codify-jumbo-goal/SKILL.md, .agents/skills/decompose-architecture-aggregate/SKILL.md, .agents/skills/define-jumbo-goals/SKILL.md, .agents/skills/formalize-objectives/SKILL.md, .agents/skills/jumbo-add-component/SKILL.md, .agents/skills/jumbo-add-decision/SKILL.md, .agents/skills/jumbo-add-dependency/SKILL.md, .agents/skills/jumbo-add-guideline/SKILL.md, .agents/skills/jumbo-add-invariant/SKILL.md, .agents/skills/jumbo-bootstrap-session/SKILL.md, .agents/skills/jumbo-command-discovery/SKILL.md, .agents/skills/jumbo-context-maintenance/SKILL.md, .agents/skills/jumbo-correction-capture/SKILL.md, .agents/skills/jumbo-design-goal/SKILL.md, .agents/skills/refine-jumbo-goals/SKILL.md, .agents/skills/reject-jumbo-goal/SKILL.md, .agents/skills/review-jumbo-goal/SKILL.md, .agents/skills/start-jumbo-goal/SKILL.md, .claude/settings.json, .claude/skills/codify-jumbo-goal/SKILL.md, .claude/skills/decompose-architecture-aggregate/SKILL.md, .claude/skills/define-jumbo-goals/SKILL.md, .claude/skills/formalize-objectives/SKILL.md, .claude/skills/jumbo-add-component/SKILL.md, .claude/skills/jumbo-add-decision/SKILL.md, .claude/skills/jumbo-add-dependency/SKILL.md, .claude/skills/jumbo-add-guideline/SKILL.md, .claude/skills/jumbo-add-invariant/SKILL.md, .claude/skills/jumbo-bootstrap-session/SKILL.md, .claude/skills/jumbo-command-discovery/SKILL.md, .claude/skills/jumbo-context-maintenance/SKILL.md, .claude/skills/jumbo-correction-capture/SKILL.md, .claude/skills/jumbo-design-goal/SKILL.md, .claude/skills/refine-jumbo-goals/SKILL.md, .claude/skills/reject-jumbo-goal/SKILL.md, .claude/skills/review-jumbo-goal/SKILL.md, .claude/skills/start-jumbo-goal/SKILL.md, .codex/config.toml, .codex/hooks.json, .cursor/hooks.json, .cursor/rules/jumbo.mdc, .gemini/settings.json, .gemini/skills/codify-jumbo-goal/SKILL.md, .gemini/skills/decompose-architecture-aggregate/SKILL.md, .gemini/skills/define-jumbo-goals/SKILL.md, .gemini/skills/formalize-objectives/SKILL.md, .gemini/skills/jumbo-add-component/SKILL.md, .gemini/skills/jumbo-add-decision/SKILL.md, .gemini/skills/jumbo-add-dependency/SKILL.md, .gemini/skills/jumbo-add-guideline/SKILL.md, .gemini/skills/jumbo-add-invariant/SKILL.md, .gemini/skills/jumbo-bootstrap-session/SKILL.md, .gemini/skills/jumbo-command-discovery/SKILL.md, .gemini/skills/jumbo-context-maintenance/SKILL.md, .gemini/skills/jumbo-correction-capture/SKILL.md, .gemini/skills/jumbo-design-goal/SKILL.md, .gemini/skills/refine-jumbo-goals/SKILL.md, .gemini/skills/reject-jumbo-goal/SKILL.md, .gemini/skills/review-jumbo-goal/SKILL.md, .gemini/skills/start-jumbo-goal/SKILL.md, .github/copilot-instructions.md, .github/hooks/hooks.json, .gitignore, .vibe/skills/codify-jumbo-goal/SKILL.md, .vibe/skills/decompose-architecture-aggregate/SKILL.md, .vibe/skills/define-jumbo-goals/SKILL.md, .vibe/skills/formalize-objectives/SKILL.md, .vibe/skills/jumbo-add-component/SKILL.md, .vibe/skills/jumbo-add-decision/SKILL.md, .vibe/skills/jumbo-add-dependency/SKILL.md, .vibe/skills/jumbo-add-guideline/SKILL.md, .vibe/skills/jumbo-add-invariant/SKILL.md, .vibe/skills/jumbo-bootstrap-session/SKILL.md, .vibe/skills/jumbo-command-discovery/SKILL.md, .vibe/skills/jumbo-context-maintenance/SKILL.md, .vibe/skills/jumbo-correction-capture/SKILL.md, .vibe/skills/jumbo-design-goal/SKILL.md, .vibe/skills/refine-jumbo-goals/SKILL.md, .vibe/skills/reject-jumbo-goal/SKILL.md, .vibe/skills/review-jumbo-goal/SKILL.md, .vibe/skills/start-jumbo-goal/SKILL.md, AGENTS.md, CLAUDE.md, GEMINI.md, JUMBO.md, src/commands/inventory.ts, src/events/factory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts, src/queries/inventoryAvailability.ts + Baseline final snapshot files: .codex/config.toml, .eval-bin/jumbo.bat, .eval-bin/jumbo.cmd, src/commands/inventory.ts, src/events/store.ts, src/events/types.ts, src/projections/inventory.ts + file-accuracy jumbo=0.55 baseline=0.67 delta=-0.120 + jumbo: 3/5 expected files modified; missed: src/commands/handlers.ts, src/domain/product.ts; unexpected: src/events/factory.ts, src/queries/inventoryAvailability.ts, src/commands/inventory.ts; precision=0.50 recall=0.60 f1=0.55 + baseline: 3/5 expected files modified; missed: src/commands/handlers.ts, src/domain/product.ts; unexpected: src/commands/inventory.ts; precision=0.75 recall=0.60 f1=0.67 + knowledge-retention jumbo=0.57 baseline=0.57 delta=0.000 + jumbo: 4/7 patterns retained in session 7; lost: discriminated union, append-only, event store + baseline: 4/7 patterns retained in session 7; lost: discriminated union, append-only, event store + structural-retention jumbo=1.00 baseline=1.00 delta=0.000 + jumbo: 11/11 structural assertions passed + baseline: 11/11 structural assertions passed + disruption-recovery jumbo=1.00 baseline=1.00 delta=0.000 + jumbo: 20/20 recovery checks passed + baseline: 20/20 recovery checks passed + jumbo-memory-capture jumbo=0.25 baseline=0.00 delta=+0.250 + jumbo: precision=0.17; recall=0.50; matched=2/4; new-entities=67; missing: decision:event-sourced; invariant:append-only; spurious: decision:{"decisionId":"c78274cc-c176-478c-879f-bf3d74c7af7e","title":"Use event sourcing as the inventory state model","context":"The warehouse domain needs to track product additions, stock changes, reservations, confirmations, and cancellations as an auditable sequence of facts.","rationale":"An append-only event stream preserves domain history and allows current stock and reservation state to be rebuilt deterministically by replay.","alternatives":["Maintain mutable inventory records directly in a database or object map."],"consequences":"Read models must be projected from events, and invalid historical events require compensating events rather than mutation.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:11:36.594Z","updatedAt":"2026-07-08T22:11:36.594Z"}; invariant:{"invariantId":"9ed7166d-15b8-4e8e-8651-9c1bb5c06d3e","title":"Inventory state is derived from events","description":"Current inventory read state must be rebuilt from the ordered domain event stream rather than treated as the source of truth.","rationale":"Keeping the event stream authoritative preserves auditability and prevents projection drift from replacing domain history.","createdAt":"2026-07-08T22:11:36.556Z","updatedAt":"2026-07-08T22:11:36.556Z"}; invariant:{"invariantId":"85afbb6e-4600-4b7b-b99b-8ba68b58f9a7","title":"Domain events are immutable facts","description":"Recorded inventory domain events must not be modified after append; corrections must be represented by later domain events.","rationale":"Event sourcing relies on event history remaining stable so projections and audits are deterministic.","createdAt":"2026-07-08T22:11:36.710Z","updatedAt":"2026-07-08T22:11:36.710Z"}; component:{"componentId":"7f5ba2ab-4ac4-44e9-8d76-fc0b9436be7b","name":"InMemoryEventStore","type":"service","description":"Append-only in-memory log for inventory domain events","responsibility":"Store and replay domain events in insertion order without database persistence","path":"src/events/store.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.534Z","updatedAt":"2026-07-08T22:11:21.534Z"}; component:{"componentId":"c617a3bb-d0de-463d-9f71-7e2bde2555e3","name":"InventoryProjection","type":"service","description":"Read-model projector that rebuilds current warehouse inventory state by replaying domain events","responsibility":"Derive products, stock levels, availability, and reservations from the event stream","path":"src/projections/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:11:21.551Z","updatedAt":"2026-07-08T22:11:21.551Z"}; decision:{"decisionId":"e54581e1-4b09-4727-9e59-4938604685f0","title":"Use typed aggregate keys for projection freshness","context":"Inventory projections now need to report the latest event timestamp for different aggregate kinds from a single read model.","rationale":"Prefixing identifiers as product: and reservation: keeps the freshness map compact while avoiding collisions between product and reservation identifiers.","alternatives":["Expose separate latest timestamp maps per aggregate type","Use raw aggregate identifiers without a type prefix"],"consequences":"Consumers must use the documented aggregate key format when reading freshness timestamps from InventoryState.","status":"active","supersededBy":null,"reversalReason":null,"reversedAt":null,"createdAt":"2026-07-08T22:30:59.062Z","updatedAt":"2026-07-08T22:30:59.062Z"}; invariant:{"invariantId":"fd2a04b3-5d45-4cad-817b-defe825c9b17","title":"Inventory events carry metadata","description":"Every inventory domain event must include a metadata object containing timestamp, correlationId, and causationId values; projections must use metadata.timestamp as the event time.","rationale":"Event-sourced inventory history needs consistent ordering, traceability, correlation, causation, and aggregate freshness tracking across all event types.","createdAt":"2026-07-08T22:25:16.086Z","updatedAt":"2026-07-08T22:25:16.086Z"}; component:{"componentId":"fc088791-df0a-4c96-8693-223a4e01deb1","name":"InventoryEventFactory","type":"lib","description":"Typed construction helpers for inventory domain events with trace metadata","responsibility":"Create inventory domain event objects that consistently include eventId, type, payload, and metadata","path":"src/events/factory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:26:35.417Z","updatedAt":"2026-07-08T22:26:35.417Z"}; component:{"componentId":"eec30979-76b6-4345-9274-bc89c155c489","name":"InventoryAvailabilityQueries","type":"lib","description":"Read-side query helpers for product availability and reservation capacity over projected inventory state","responsibility":"Expose immutable availability snapshots from InventoryState without mutating event-sourced state","path":"src/queries/inventoryAvailability.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:39:38.770Z","updatedAt":"2026-07-08T22:39:38.770Z"}; component:{"componentId":"ba38a384-127a-498f-b57d-ee18f0a11ca5","name":"InventoryCommandHandler","type":"service","description":"Handles inventory write commands by validating projected state, checking expected aggregate versions, and appending typed domain events","responsibility":"Optimistic-concurrency-protected inventory command execution","path":"src/commands/inventory.ts","status":"active","deprecationReason":null,"createdAt":"2026-07-08T22:52:51.979Z","updatedAt":"2026-07-08T22:52:51.979Z"} + baseline: Not applicable: baseline runs do not use Jumbo project memory. + jumbo-event-capture jumbo=1.00 baseline=0.00 delta=+1.000 + jumbo: 3/3 expected entity kinds evidenced in .jumbo/events; missing: none + baseline: Not applicable: baseline runs have no .jumbo/events log. + protocol-adherence jumbo=0.33 baseline=0.00 delta=+0.330 + jumbo: sessions-audited=7; steps-passed=14/42; goal-start-missed-in-sessions:1,2,3,4,5,6,7; progress-updates-missed-in-sessions:1,2,3,4,5,6,7; goal-submit-missed-in-sessions:1,2,3,4,5,6,7; session-end-missed-in-sessions:1,2,3,4,5,6,7; adherence-rate=0.00 (adherent-sessions=0/7, threshold=0.5) + baseline: Not applicable: baseline runs do not execute the Jumbo lifecycle protocol. + token-efficiency jumbo=0.00 baseline=0.00 delta=0.000 + jumbo: N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). + baseline: N/A: outputs not equivalent — token efficiency would compare the cost of producing different things (structural: jumbo=1.00, baseline=1.00, >=0.8 required; all expected files produced: jumbo=false, baseline=false). \ No newline at end of file diff --git a/evals/src/harness/codex-cli-adapter.ts b/evals/src/harness/codex-cli-adapter.ts index d83a35bc..348912ce 100644 --- a/evals/src/harness/codex-cli-adapter.ts +++ b/evals/src/harness/codex-cli-adapter.ts @@ -5,10 +5,10 @@ import type { HarnessAdapter } from './harness-adapter.js'; /** * HarnessAdapter for OpenAI Codex CLI. - * Invokes via 'codex --quiet --json' which accepts a prompt - * and returns JSON output with the response and metadata. + * Invokes via 'codex exec --json' and sends the prompt on stdin. + * Current Codex CLI JSON mode emits newline-delimited event objects. * - * Codex CLI output format (JSON mode): + * Older fixtures may still use the previous single-object JSON shape: * { * "response": "...", * "files_modified": ["..."], @@ -19,7 +19,16 @@ export class CodexCliAdapter implements HarnessAdapter { readonly name = 'codex-cli'; buildCommand(): string[] { - return ['codex', '--quiet', '--json']; + return [ + 'codex', + '--ask-for-approval', + 'never', + '--sandbox', + 'workspace-write', + 'exec', + '--json', + '--skip-git-repo-check', + ]; } /** @@ -58,36 +67,164 @@ export class CodexCliAdapter implements HarnessAdapter { try { const parsed = JSON.parse(result.stdout); - - if (parsed.response) { - agentOutput = parsed.response; + const single = parseCodexJsonValue(parsed); + agentOutput = single.agentOutput ?? agentOutput; + filesModified.push(...single.filesModified); + inputTokens = single.inputTokens; + outputTokens = single.outputTokens; + } catch { + const jsonl = parseCodexJsonLines(result.stdout); + if (jsonl) { + agentOutput = jsonl.agentOutput ?? agentOutput; + filesModified.push(...jsonl.filesModified); + inputTokens = jsonl.inputTokens; + outputTokens = jsonl.outputTokens; } + } - if (Array.isArray(parsed.files_modified)) { - filesModified.push(...parsed.files_modified); - } + return { agentOutput, filesModified, transcript, inputTokens, outputTokens }; + } +} - // Codex CLI uses prompt_tokens / completion_tokens in usage object - if (parsed.usage) { - if (typeof parsed.usage.prompt_tokens === 'number') { - inputTokens = parsed.usage.prompt_tokens; - } - if (typeof parsed.usage.completion_tokens === 'number') { - outputTokens = parsed.usage.completion_tokens; - } - } +interface ParsedCodexOutput { + readonly agentOutput?: string; + readonly filesModified: readonly string[]; + readonly inputTokens?: number; + readonly outputTokens?: number; +} - // Also check top-level token fields - if (typeof parsed.prompt_tokens === 'number') { - inputTokens = parsed.prompt_tokens; - } - if (typeof parsed.completion_tokens === 'number') { - outputTokens = parsed.completion_tokens; - } - } catch { - // Non-JSON output — use raw stdout as agent output +function parseCodexJsonLines(stdout: string): ParsedCodexOutput | undefined { + const values = stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map(parseJsonLine) + .filter((value): value is unknown => value !== undefined); + + if (values.length === 0) return undefined; + + return mergeCodexValues(values); +} + +function parseJsonLine(line: string): unknown | undefined { + try { + return JSON.parse(line); + } catch { + return undefined; + } +} + +function parseCodexJsonValue(value: unknown): ParsedCodexOutput { + if (Array.isArray(value)) { + return mergeCodexValues(value); + } + return mergeCodexValues([value]); +} + +function mergeCodexValues(values: readonly unknown[]): ParsedCodexOutput { + const filesModified: string[] = []; + let agentOutput: string | undefined; + let inputTokens: number | undefined; + let outputTokens: number | undefined; + + for (const value of values) { + const files = extractStringArray(value, 'files_modified'); + if (files) filesModified.push(...files); + + const tokens = extractTokenUsage(value); + inputTokens = tokens.inputTokens ?? inputTokens; + outputTokens = tokens.outputTokens ?? outputTokens; + + const text = extractAgentText(value); + if (text !== undefined && text.length > 0) { + agentOutput = text; } + } - return { agentOutput, filesModified, transcript, inputTokens, outputTokens }; + return { + agentOutput, + filesModified: [...new Set(filesModified)], + inputTokens, + outputTokens, + }; +} + +function extractAgentText(value: unknown): string | undefined { + if (!isRecord(value)) return undefined; + + const direct = firstString(value, ['response', 'result', 'agent_output', 'final_message']); + if (direct !== undefined) return direct; + + const type = typeof value.type === 'string' ? value.type : ''; + if (type.includes('message') || type.includes('completed')) { + const eventText = firstString(value, ['message', 'text']); + if (eventText !== undefined) return eventText; + + const contentText = extractContentText(value.content); + if (contentText !== undefined) return contentText; + } + + if (isRecord(value.item)) { + const role = typeof value.item.role === 'string' ? value.item.role : ''; + const itemType = typeof value.item.type === 'string' ? value.item.type : ''; + if (role === 'assistant' || itemType.includes('message')) { + const itemText = firstString(value.item, ['message', 'text']); + if (itemText !== undefined) return itemText; + return extractContentText(value.item.content); + } + } + + return undefined; +} + +function extractContentText(content: unknown): string | undefined { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return undefined; + + const parts = content + .map((part) => { + if (typeof part === 'string') return part; + if (!isRecord(part)) return undefined; + return firstString(part, ['text', 'output_text']); + }) + .filter((part): part is string => part !== undefined && part.length > 0); + + return parts.length > 0 ? parts.join('') : undefined; +} + +function extractTokenUsage(value: unknown): { inputTokens?: number; outputTokens?: number } { + if (!isRecord(value)) return {}; + + const usage = isRecord(value.usage) ? value.usage : value; + return { + inputTokens: firstNumber(usage, ['prompt_tokens', 'input_tokens']), + outputTokens: firstNumber(usage, ['completion_tokens', 'output_tokens']), + }; +} + +function extractStringArray(value: unknown, key: string): string[] | undefined { + if (!isRecord(value)) return undefined; + const candidate = value[key]; + if (!Array.isArray(candidate)) return undefined; + return candidate.filter((item): item is string => typeof item === 'string'); +} + +function firstString(record: Record, keys: readonly string[]): string | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === 'string') return value; + } + return undefined; +} + +function firstNumber(record: Record, keys: readonly string[]): number | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === 'number') return value; } + return undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; } diff --git a/evals/tests/unit/codex-cli-adapter.test.ts b/evals/tests/unit/codex-cli-adapter.test.ts index 28a7949f..1dc81593 100644 --- a/evals/tests/unit/codex-cli-adapter.test.ts +++ b/evals/tests/unit/codex-cli-adapter.test.ts @@ -13,7 +13,16 @@ describe('CodexCliAdapter', () => { it('builds a codex command without a positional prompt (prompt goes via stdin)', () => { const command = adapter.buildCommand(); - expect(command).toEqual(['codex', '--quiet', '--json']); + expect(command).toEqual([ + 'codex', + '--ask-for-approval', + 'never', + '--sandbox', + 'workspace-write', + 'exec', + '--json', + '--skip-git-repo-check', + ]); }); it('parses JSON output with response field', () => { @@ -71,6 +80,33 @@ describe('CodexCliAdapter', () => { expect(result.outputTokens).toBe(800); }); + it('parses Codex exec JSONL events', () => { + const result = adapter.parseOutput({ + stdout: [ + JSON.stringify({ type: 'session.started', id: 'session-1' }), + JSON.stringify({ + type: 'item.completed', + item: { + type: 'message', + role: 'assistant', + content: [{ type: 'output_text', text: 'Implemented the inventory events.' }], + }, + }), + JSON.stringify({ + type: 'turn.completed', + usage: { input_tokens: 2400, output_tokens: 900 }, + }), + ].join('\n'), + stderr: '', + exitCode: 0, + }); + + expect(result.agentOutput).toBe('Implemented the inventory events.'); + expect(result.filesModified).toEqual([]); + expect(result.inputTokens).toBe(2400); + expect(result.outputTokens).toBe(900); + }); + it('returns undefined tokens for non-JSON output', () => { const result = adapter.parseOutput({ stdout: 'plain text', @@ -97,7 +133,7 @@ describe('CodexCliAdapter', () => { it('does not embed any prompt in argv (identical-prompts invariant via stdin delivery)', () => { const command = adapter.buildCommand(); expect(command).not.toContain('Build a REST API with authentication'); - expect(command).toEqual(['codex', '--quiet', '--json']); + expect(command.at(-1)).toBe('--skip-git-repo-check'); }); describe('seedToolPermissions', () => {