diff --git a/.claude/commands/ai-review-local.md b/.claude/commands/ai-review-local.md index 7f14d284..8e73cab2 100644 --- a/.claude/commands/ai-review-local.md +++ b/.claude/commands/ai-review-local.md @@ -16,7 +16,7 @@ Two backends are supported: | Backend | Latency | Cost | Quality | |---|---|---|---| -| `api` (`gpt-5.6-sol`) | 30-60s | metered at the `PRICING` rates in `openai_review.py` via `OPENAI_API_KEY`; `--backend api --dry-run` prints the estimate for your diff | Single-shot — won't grep, can't load files on its own initiative | +| `api` (`gpt-6.1-sol`) | 30-60s (measured on gpt-5.6-sol; untested on gpt-6.1-sol, the codex backend is the validated path) | metered at the `PRICING` rates in `openai_review.py` via `OPENAI_API_KEY`; `--backend api --dry-run` prints the estimate for your diff | Single-shot — won't grep, can't load files on its own initiative | | `codex` (any auth) | 3-15 min | depends on your `codex login` mode (subscription vs API key) — see codex docs | Agentic — matches CI Codex reviewer, can grep / load files / multi-turn | Choose with `--backend {auto,codex,api}` (default `auto`): @@ -66,7 +66,7 @@ Notes: *Api backend only.* - `--force-fresh`: Skip delta-diff mode, run a full fresh review even if previous state exists - `--full-registry`: Include the entire REGISTRY.md instead of selective sections -- `--model `: Override the model (default: `gpt-5.6-sol`). Applies to both backends. +- `--model `: Override the model (default: `gpt-6.1-sol`). Applies to both backends. - `--timeout `: applies to both backends: on the api backend an HTTP request timeout (900 s default for reasoning models: gpt-5.4/gpt-5.5/gpt-5.6*/gpt-6*, *-pro, o1/o3/o4; 300 otherwise), on the codex backend a hard wall-clock cap on the codex process (uncapped when omitted). `0` means no timeout on either backend. - `--dry-run`: Print the compiled prompt without invoking the chosen backend (no API call, no codex subprocess) @@ -108,7 +108,7 @@ Step 5 invokes the chosen backend: Parse `$ARGUMENTS` for the optional flags listed above. All flags are optional — the default behavior (auto-detect backend, standard context for api or -agentic loading for codex, selective registry, gpt-5.6-sol) +agentic loading for codex, selective registry, gpt-6.1-sol) requires no arguments. ### Step 2: Validate Prerequisites @@ -419,7 +419,7 @@ Note: `--force-fresh` is a skill-only flag — it controls whether delta diffs a generated in Step 4 and is NOT passed to the script. **Reasoning model handling:** If the model starts with `gpt-5.4`/`gpt-5.5`/`gpt-5.6`/`gpt-6`, contains `-pro`, or starts with -`o1`/`o3`/`o4` (e.g., `gpt-5.6-sol`, `gpt-6-astra`, `gpt-5.4-pro`, `o3`, `o4-mini`): +`o1`/`o3`/`o4` (e.g., `gpt-6.1-sol`, `gpt-6-astra`, `gpt-5.4-pro`, `o3`, `o4-mini`): - The script auto-resolves `--timeout` to 900s for reasoning models when omitted, so no extra flag is required unless overriding - Run the Bash command with `run_in_background: true` (bypasses the 600s Bash tool timeout cap) diff --git a/.claude/scripts/openai_review.py b/.claude/scripts/openai_review.py index 0bf9837f..2d90adbd 100644 --- a/.claude/scripts/openai_review.py +++ b/.claude/scripts/openai_review.py @@ -878,6 +878,7 @@ def apply_token_budget( "gpt-5.6-luna": (0.20, 1.20), "gpt-6-astra": (10.00, 50.00), "gpt-6-sol": (2.00, 10.00), + "gpt-6.1-sol": (2.00, 10.00), "gpt-6-luna": (0.10, 0.50), "gpt-4.1": (2.00, 8.00), "gpt-4.1-mini": (0.40, 1.60), @@ -1142,7 +1143,7 @@ def compile_prompt( # --------------------------------------------------------------------------- ENDPOINT = "https://api.openai.com/v1/responses" -DEFAULT_MODEL = "gpt-5.6-sol" +DEFAULT_MODEL = "gpt-6.1-sol" DEFAULT_TIMEOUT = 300 # seconds REASONING_TIMEOUT = 900 # seconds DEFAULT_MAX_TOKENS = 16384 @@ -1347,7 +1348,7 @@ def _build_codex_cmd( ) -> "list[str]": """Construct the argv for `codex exec`. - Pinned to match the CI Codex action's invocation (gpt-5.6-sol + xhigh effort + + Pinned to match the CI Codex action's invocation (gpt-6.1-sol + xhigh effort + read-only sandbox) so local reviews give CI-equivalent quality. ``effort`` defaults to the production/CI value (xhigh) so this argv stays byte-identical for production callers; the reviewer-eval harness passes other levels diff --git a/.claude/skills/plan-review/SKILL.md b/.claude/skills/plan-review/SKILL.md index 6cdd86fb..0079b93d 100644 --- a/.claude/skills/plan-review/SKILL.md +++ b/.claude/skills/plan-review/SKILL.md @@ -1,14 +1,17 @@ --- name: plan-review -description: Review a Claude Code plan file with the campaign-selected dual engine (Opus reviewer + codex-sol, merge/verify), or revise a plan from its review. Produces the .review.md the ExitPlanMode content-hash gate requires. +description: Review a Claude Code plan file with the dual engine (Opus reviewer + codex gpt-6.1-sol, merge/verify; the Opus+Sol pair was Campaign-1 selected, run with gpt-5.6-sol, and 6.1-sol is carried over from the 2026-10 code-review campaign), or revise a plan from its review. Produces the .review.md the ExitPlanMode content-hash gate requires. --- # Plan Review (dual engine) Owns the plan-review → revise cycle. The review ENGINE is **dual** — two blind -reviewers (Claude @ Opus 4.8 + codex `gpt-5.6-sol`) then a merge/verify pass — -the configuration Campaign 1 selected: its exploratory run had dual reliably -catch **7/9** must-catch plan defects vs **1/9** for any single reviewer. That +reviewers (Claude @ Opus 4.8 + codex `gpt-6.1-sol`) then a merge/verify pass — +the configuration Campaign 1 selected (measured with gpt-5.6-sol; the codex pin +now tracks the production code-review model, carried over from the 2026-10 +code-review campaign 2 and not re-validated by plan-review-eval): its +exploratory run had dual reliably catch **7/9** must-catch plan defects vs +**1/9** for any single reviewer (measured with gpt-5.6-sol). That campaign was **NON-GATING** (contaminated negatives + a criteria regression); the sensitivity signal is what selects dual, and a clean re-validation is tracked — see `tools/plan-review-eval/verdicts/campaign-1.md`. The content-hash @@ -27,8 +30,9 @@ Bundled in this skill dir (`.claude/skills/plan-review/`): - `render.py` — strict `__TOKEN__` renderer. **Always render prompts with it; never free-text-substitute the templates** (that is a different, unmeasured engine). -- `codex_review.py` — the codex (reviewer 2) half, with the campaign - model/effort pins and loud-fallback exit codes. +- `codex_review.py` — the codex (reviewer 2) half, with the production-tracked + model/effort pins (see its header comment for validation status) and + loud-fallback exit codes. ## Modes @@ -137,7 +141,7 @@ read-only intent — whose prompt is the exact contents of and returns the findings list + summary table. Write its output to `/review_a.md`. -### 4. Reviewer 2 — codex-sol (blind) — DUAL mode only +### 4. Reviewer 2 — codex gpt-6.1-sol (blind) — DUAL mode only **Single mode**: skip steps 4-5 and take the **Single-reviewer mode (deliberate)** path below — the review body is reviewer 1's output with the @@ -167,7 +171,7 @@ python3 ".claude/skills/plan-review/codex_review.py" \ - **Exit 0** → codex review is in `/review_b.md`; go to step 5. - **Non-zero (2 = codex absent, 3 = timeout/error)** → codex is unavailable; take the **Loud fallback** below and skip steps 5-6. A hung codex cannot - wedge the gate — `codex_review.py` caps at 2400s and exits 3. + wedge the gate — `codex_review.py` caps at 3600s and exits 3. ### 5. Merge + verify — Claude @ Opus @@ -200,7 +204,7 @@ verbatim, then the contents of `/review_a.md`: ``` > ⚠ **codex unavailable — SINGLE-Claude fallback.** This is the un-validated -> weak arm (Campaign 1: caught 1/9 hard plan defects vs 7/9 for the dual +> weak arm (Campaign 1, measured with gpt-5.6-sol: caught 1/9 hard plan defects vs 7/9 for the dual > engine). Run `codex login` and re-review for full coverage. ``` @@ -216,7 +220,7 @@ failure), then the contents of `/review_a.md`: ``` > Single-Opus review (deliberate one-reviewer choice, not the dual engine). -> The dual engine caught 7/9 hard plan defects vs 1/9 single in Campaign 1 — +> The dual engine caught 7/9 hard plan defects vs 1/9 single in Campaign 1 (measured with gpt-5.6-sol) — > re-review in dual mode if this plan turns out higher-risk than expected. ``` diff --git a/.claude/skills/plan-review/codex_review.py b/.claude/skills/plan-review/codex_review.py index 2a0b11cd..fc71d5c1 100644 --- a/.claude/skills/plan-review/codex_review.py +++ b/.claude/skills/plan-review/codex_review.py @@ -2,7 +2,7 @@ Called by SKILL.md with an already-rendered reviewer prompt. Detects codex, invokes `openai_review.call_codex` with the campaign-graded model/effort -pins (gpt-5.6-sol @ xhigh, arm C as-run) under a read-only sandbox, writes the codex +pins (gpt-6.1-sol @ xhigh; the production code-review pin) under a read-only sandbox, writes the codex review. Before invoking, it runs openai_review's own sensitive-file scan + stderr notice (the codex `--cd` read surface is the whole repo — the same notice the `/ai-review-local` codex path prints). Exit codes let SKILL.md fall @@ -21,15 +21,24 @@ import shutil import sys -# Campaign-graded codex pins (Campaign 1: arm C, as-run — the campaign was -# exploratory / non-gating). Do not change without re-validation — -# tests/test_plan_review_skill.py asserts these. -CODEX_MODEL = "gpt-5.6-sol" +# Codex pins follow the PRODUCTION code-review model (lockstep-tested against the +# CI workflow and openai_review.DEFAULT_MODEL). gpt-5.6-sol was the engine +# Campaign 1 graded (arm C, as-run; exploratory / non-gating); gpt-6.1-sol was +# carried over by the 2026-10 code-review campaign 2 +# (tools/reviewer-eval/campaigns/2026-10-gpt61sol-VERDICT.md), NOT re-run through +# plan-review-eval. To re-validate: first re-pin +# tools/plan-review-eval/config/configs.json cli_version (the composite +# " | codex-cli " string on all five arms) to the +# live Claude Code + codex CLI pair, add a gpt-6.1-sol arm, run that harness +# against the sol baseline (TODO.md row). tests/test_plan_review_skill.py asserts +# these pins. +CODEX_MODEL = "gpt-6.1-sol" CODEX_EFFORT = "xhigh" -# The timeout is NOT the campaign value (the campaign ran unattended at -# CODEX_TIMEOUT_S=3600). It is an interactive-gate ceiling: high enough to clear -# real plan-review runtimes with room to spare, low enough to bound the -# interactive wait. A timeout is treated as codex being unavailable (exit 3 → +# The timeout equals the code-review harness's CALL_TIMEOUT_S +# (tools/reviewer-eval/adapters/codex_reviewer.py; tests/test_plan_review_skill.py +# pins the equality). It is still an interactive-gate ceiling, not a campaign +# setting: high enough to clear real plan-review runtimes with room to spare, +# low enough to bound the interactive wait. A timeout is treated as codex being unavailable (exit 3 → # LOUD single-Claude fallback), so err generous. # # Raised 1200 → 2400 on 2026-08-08. The old value was calibrated to the campaign's @@ -45,7 +54,13 @@ # Changing this does NOT require engine re-validation: the "do not change without # re-validation" note above binds CODEX_MODEL and CODEX_EFFORT (what Campaign 1 # graded). The ceiling is an operational knob and is graded by nothing. -CODEX_TIMEOUT_S = 2400.0 +# +# Raised 2400 -> 3600 on 2026-10-17 with the gpt-6.1-sol swap, to equal the +# code-review harness ceiling (the 2026-10 campaigns measured 6.1-sol at +# ~180 s mean / 277 s p95 per code review, faster than 5.6-sol, but plan review +# is the slower surface and the two pins are kept in lockstep); still a hard +# cap, exit 3 = loud single-Claude fallback. +CODEX_TIMEOUT_S = 3600.0 def _load_openai_review(repo_root: str): diff --git a/.github/workflows/ai_pr_review.yml b/.github/workflows/ai_pr_review.yml index 836d224c..fa64a94b 100644 --- a/.github/workflows/ai_pr_review.yml +++ b/.github/workflows/ai_pr_review.yml @@ -530,7 +530,7 @@ jobs: # configs.json pin + a harness smoke (the lockstep test enforces equality). codex-version: 0.162.1 # Recommended by OpenAI for review quality/consistency: - model: gpt-5.6-sol + model: gpt-6.1-sol effort: xhigh - name: Post PR comment (new on every event except initial open) diff --git a/CLAUDE.md b/CLAUDE.md index 3dc0410c..df7fef7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -250,9 +250,11 @@ Before calling `ExitPlanMode`, ALWAYS offer all three review options via `AskUserQuestion`, with the `(Recommended)` tag chosen ADAPTIVELY from the plan's complexity and risk (not a fixed default) — set exactly one: - **Dual review** — the campaign-selected engine (two blind reviewers, Claude @ - Opus + codex `gpt-5.6-sol`, then merge/verify; Campaign 1 was exploratory / - NON-GATING but showed dual catching 7/9 must-catch plan defects vs 1/9 - single). Recommend for SUBSTANTIVE / high-risk plans: + Opus + codex `gpt-6.1-sol`, then merge/verify; Campaign 1, run with gpt-5.6-sol, + was exploratory / NON-GATING but showed dual catching 7/9 must-catch plan + defects vs 1/9 single; gpt-6.1-sol is carried over from the 2026-10 code-review + campaign and not re-validated by plan-review-eval). Recommend for SUBSTANTIVE / + high-risk plans: estimator, methodology, variance/inference, or `docs/methodology/REGISTRY.md` changes; multi-file, architectural, or public-API changes. - **Single review** (Opus only, no codex) — recommend for LOCALIZED, mechanical @@ -264,7 +266,7 @@ plan's complexity and risk (not a fixed default) — set exactly one: (`.claude/skills/plan-review/`) in the chosen mode. Its Review phase snapshots the plan via the tested helper (confirm the printed `plan_path` — the ingress file is shared per-worktree), runs the reviewers (dual = Opus reviewer + codex -`gpt-5.6-sol` + merge/verify; single = the Opus reviewer alone), writes the +`gpt-6.1-sol` + merge/verify; single = the Opus reviewer alone), writes the review to the helper-derived `review_path`, and runs `plan_snapshot.py persist` (which re-verifies the live plan against the recorded snapshot — exit 3 = plan changed mid-review, not persisted, re-review — stamps `plan:`/`plan_sha256:` diff --git a/DEFERRED.md b/DEFERRED.md index b39b7b4f..98ac11f5 100644 --- a/DEFERRED.md +++ b/DEFERRED.md @@ -123,7 +123,6 @@ For survey-specific limitations (`NotImplementedError` paths), see the | `HeterogeneousAdoptionDiD` `covariates=` (Theorem 6 multivariate-covariate extension) not implemented — `fit(covariates=...)` raises `NotImplementedError` via the shipped future-work trap (locked by the `test_had.py` / `test_methodology_had.py` L73 tests); the deferred work is the Theorem 6 extension itself. | `had.py` | Phase 2a | Low | | MultiPeriodDiD wild bootstrap not supported (falls back to analytical, n_bootstrap-independent) — user-facing edge-case limitation; the 4.0 removal replaces the fallback with a raise (v4-design §4.1), and the merged TWFE event-study mode already RAISES since 3.9 (Phase 3(a)) — this row now governs only the deprecated class itself. | `estimators.py` (MultiPeriodDiD.fit wild-fallback block) | — | Low | | `predict()` raises `NotImplementedError` — rarely needed; user-facing limitation. | `estimators.py:890-911` | — | Low | -| `gpt-6-sol` cost-fallback probe for the Codex reviewer: a ChatGPT-account `codex login` rejects `gpt-6-sol` (400, "not supported when using Codex with a ChatGPT account"), so the local harness cannot run a cheaper-tier arm; needs an API-key codex login (metered) before an arm can be added. | `tools/reviewer-eval/config/configs.json` | #849 | Low | ## Version-gated (v4) diff --git a/TODO.md b/TODO.md index e04d90bc..de8553b1 100644 --- a/TODO.md +++ b/TODO.md @@ -86,6 +86,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| +| Re-validate the plan-review dual engine on `gpt-6.1-sol` with `tools/plan-review-eval`: re-pin every arm's composite `cli_version` to the live Claude Code + codex CLI pair, add a gpt-6.1-sol arm, run the harness against the gpt-5.6-sol baseline (the 2026-10 swap carried the model over from the code-review campaign 2 verdict without a plan-review campaign). | `tools/plan-review-eval/config/configs.json` | reviewer-eval gpt61sol PR-2 | Heavy | Low | | TWFE-weights tutorial section: `attgt_weights` (all three `type=` values side by side) + `decompose_twfe_weights` + `plot_twfe_weights` on `mpdta`, either as a section of `02_staggered_did.ipynb` or a new notebook registered in `docs/tutorials/index.rst`; follow-on PR by decision, the feature ships without it | `docs/tutorials/02_staggered_did.ipynb` | #812 | Mid | Medium | | Geo-experiments tutorial: the SDiD fits on the `generate_factor_data(...)` panel (conversions ~1,500, sigma-hat ~310) hit the Frank-Wolfe iteration cap - the placebo fit warns once and the bootstrap cross-check warns on 100/100 draws - yet the narrative presents the bootstrap SE as a paper-faithful robustness check. Pre-existing on `main` under BOTH backends (surfaced when the notebook outputs were refreshed for the shape-only fit diagnostic); the committed April outputs predate the aggregated non-convergence warning. Either rescale / re-tune the DGP (or raise `max_iter` / loosen `min_decrease`) so the refits converge and re-execute, or label the bootstrap result unreliable and drop it from the "Validity evidence" claim. | `docs/tutorials/18_geo_experiments.ipynb` | local codex review of the SDiD shape-only fit-RMSE PR | Mid | Low | | Replicate Chang (2020) §4.2.1 ML-design RCS DGP — requires a penalized (Lasso-class) propensity learner or a maintainer fixture via the duck-typed `propensity_learner=` object route; native logit+linear verified pure noise at (N=500, p=100) across ~20 review seeds (att ~ −23..+16, SE ~4-10, EPV ~1.4-1.6 vs threshold 10, fitted out-of-fold clipping ~5-29%, mean ~14%); at (N=200, p=100) `outcome_learner="linear"` fails closed on control-fold rank deficiency even under an oracle propensity (use ridge/sieve there); §4.2.2 is replicated | `tests/test_methodology_dml_did.py`, `diff_diff/_learners.py` | DML PR-B2 | Mid | Low | @@ -101,7 +102,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Codex reviewer isolation (repo-wide): `codex --sandbox read-only` blocks writes but does NOT confine READS to the worktree/repo (verified: a probe read `/etc/hosts`), and prompt guards are not a security boundary. Affects `/ai-review-local` (which already documents and accepts this surface) and plan-review-eval dual arms equally. Evaluate OS-level isolation for codex invocations (container / `sandbox-exec` profile / dedicated low-privilege account exposing only the worktree). | `.claude/scripts/openai_review.py`, `tools/plan-review-eval/` | plan-review-eval local review R6 | Heavy | Medium | | plan-review-eval dual arms: terminate the surviving peer subprocess when the first reviewer of a dual pair fails (today an early Claude failure still waits out the codex peer — up to its 3600s ceiling — before the run becomes INFRA_ERROR; needs process handles threaded through `_call_claude`/`call_codex` or a shared cancellation event). | `tools/plan-review-eval/plan_adapters/plan_reviewer.py` | plan-review-eval local review R3 | Mid | Low | | Reimplement the retired `/review-plan` `--updated` (delta re-review, `## Delta Assessment` section) and `--pr ` (PR-comment feedback-coverage) modes in the plan-review skill — descoped from the initial skill ship (the advertised-but-unimplemented flags were removed). `--updated` needs prior-review load + a delta diff; `--pr` needs a `gh` comment fetch + a new reviewer-prompt token. | `.claude/skills/plan-review/SKILL.md` | plan-review skill local review | Mid | Low | -| plan-review skill dual-mode peer cancellation: when reviewer 1 (Opus subagent) fails, terminate the concurrently-running codex `codex_review.py` subprocess instead of waiting out its 2400s ceiling (and vice-versa). Interactive analogue of the eval-harness peer-cancellation row above; needs the codex process handle surfaced from `codex_review.py` so the skill can signal it. | `.claude/skills/plan-review/SKILL.md`, `.claude/skills/plan-review/codex_review.py` | plan-review skill local review R2 | Mid | Low | +| plan-review skill dual-mode peer cancellation: when reviewer 1 (Opus subagent) fails, terminate the concurrently-running codex `codex_review.py` subprocess instead of waiting out its 3600s ceiling (and vice-versa). Interactive analogue of the eval-harness peer-cancellation row above; needs the codex process handle surfaced from `codex_review.py` so the skill can signal it. | `.claude/skills/plan-review/SKILL.md`, `.claude/skills/plan-review/codex_review.py` | plan-review skill local review R2 | Mid | Low | | Clean-negative precision re-run: Campaign 1's `s3_negative` plans were base_sha-contaminated (real defects at the pinned base), so hallucination rate cleared (dual 3.4%) but trivia-flooding on a *genuinely* clean plan is unmeasured. Small future run (~30 s3-style reviews on 3-5 constructed-clean plans, not the full 120-matrix). | `tools/plan-review-eval/` | campaign-1 verdict | Mid | Low | | `trop-silent-drop` criteria regression: the rewritten `criteria.md` dropped a silent-failure catch the OLD criteria had (arm A caught it, B/C missed — campaign-1 A-vs-B contrast). Patch the criteria + re-validate (re-opens the criteria identity, so gated behind a re-validation run). | `.claude/skills/plan-review/criteria.md`, `tools/plan-review-eval/` | campaign-1 verdict | Mid | Medium | | The Stata `jwdid` parity arms cover BALANCED panels only. `jwdid_alltreated` pins the all-eventually-treated cell set, `N` and SE ratio on the `mpdta` subset (191 units, 955 rows, 764 estimated), but every arm is a balanced frame, so REGISTRY's parity claim is scoped to that case. The comparison-support predicate's UNBALANCED behavior -- where `G_max` may be unobserved at later periods so the closed form `t < G_max - anticipation` does not hold and support is set by whichever cohorts are actually observed -- is verified against the predicate directly, not against Stata. Add an unbalanced `jwdid` arm to close that gap. | `benchmarks/stata/generate_etwfe_cs_golden.do`, `tests/test_etwfe_cs_stata_parity.py` | #729-followup | Mid | Low | diff --git a/changelog.d/20261017-reviewer-eval-gpt61sol-prereg.md b/changelog.d/20261017-reviewer-eval-gpt61sol-prereg.md new file mode 100644 index 00000000..df680ccf --- /dev/null +++ b/changelog.d/20261017-reviewer-eval-gpt61sol-prereg.md @@ -0,0 +1,23 @@ +### Internal +- **Codex reviewer swap gpt-5.6-sol -> gpt-6.1-sol (reviewer-eval 2026-10 campaigns, PR-2)**: + two blinded, pre-registered 44-review campaigns on the frozen `tools/reviewer-eval` + harness. Campaign 1 (B = gpt-5.6-sol vs E = gpt-6-astra, both @ xhigh, k=2): mechanical + DEFINITIVE **NO-GO** on one regression (`s4-cs-unbalanced-nan-outcome`), accepted by the + user; no astra swap (`campaigns/2026-10-gpt6-VERDICT.md`). Campaign 2 (B vs F = + gpt-6.1-sol @ xhigh, k=2, fresh control sample, same rule re-issued for F): + mechanical DEFINITIVE **PARITY** with zero rule-defined regressions, zero downgrades, + 0 vs 0 false positives, two favourable catch-status changes against one + UNSTABLE-to-MISSED judgment flag, ~15% lower latency and half the input price; the + user accepted PARITY as GO + (`campaigns/2026-10-gpt61sol-VERDICT.md`). Both verdicts ship with their redacted + grading records (`campaigns/2026-10-gpt6*-grading/`). Production pins move in lockstep + to `gpt-6.1-sol`: the CI workflow `model`, `openai_review.DEFAULT_MODEL` and the + plan-review `CODEX_MODEL` (carried over from the code-review campaign, not re-run + through plan-review-eval; TODO.md row), with the plan-review `CODEX_TIMEOUT_S` raised + 2400 -> 3600 to equal the harness ceiling. Harness: arm `F` registered with the contrast + `(B, F)`, campaign 1's rule archived as `campaigns/2026-10-gpt6-DECISION_RULE.md`, + `run --configs` defaults to the last registered shape, `PRICING` gains `gpt-6.1-sol`, + arm F takes `role=control` (B retained as previous production), the candidate-exposure + gate exempts the control arm so bare `smoke` keeps working after a swap, and the + DEFERRED `gpt-6-sol` cost-fallback row is retired (codex-cli 0.162.1's ChatGPT-account + login accepts the gpt-6 Sol tier; campaign 2 was that evaluation). diff --git a/tests/test_evals_runtime.py b/tests/test_evals_runtime.py index 66007894..8bb6718c 100644 --- a/tests/test_evals_runtime.py +++ b/tests/test_evals_runtime.py @@ -2303,16 +2303,28 @@ def test_live_configs_resolve_control_and_candidate(): assert b.model != e.model and b.effort == e.effort == "xhigh" assert b.sandbox == e.sandbox and b.action_version == e.action_version assert b.cli_version == e.cli_version == "codex-cli 0.162.1" - assert run_eval._control_id() == "B" + assert run_eval._control_id() == "F" # production since the 2026-10 campaign-2 swap assert run_eval._treatment_fields() == ("model", "effort") - # Bare `run` is the registered campaign shape end to end: the COMPLETE default - # argument combination passes the candidate-exposure gate (configs B,E at k=2, - # no --k-per, no --strata), so `python run_eval.py run` is not configured to refuse. + # Campaign 2 (2026-10-17): arm F = gpt-6.1-sol differs from B in model only, + # same effort/sandbox/action/CLI pins; (B, F) is a registered contrast too. + cfgs_f = run_eval._resolve_configs("B,F") + assert cfgs_f is not None and [c.id for c in cfgs_f] == ["B", "F"] + b2, f = cfgs_f + assert f.model == "gpt-6.1-sol" and f.model != b2.model != e.model + assert f.effort == "xhigh" and f.sandbox == b2.sandbox + assert f.action_version == b2.action_version and f.cli_version == b2.cli_version + import reviewer_verdict + + assert reviewer_verdict.REGISTERED_CONTRASTS == (("B", "E"), ("B", "F")) + # Bare `run` is the ACTIVE registered campaign shape end to end: the COMPLETE + # default argument combination passes the candidate-exposure gate (configs B,F + # at k=2, no --k-per, no --strata), so `python run_eval.py run` is not + # configured to refuse. ns = run_eval._build_parser().parse_args(["run"]) - assert ns.configs == "B,E" and ns.k == 2 and ns.k_per == "" and ns.strata is None + assert ns.configs == "B,F" and ns.k == 2 and ns.k_per == "" and ns.strata is None assert ( run_eval._candidate_exposure_violations( - cfgs, ns.k, run_eval._parse_k_per(ns.k_per, cfgs), ns.strata, "run" + cfgs_f, ns.k, run_eval._parse_k_per(ns.k_per, cfgs_f), ns.strata, "run" ) == [] ) @@ -2882,6 +2894,10 @@ def _never(repo_root): dict(configs="B,E", strata=["s1_synthetic"], subdir="x", k=2, max_parallel=1), dict(configs="B,E", strata=None, subdir="x", k=2, k_per="E=1", max_parallel=1), dict(configs="A,B,E", strata=None, subdir="x", k=2, max_parallel=1), + # E with campaign 2's F (F is production/control after the swap and exempt + # from the gate; E is still a confined candidate) + dict(configs="E,F", strata=None, subdir="x", k=2, max_parallel=1), + dict(configs="B,E,F", strata=None, subdir="x", k=2, max_parallel=1), ], ) def test_run_refuses_candidate_exposure_outside_registered_campaign( @@ -2899,10 +2915,31 @@ def test_smoke_refuses_candidate_arm(tmp_path, monkeypatch, capsys): assert "may not be smoked" in capsys.readouterr().err +def test_control_arm_is_exempt_from_the_exposure_gate(): + """After the 2026-10 swap F is production (role=control): it is measured by + definition, so bare `smoke` (which resolves the control) and any `run` shape + that includes F pass the gate; E stays confined to its registered shape.""" + run_eval = _run_eval() + assert run_eval._control_id() == "F" + f_only = run_eval._resolve_configs("F") + assert run_eval._candidate_exposure_violations(f_only, 1, {}, ["s1_synthetic"], "smoke") == [] + assert ( + run_eval._candidate_exposure_violations( + run_eval._resolve_configs("B,F"), 1, {}, None, "run" + ) + == [] + ) + bad = run_eval._candidate_exposure_violations( + run_eval._resolve_configs("E,F"), 2, {}, None, "run" + ) + assert bad and "candidate E runs only as --configs B,E" in bad[0] + + def test_registered_campaign_shape_passes_the_gate(): run_eval = _run_eval() - cfgs = run_eval._resolve_configs("B,E") - assert run_eval._candidate_exposure_violations(cfgs, 2, {}, None, "run") == [] + for contrast in ("B,E", "B,F"): + cfgs = run_eval._resolve_configs(contrast) + assert run_eval._candidate_exposure_violations(cfgs, 2, {}, None, "run") == [] assert ( run_eval._candidate_exposure_violations( run_eval._resolve_configs("A,B"), 1, {}, ["s1_synthetic"], "run" diff --git a/tests/test_openai_review.py b/tests/test_openai_review.py index 4c915805..9e17dee7 100644 --- a/tests/test_openai_review.py +++ b/tests/test_openai_review.py @@ -1654,7 +1654,7 @@ def test_rejects_traversal(self, review_mod, repo_root): class TestIsReasoningModel: def test_gpt56_family_is_reasoning(self, review_mod): - """The production default (gpt-5.6-sol) and its siblings must classify as + """The gpt-5.6 family (previous production gpt-5.6-sol and siblings) must classify as reasoning models - this drives the 900s timeout and token limits.""" for m in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): assert review_mod._is_reasoning_model(m) is True @@ -1673,6 +1673,10 @@ def test_gpt6_pricing_entries_present(self, review_mod): """The gpt-6 rows (2026-10-15) and estimate_cost resolving the candidate.""" assert review_mod.PRICING["gpt-6-astra"] == (10.00, 50.00) assert review_mod.PRICING["gpt-6-sol"] == (2.00, 10.00) + assert review_mod.PRICING["gpt-6.1-sol"] == ( + 2.00, + 10.00, + ) # production since 2026-10-17 (campaign 2) assert review_mod.PRICING["gpt-6-luna"] == (0.10, 0.50) assert review_mod.estimate_cost(1_000_000, 0, "gpt-6-astra").startswith("$10.00") @@ -1680,10 +1684,11 @@ def test_gpt6_family_is_reasoning(self, review_mod): assert review_mod._is_reasoning_model("gpt-6-astra") is True assert review_mod._resolve_timeout(None, "gpt-6-astra") == review_mod.REASONING_TIMEOUT - def test_default_model_is_gpt56_sol(self, review_mod): + def test_default_model_is_production_pin(self, review_mod): """DEFAULT_MODEL is the production reviewer pin (kept in lockstep with the - CI workflow's model: input; the eval harness validated this pairing).""" - assert review_mod.DEFAULT_MODEL == "gpt-5.6-sol" + CI workflow's model: input; the 2026-10 reviewer-eval campaign 2 selected + gpt-6.1-sol, PARITY accepted as GO).""" + assert review_mod.DEFAULT_MODEL == "gpt-6.1-sol" def test_o3_is_reasoning(self, review_mod): assert review_mod._is_reasoning_model("o3") is True diff --git a/tests/test_plan_review_skill.py b/tests/test_plan_review_skill.py index 8c3b4329..265687ea 100644 --- a/tests/test_plan_review_skill.py +++ b/tests/test_plan_review_skill.py @@ -168,9 +168,16 @@ def test_reviewer_invocations_are_pinned(): byte-match silently. The codex pins live at the real `call_codex` site (`codex_review.py`); the Claude subagent model pin lives in SKILL.md.""" codex = (_SKILL / "codex_review.py").read_text() - assert 'CODEX_MODEL = "gpt-5.6-sol"' in codex + assert 'CODEX_MODEL = "gpt-6.1-sol"' in codex assert 'CODEX_EFFORT = "xhigh"' in codex - assert "CODEX_TIMEOUT_S = 2400" in codex + # The ceiling EQUALS the code-review harness's CALL_TIMEOUT_S (read with a + # regex: codex_reviewer.py imports eval_core/adapters, which this file's + # sys.path does not provide), so a probe-driven raise cannot leave the two + # out of step. + harness = (_REPO / "tools" / "reviewer-eval" / "adapters" / "codex_reviewer.py").read_text() + call_cap = re.search(r"^\s*CALL_TIMEOUT_S\s*=\s*([\d.]+)", harness, re.M) + assert call_cap, "CALL_TIMEOUT_S not found in codex_reviewer.py" + assert f"CODEX_TIMEOUT_S = {float(call_cap.group(1)):.1f}" in codex # and codex_review.py actually passes them to call_codex assert "model=CODEX_MODEL" in codex and "effort=CODEX_EFFORT" in codex assert "timeout_s=CODEX_TIMEOUT_S" in codex @@ -348,9 +355,9 @@ def test_codex_review_forwards_pins_and_prints_sensitive_notice(tmp_path, monkey assert len(fake.codex_calls) == 1 call = fake.codex_calls[0] assert call["prompt"] == "RENDERED PROMPT" - assert call["model"] == cr.CODEX_MODEL == "gpt-5.6-sol" + assert call["model"] == cr.CODEX_MODEL == "gpt-6.1-sol" assert call["effort"] == cr.CODEX_EFFORT == "xhigh" - assert call["timeout_s"] == cr.CODEX_TIMEOUT_S == 2400.0 + assert call["timeout_s"] == cr.CODEX_TIMEOUT_S == 3600.0 def test_codex_review_absent_returns_2_and_writes_nothing(tmp_path, monkeypatch): diff --git a/tests/test_reviewer_verdict.py b/tests/test_reviewer_verdict.py index 84a4b442..90ff097f 100644 --- a/tests/test_reviewer_verdict.py +++ b/tests/test_reviewer_verdict.py @@ -573,6 +573,8 @@ def test_verdict_cli_happy_path_and_audit(tmp_path, monkeypatch, capsys): "control, candidate, msg", [ ("E", "B", "registered contrast"), + ("F", "B", "registered contrast"), # campaign-2 contrast inverted + ("E", "F", "registered contrast"), # two candidates, no control ("B", "B", "must differ"), ("A", "B", "registered contrast"), ], diff --git a/tools/reviewer-eval/DECISION_RULE.md b/tools/reviewer-eval/DECISION_RULE.md index 1a9c8ec0..f5b81817 100644 --- a/tools/reviewer-eval/DECISION_RULE.md +++ b/tools/reviewer-eval/DECISION_RULE.md @@ -1,12 +1,26 @@ -# Pre-Registered Decision Rule: gpt-5.6-sol -> gpt-6-astra Reviewer Evaluation +# Pre-Registered Decision Rule: gpt-5.6-sol -> gpt-6.1-sol Reviewer Evaluation (campaign 2) -Date: 2026-10-16. Pre-registered in PR #849 (frozen at that PR's labeled, CI-green HEAD). The +Date: 2026-10-17. Pre-registered in PR #PR1B (frozen at that PR's labeled, CI-green HEAD). The candidate arm is NEVER run on the corpus before the campaign: its first corpus run is the campaign itself, so pre-registration holds by construction. The verdict is computed MECHANICALLY by `run_eval.py verdict` (`reviewer_verdict.py`) from the reconciled grading table; this file may not be edited after the campaign starts (the campaign's frozen PR-1 HEAD pins it). +**Relation to campaign 1 (gpt-6-astra, `campaigns/2026-10-gpt6-DECISION_RULE.md`):** +campaign 1 is complete (44/44 runs on `225a4afb`, local tag `campaign-gpt6`, +mechanical DEFINITIVE NO-GO on one regression, accepted by the user on +2026-10-17; its verdict file lands in PR-2 together with this campaign's). +This rule is the SAME rule with arm F = gpt-6.1-sol in the candidate seat: +the gates, definitions, precedence, grading protocol, audit, ledger and stop +conditions below are unchanged in substance, only the candidate letter and +model differ. F is judged against a FRESH control sample (B is re-run in the +new subdir; the campaign-1 B artifacts are never mixed in). Astra's numbers +are shown next to F's in the verdict file DESCRIPTIVELY only (same corpus, +same control model, different control sample); no gate compares E and F. +`REGISTERED_CONTRASTS` keeps `(B, E)` so the campaign-1 ledger records stay +interpretable; `(B, F)` is the contrast this campaign registers. + This file is NOT grader-facing. Graders receive `GRADING_RUBRIC.md` only. ## The experiment @@ -14,7 +28,7 @@ This file is NOT grader-facing. Graders receive `GRADING_RUBRIC.md` only. | Arm | Model | Effort | Repeats | Role | |-----|-------|--------|---------|------| | B | gpt-5.6-sol | xhigh | k=2 | control (current production) | -| E | gpt-6-astra | xhigh | k=2 | candidate | +| F | gpt-6.1-sol | xhigh | k=2 | candidate | No probe arms. CLI pinned to codex-cli 0.162.1 on every arm (the runner refuses a live `codex --version` mismatch); CI pins the same version. Every harness codex @@ -42,22 +56,22 @@ per-cell field (`leak: none | confirmed` in the reconciled table; graders write `known_topic_id` exemption branch is NOT ported. The bundle's "do NOT flag" wording for known topics stays (reviewer advice, not grader instruction). -## Gates (E vs B; symmetric on catches) +## Gates (F vs B; symmetric on catches) 1. **Regression NO-GO** = ANY ground-truth bug (`must_catch` true OR false) - that B reliably catches and E has status MISSED (the status definitions + that B reliably catches and F has status MISSED (the status definitions under "Voided repeats" are the executable wording). `must_catch` stays in - the case files as a reporting label only. B UNSTABLE / E MISSED is a + the case files as a reporting label only. B UNSTABLE / F MISSED is a judgment FLAG reported next to the mechanical outcome; it never changes the label by itself. -2. **FP NO-GO** = E's S3 FP total > B's. Small-sample note (2 S3 cases x k=2, +2. **FP NO-GO** = F's S3 FP total > B's. Small-sample note (2 S3 cases x k=2, B expected at 0): the verdict prints every FP's text next to this gate so the user can rule a control invalid, as happened in 2026-07; the mechanical verdict stands unless the user overrides it in writing in the verdict. 3. **GO** = zero regressions, the FP gate passes, FEWER THAN 2 - reliable-to-unstable downgrades (bugs B catches 2/2 and E catches 1/2), and - >= 1 strict improvement: E reliably catches ANY bug on which B has status - MISSED (the two s4-wcr bugs DO count), OR E has strictly fewer S3 FPs than B + reliable-to-unstable downgrades (bugs B catches 2/2 and F catches 1/2), and + >= 1 strict improvement: F reliably catches ANY bug on which B has status + MISSED (the two s4-wcr bugs DO count), OR F has strictly fewer S3 FPs than B (only reachable if B posts >= 1 FP). 4. **PARITY** = zero regressions and the FP gate passes, and EITHER no strict improvement OR >= 2 reliable-to-unstable downgrades -> the user decides, @@ -78,8 +92,8 @@ per-bug status is computed from its remaining (non-void, OK) repeats: - MISSED = caught in no remaining repeat (>= 1 remaining); - UNGRADEABLE = no remaining repeat. -Gates use these statuses (regression = B RELIABLE and E MISSED; downgrade = B -RELIABLE and E UNSTABLE; improvement = E RELIABLE and B MISSED; UNGRADEABLE +Gates use these statuses (regression = B RELIABLE and F MISSED; downgrade = B +RELIABLE and F UNSTABLE; improvement = F RELIABLE and B MISSED; UNGRADEABLE never satisfies a gate, it only contests). A void or ungradeable cell ALWAYS contests: its case is reported "contested by leak" on whichever gates the case feeds (gates 1/3/downgrades for a bug case, gate 2 for an S3 case). Audit hits @@ -111,7 +125,7 @@ campaigns are handled by the stop conditions below. 2x-p95 flag, the input-cost estimate from `openai_review.PRICING`); the vendor-lifecycle read is prose the verdict file carries. -- Per-arm latency (flag if E p95 > 2x B). +- Per-arm latency (flag if F p95 > 2x B). - Hallucination / unverified external-tool-claim count per arm (counted once per (case, arm, repeat)). - Leakage count per arm (distinct confirmed cells) and unconfirmed-flag count. diff --git a/tools/reviewer-eval/README.md b/tools/reviewer-eval/README.md index 991b4775..17639afd 100644 --- a/tools/reviewer-eval/README.md +++ b/tools/reviewer-eval/README.md @@ -7,14 +7,20 @@ raw review, and emits a side-by-side bundle you (or an LLM) read into a caught / missed / false-positive table — optionally **blinded** (arm identities stripped) so graders can't favor a model. -**Status:** harness supports N-arm matrices (current config: a 5-arm file; the -2026-10 campaign is `--configs B,E --k 2`, control gpt-5.6-sol vs candidate -gpt-6-astra, outcome: pending), per-arm repeat counts, blinded grading sealed to -the run artifacts, a post-unblinding session-log audit and a mechanical verdict -CLI. Local-only — not wired to CI. The go/no-go rule is pre-registered in -`DECISION_RULE.md`; the grader-facing rubric is `GRADING_RUBRIC.md` (graders -never see the rule or any campaign history). ChatGPT-account `codex login` -rejects `gpt-6-sol`, so a cheaper-tier probe arm is parked (DEFERRED.md). +**Status:** harness supports N-arm matrices (current config: a 6-arm file). +2026-10 campaign 1 was `--configs B,E --k 2`, control gpt-5.6-sol vs candidate +gpt-6-astra: mechanical NO-GO (one regression; rule archived as +`campaigns/2026-10-gpt6-DECISION_RULE.md`). Campaign 2 is `--configs B,F --k 2`, +the same control vs candidate gpt-6.1-sol: mechanical PARITY, accepted as GO by +the user, so gpt-6.1-sol @ xhigh is production (CI + local + plan-review) and arm +F carries role=control (`campaigns/2026-10-gpt61sol-VERDICT.md`). Per-arm repeat +counts, blinded grading sealed to the run artifacts, a post-unblinding +session-log audit and a mechanical verdict CLI. Local-only — not wired to CI. +The go/no-go rule is pre-registered in `DECISION_RULE.md`; the grader-facing +rubric is `GRADING_RUBRIC.md` (graders never see the rule or any campaign +history). On codex-cli 0.162.1 the ChatGPT-account `codex login` accepts the +gpt-6 Sol tier (the 0.153.4 rejection that parked a cheaper-tier arm no longer +holds), which is what makes campaign 2 possible. ## Why this exists @@ -49,26 +55,29 @@ was its first consumer) — generic glue with no diff-diff coupling. python tools/reviewer-eval/run_eval.py verify-corpus # 2. Smoke test (1 case, first real codex call) for the CONTROL only. Bare -# `smoke` exercises the role=control arm (derived from configs.json). The +# `smoke` exercises the role=control arm (derived from configs.json; after +# the 2026-10 swap that is F = gpt-6.1-sol, production). A registered # candidate's first corpus run is the campaign itself (pre-registration by -# construction); `smoke` REFUSES a registered candidate arm, and `run` -# accepts one only as the exact registered campaign shape below. A review -# runs 2-15 min. +# construction); `smoke` REFUSES a registered candidate arm other than the +# control, and `run` accepts one only as the exact registered campaign shape +# below; the next campaign registers its own (control, candidate) pair in +# its own PR-1. A review runs 2-15 min. python tools/reviewer-eval/run_eval.py smoke -# 3. Full matrix — k=2 repeats on control vs candidate (one run = one experiment) -python tools/reviewer-eval/run_eval.py run --subdir gpt6 --configs B,E --k 2 +# 3. Full matrix — k=2 repeats on control vs candidate (one run = one experiment; +# the 2026-10 campaign-2 shape, now complete - a new campaign uses its own subdir) +python tools/reviewer-eval/run_eval.py run --subdir gpt61sol --configs B,F --k 2 # 4. Emit the side-by-side bundle to grade (+ the identity-stripped, sealed one) -python tools/reviewer-eval/run_eval.py compare --subdir gpt6 --blinded +python tools/reviewer-eval/run_eval.py compare --subdir gpt61sol --blinded # 5. After unblinding: the session-log audit (outputs go OUTSIDE the repo or # under the gitignored runs/ — a root-level file dirties the tree and trips # the identity guard) and the mechanical verdict (PROVISIONAL without # --audit-hits, DEFINITIVE with it) -python tools/reviewer-eval/run_eval.py audit --subdir gpt6 -o /tmp//hits.json -python tools/reviewer-eval/run_eval.py verdict --subdir gpt6 --grades \ - --control B --candidate E [--audit-hits /tmp//hits.json] [--expect-head ] +python tools/reviewer-eval/run_eval.py audit --subdir gpt61sol -o /tmp//hits.json +python tools/reviewer-eval/run_eval.py verdict --subdir gpt61sol --grades \ + --control B --candidate F [--audit-hits /tmp//hits.json] [--expect-head ] ``` Run artifacts and the bundles land under `runs/` (gitignored). `--blinded` also diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-DECISION_RULE.md b/tools/reviewer-eval/campaigns/2026-10-gpt6-DECISION_RULE.md new file mode 100644 index 00000000..1a9c8ec0 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-DECISION_RULE.md @@ -0,0 +1,290 @@ +# Pre-Registered Decision Rule: gpt-5.6-sol -> gpt-6-astra Reviewer Evaluation + +Date: 2026-10-16. Pre-registered in PR #849 (frozen at that PR's labeled, CI-green HEAD). The +candidate arm is NEVER run on the corpus before the campaign: its first corpus +run is the campaign itself, so pre-registration holds by construction. The +verdict is computed MECHANICALLY by `run_eval.py verdict` (`reviewer_verdict.py`) +from the reconciled grading table; this file may not be edited after the +campaign starts (the campaign's frozen PR-1 HEAD pins it). + +This file is NOT grader-facing. Graders receive `GRADING_RUBRIC.md` only. + +## The experiment + +| Arm | Model | Effort | Repeats | Role | +|-----|-------|--------|---------|------| +| B | gpt-5.6-sol | xhigh | k=2 | control (current production) | +| E | gpt-6-astra | xhigh | k=2 | candidate | + +No probe arms. CLI pinned to codex-cli 0.162.1 on every arm (the runner refuses a +live `codex --version` mismatch); CI pins the same version. Every harness codex +call runs under an isolated `CODEX_HOME` that mirrors CI's codex-action home +(credential + `sandbox_mode = "read-only"` + `approval_policy = "never"`; no MCP +servers, hooks or memories; codex's DEFAULT features otherwise, web search +included). + +## Definitions + +By reference to `GRADING_RUBRIC.md`. Grades are exactly caught / partial / +missed (partial = missed for every gate). "Leak" is NOT a grade but a separate +per-cell field (`leak: none | confirmed` in the reconciled table; graders write +`none` or `flag`, the verifier rules); a `confirmed` cell is VOID (below). + +- **Reliably caught** (per arm, per bug) = caught in EVERY non-void repeat of + that arm AND no void repeat on that (case, arm). +- **FP total per arm** = sum of FPs over all its non-void repeats on S3 cases, + where an FP is a finding whose severity is above the case's + `allow_severities`, with NO topic-based exemption: on this corpus the only + known-FP topic (s3-wcr) has `would_be_severity_if_flagged` equal to the + allowed P3, so a P3 mention is already allowed and anything above P3 is the + 2026-07 "calibration failure" FP. The older "known-FP topics are never + counted" wording is retired as redundant; the sibling engine's + `known_topic_id` exemption branch is NOT ported. The bundle's "do NOT flag" + wording for known topics stays (reviewer advice, not grader instruction). + +## Gates (E vs B; symmetric on catches) + +1. **Regression NO-GO** = ANY ground-truth bug (`must_catch` true OR false) + that B reliably catches and E has status MISSED (the status definitions + under "Voided repeats" are the executable wording). `must_catch` stays in + the case files as a reporting label only. B UNSTABLE / E MISSED is a + judgment FLAG reported next to the mechanical outcome; it never changes the + label by itself. +2. **FP NO-GO** = E's S3 FP total > B's. Small-sample note (2 S3 cases x k=2, + B expected at 0): the verdict prints every FP's text next to this gate so + the user can rule a control invalid, as happened in 2026-07; the mechanical + verdict stands unless the user overrides it in writing in the verdict. +3. **GO** = zero regressions, the FP gate passes, FEWER THAN 2 + reliable-to-unstable downgrades (bugs B catches 2/2 and E catches 1/2), and + >= 1 strict improvement: E reliably catches ANY bug on which B has status + MISSED (the two s4-wcr bugs DO count), OR E has strictly fewer S3 FPs than B + (only reachable if B posts >= 1 FP). +4. **PARITY** = zero regressions and the FP gate passes, and EITHER no strict + improvement OR >= 2 reliable-to-unstable downgrades -> the user decides, + with these inputs reported: judgment flags, the downgrade and upgrade + lists, latency, the input-token cost estimate, hallucination and leakage + counts, the session-log audit, and the vendor lifecycle. + +### Voided repeats + +A (case, arm, repeat) with a verifier-CONFIRMED leak is VOID for every gate, +both its bug rows and its FP list, whichever arm it belongs to. An arm's +per-bug status is computed from its remaining (non-void, OK) repeats: + +- RELIABLE = caught in every remaining repeat AND no void repeat on that + (case, arm); +- UNSTABLE = caught in some remaining repeat, or caught in all remaining + repeats but at least one repeat void; +- MISSED = caught in no remaining repeat (>= 1 remaining); +- UNGRADEABLE = no remaining repeat. + +Gates use these statuses (regression = B RELIABLE and E MISSED; downgrade = B +RELIABLE and E UNSTABLE; improvement = E RELIABLE and B MISSED; UNGRADEABLE +never satisfies a gate, it only contests). A void or ungradeable cell ALWAYS +contests: its case is reported "contested by leak" on whichever gates the case +feeds (gates 1/3/downgrades for a bug case, gate 2 for an S3 case). Audit hits +never contest a gate (see "Hit list"). A contested gate goes to the user, never +decided mechanically: the user records in the verdict one of "accept +mechanical", "override to GO" or "override to NO-GO", with the reason, and +PR-2 swaps exactly when that final call is GO. A void cell is NOT re-run +(resume would reuse the cached artifact and the leak is a property of that +review); it stays void. + +### Precedence (the executable order) + +(0) validate the grades against the manifest-scoped runs; (1) UNDETERMINED if +any compared (case, arm) has fewer OK repeats than scheduled because of +INFRA_ERROR (void cells do NOT count here, they are graded cells); (2) apply +void/ungradeable marking from `leak: confirmed`; (3) NO-GO on any regression; +(4) NO-GO on FP excess; (5) PARITY if >= 2 downgrades; (6) GO if >= 1 strict +improvement; (7) PARITY. "Contested" is an annotation added at steps (3)-(6) +for the gates a void/ungradeable cell touched; the session-log hit list is +attached to every gate as an informational annotation and never contests. A +live campaign with any INFRA_ERROR has a failed manifest that `verdict` +refuses outright, so (1) is reachable only on synthetic tables; incomplete +campaigns are handled by the stop conditions below. + +## Informational, non-gating + +`run_eval.py verdict` computes the first three reads from the sealed artifacts +(`campaign_reads` in its JSON output: per-arm n/mean/p95/max latency, the +2x-p95 flag, the input-cost estimate from `openai_review.PRICING`); the +vendor-lifecycle read is prose the verdict file carries. + +- Per-arm latency (flag if E p95 > 2x B). +- Hallucination / unverified external-tool-claim count per arm (counted once + per (case, arm, repeat)). +- Leakage count per arm (distinct confirmed cells) and unconfirmed-flag count. +- Rough CI input cost per arm = `prompt_tokens_est` x official input price (a + +/-50% initial-prompt estimate; output/reasoning/tool tokens are not + recorded). +- **Session-log audit** (`run_eval.py audit`, `reviewer_audit.py`), run AFTER + unblinding: for every manifest run artifact it strips the run's own worktree + path from each `usage.session_commands` entry (the shell command lines + `call_codex` parsed from codex's stderr), then scans the remainder for the + ONE authoritative leak marker list (source of truth `reviewer_audit.LEAK_RULES`: + the literal `LEAK_MARKERS` plus the seven regexes, rendered verbatim in + `GRADING_RUBRIC.md`; this file does not restate it) and for the computed + per-campaign markers (the enclosing checkout, its branch name, the git + common dir and its parent checkout, every other worktree, every other + recorded ref name, the isolated codex home). It over-matches BY DESIGN + (routine orientation such as `git log -1`, `git show HEAD`, `HEAD~` are + markers) and is BEST-EFFORT: codex's default non-shell channels (web + search, sub-agents, browser) leave no shell `exec` line - the 2026-10 astra + latency probe used web search 26 times in one review, and the pre-freeze + rehearsal showed the CONTROL (sol) web-searching too (12 events across its + two S1 reviews, both cells `unaudited`, audit VACUOUS) - so any run that + printed a non-shell event header is `unaudited` rather than covered. User + decision (2026-10-16, after the rehearsal): the campaign ACCEPTS a vacuous + or near-vacuous audit as-is rather than disabling web search (CI parity, + decision 8) or capturing the queries; the grader `LEAK?` flag and the + verifier passes remain the only leak check, and the verdict reports the + `unaudited` labels and the vacuity verbatim. The rehearsal expectation is + therefore "coverage classes computed and reported" - not `unaudited == 0`. + - **Coverage classes** (the single definition, used here, by + `reviewer_audit.coverage_class` and by every test). Both `covered` and + `no_commands` REQUIRE `usage["codex_home_isolated"] == True` and + `usage["session_other_events"] == 0`. `covered` = `session_commands` is + non-empty, no entry ends in `[truncated]`, `session_commands_truncated` + is exactly `False`, and `exec_lines == start_lines == len(session_commands)`. + `no_commands` = the list is empty, `start_lines == 0`, `exec_lines == 0` + and `session_commands_truncated` is exactly `False` (codex ran no shell + command). `unaudited` = everything else (key absent, a truncated entry or + stream, a missing or non-boolean truncation flag, an empty list with a + non-zero count, `exec_lines > 0` with `start_lines == 0`, a non-zero + `other_events`, a non-isolated home). The audit is VACUOUS if NO + artifact is `covered` or `no_commands`; a vacuous audit is reported as + such (the hit list is empty for a technical reason). + - **Hit list** (user decision 6): the audit never decides whether a hit + matters and never marks a gate contested. The verdict prints, for EACH + gate, the cells that feed it (gates 1/3/downgrades: every (case, arm, + repeat) of every bug case; gate 2: every S3 cell) with the cell's coverage + class and its hits (marker names + the stripped command line). A hit is + **orientation-only** iff all its marker names are in + `reviewer_audit.ORIENTATION_MARKERS`; a cell is orientation-only iff all + its hits are; the verdict's first paragraph states how many gate-feeding + cells carry hits and how many are orientation-only, and lists the + non-orientation markers of every remaining hit. The user reads that list + and decides; the mechanical label stands unless overridden in writing. + +## Grading protocol (blinded, multi-grader) + +1. `compare --blinded` writes `comparison.blinded.md` (M* labels, model + self-references scrubbed, latency redacted, a visible Bundle ID) and a + sealed `blinding.json` (mapping, `bundle_id`, `run_ids_sha`, + `artifacts_sha`, `rubric_sha256`, `prompt_sha256`) that graders NEVER read. + `verdict` REGENERATES the blinded bundle deterministically from the sealed + artifacts and refuses unless the file is byte-identical, so the + grader-visible evidence is bound to the artifacts, not only to itself. +2. 3 INDEPENDENT blinded grader subagents receive ONLY the blinded bundle, + `GRADING_RUBRIC.md` and `.github/codex/prompts/pr_review.md` (never this + file, never the 2026-07 verdict, never raw artifacts). Isolation is by + instruction plus two checks (the prompt lists the three permitted paths + and forbids Bash; each returned table is grepped for paths, run ids, + config ids and model names before acceptance), not a sandbox. +3. One verifier subagent (same three inputs) reconciles disagreements and + rules on every `LEAK?` flag with quoted evidence (pass 1), then examines + EVERY remaining bug cell and S3 cell against the confirmation criteria in + one pass (pass 2, at most 48 cells). Only diff-independent evidence confirms. +4. Unblind only after the tables are final; apply the gates mechanically + (`verdict` without `--audit-hits` = PROVISIONAL); then run `audit` and the + same `verdict` with `--audit-hits` (= DEFINITIVE). Both outputs go in the + verdict file. + +Blinding caveat: with two k=2 arms the labels are a clean 50/50 blind. + +## Corpus floor and provenance + +The campaign runs only with >= 8 verified cases including >= 2 S4 cases +(`verdict` refuses below the floor). Provenance recorded in the verdict: +frozen PR-1 HEAD sha, tree sha, local tag, PR number, subdir, later main +merge sha. The manifest's `repo_head_sha`/`repo_tree_sha` must equal them, +with `run_head_shas == []`, `run_head_missing == 0`, +`run_dirty_any == false`, `run_marker_mismatch == 0`, `run_not_isolated == 0`; +`verdict` recomputes the aggregates from the sealed artifacts and refuses on +any difference. Any push to PR-1 after the campaign starts VOIDS the run (a +voided campaign is re-run into a fresh subdir on the new HEAD). + +**Campaign ledger (enforced by `run_eval.py`):** the first dispatch of the +registered contrast creates the local git ref +`refs/campaign/-/frozen` at the preflight HEAD (in the +repository, never pushed, outside the disposable `runs/` dir), and every +dispatch appends `{contrast, head, tree, subdir, ledger_reset}` to the +campaign-global `runs/campaign-ledger.jsonl`; the FIRST record is the frozen +candidate-exposure revision and must equal the ref. `verdict` refuses without +a ledger record, without the ref, when the ledger's first head differs from +the ref, or when the first record is marked `ledger_reset` (the ledger was +deleted after the ref existed) - all three are protocol-guard refusals, not +labels - and otherwise judges the manifest against the first record: mechanical iff this subdir IS +the first dispatch, or the HEAD moved (a voided campaign re-run into a fresh +subdir) and the registered surfaces (`tools/reviewer-eval/`, +`tools/eval_core/`, `.claude/scripts/openai_review.py`, +`.github/codex/prompts/pr_review.md`, `tools/notebook_md_extract.py`) are +untouched between the two commits. **Pre-registered exception:** a diff that +is exactly the single `CALL_TIMEOUT_S = ` line in +`adapters/codex_reviewer.py` with the value strictly INCREASED (a lower cap +changes censoring and is not exempt) keeps the campaign pre-registered, +because a timeout cannot change what a completed review says. A second subdir +on the SAME head (re-exposure), the FIRST subdir reused after a HEAD change +(a voided campaign re-runs into a fresh subdir), or any other +registered-surface change makes the verdict "RE-REGISTERED after candidate +exposure": still computed and +reported, labelled non-mechanical with the surface diff stat, final call the +user's, never a mechanical GO. The ledger lists every subdir the contrast +ran in. + +Known, accepted limit: provenance is sampled at the start of each review and +at manifest write; a tracked input edited after one sample and restored +before the next is invisible to the fields (the tree-identity guard pins the +tree only at the post-run stages). + +## Stop conditions (enforced by `run_eval.py run`) + +Both integrity logs (`runs/-attempts.jsonl`, `runs/campaign-ledger.jsonl`) +FAIL CLOSED: a malformed line or a record missing a required field refuses +the stage with the file and line number, never a silent skip. + +At most 3 reruns after the first attempt (resume reuses OK artifacts). Every +`run` attempt appends a START record before dispatch and an END record (the +per-attempt failure record: case id, arm, repeat, first 120 chars of +`infra_error`) to the append-only `runs/-attempts.jsonl` BEFORE +anything can overwrite the artifacts; an attempt that raised before dispatch +(`CLIVersionMismatch`, `ConfoundMismatch`) has no END record and does not +count. An attempt interrupted or failed AFTER dispatch (an interrupt, an +unexpected exception) gets an END record marked `aborted` and counts toward +the cap; a fully cached resume (no codex call) writes no END record and does +not count. Every END record lists the OK cells of that attempt with their +artifact hashes - an aborted attempt's record is rebuilt from the store for +every planned cell (cached and freshly persisted alike; each review is +persisted by the worker that produced it before the coordinator sees it, so +an interruption of the coordinator still protects every finished review); `run` +refuses a new attempt when any artifact listed OK by ANY earlier END record +(the cumulative set) is missing or changed (a rerun RESUMES, it never +re-samples a successful cell). One attempt per subdir at a time: `run` holds +an exclusive cross-process lock (`runs/.lock`) from the stop-condition +check through the manifest write and refuses a concurrent `run` into the same +subdir, so no artifact set can be interleaved from two invocations and sealed +by the last one. `run` REFUSES a fifth dispatched attempt into the same subdir and +refuses to start when a (case, arm, repeat) timed out in the last 2 +consecutive attempts; both checks run BEFORE anything on disk is touched, so +a refused attempt leaves the previous successful manifest and its sealed +artifacts intact; both are surfaced to the user (never silently excluded, +never graded partial). A provenance PREFLIGHT also runs before the manifest +is touched and before any codex call: a dirty working tree or an incomplete +git sample (a `None` ref or worktree list, an empty sha/branch/common dir) +refuses the attempt, so a campaign the post-run gate would reject never +spends the candidate's first corpus exposure. `run` also fails closed (failure marker, +exit 2) when its write-time provenance sample is incomplete (a `None` ref or +worktree list, an empty sha/branch/common dir), and `verdict` refuses such a +manifest at its protocol guard. The operator's run-log headers duplicate +this record for the verdict file. + +## Candidate exposure (enforced by `run_eval.py`) + +A registered candidate arm (`reviewer_verdict.REGISTERED_CONTRASTS`) can be +selected ONLY as the exact registered campaign: `run --configs , +--k 2` with no `--k-per` and no `--strata`; `smoke` refuses it outright. So +"the candidate's first corpus run is the campaign" holds by construction in +code, not only by protocol. An EMPTY `session_commands` +on the 0.162.1 control smoke was a STOP condition before the PR-1 commit (it +did not occur: the smoke parsed 8/8 exec blocks). diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-VERDICT.md b/tools/reviewer-eval/campaigns/2026-10-gpt6-VERDICT.md new file mode 100644 index 00000000..b107a8a7 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-VERDICT.md @@ -0,0 +1,151 @@ +# GPT-6 Astra Reviewer Evaluation - Verdict (2026-10-17): NO-GO + +Pre-registered rule: `2026-10-gpt6-DECISION_RULE.md` (archived from `DECISION_RULE.md` +when campaign 2 re-issued it for arm F; pre-registered in PR #849, frozen at the +#849 squash merge `225a4afb` on `main`). Arms: B = gpt-5.6-sol @ xhigh k=2 (control, +production at the time), E = gpt-6-astra @ xhigh k=2 (candidate; its first corpus +run was this campaign). 44/44 runs OK in one attempt (41 minutes at +`--max-parallel 4`), 11 cases, zero INFRA_ERROR. + +**Mechanical verdict (`run_eval.py verdict`, DEFINITIVE with `--audit-hits`): `NO-GO (DEFINITIVE)`.** +Gate (1) regression: `s4-cs-unbalanced-nan-outcome:b1` is RELIABLE for B and MISSED for E. +Gate (2) FP: E 0 vs B 0, passes. Downgrades: 0. Strict improvements: `s2-nan-dof-fail-closed:b1` +(E reliable, B missed), which precedence step (3) does not reach. Judgment flag: +`s2-acrt-single-dose:b1` B UNSTABLE / E MISSED (reported, not gating). Contested gates: none. + +**User's final call (2026-10-17): accept mechanical (NO-GO).** No swap to gpt-6-astra. +Reason recorded: the regression is a methodology miss of the kind the reviewer +exists to catch (sol named the NaN-outcome routing defect in both repeats; astra +twice reported a neighbouring NaN-cohort issue in the same file instead), the +arms are otherwise close to even on this corpus, and astra costs 2.5x on CI. + +## Catch table (10 ground-truth bugs; C = caught, P = partial, M = missed; partial counts as missed for every gate) + +| Bug (stratum) | B r0 / r1 | E r0 / r1 | B status | E status | +|---|---|---|---|---| +| s1-coef-dict-collision:b1 (S1) | C / C | C / C | reliable | reliable | +| s1-cs-dr-plugin-se:b1 (S1) | C / C | C / C | reliable | reliable | +| s2-acrt-single-dose:b1 (S2) | P / C | P / M | unstable | missed | +| s2-cic-practitioner-screen:b1 (S2) | P / P | P / P | missed | missed | +| s2-had-cluster-guard:b1 (S2) | C / C | C / C | reliable | reliable | +| s2-nan-dof-fail-closed:b1 (S2) | P / P | C / C | missed | reliable | +| s4-cs-unbalanced-nan-outcome:b1 (S4) | C / C | P / P | reliable | missed | +| s4-lpdid-survey-unreduced-design:b1 (S4) | P / C | C / C | unstable | reliable | +| s4-wcr-saturated-guard-order:b1 (S4) | P / M | M / M | missed | missed | +| s4-wcr-saturated-guard-order:b2 (S4) | M / M | M / M | missed | missed | +| **Reliable catches** | | | **4/10** | **5/10** | +| **Caught repeat-cells (of 20)** | | | **10** | **10** | + +Mapping (unblinded after the tables were final): B = M2, E = M1. + +## Pre-registered gates (E vs B) + +1. **Regression NO-GO: FAIL.** `s4-cs-unbalanced-nan-outcome:b1`: B caught 2/2, E partial 2/2 (three graders unanimous). +2. **FP NO-GO: PASS.** 0 vs 0 on both negative controls (every finding at or below the allowed severities; the s3-wcr P3 on the documented floor appears in every arm and is allowed). +3. **GO conditions:** not reached (precedence step 3 fires first). For the record: zero downgrades, one strict improvement (`s2-nan-dof-fail-closed:b1`, E caught 2/2 where B was partial 2/2), and E also reliable on `s4-lpdid-survey-unreduced-design:b1` where B was unstable (not a rule improvement because B is not MISSED). +4. **PARITY:** not reached. + +Precedence (0)-(7) applied as written; outcome at step (3). + +## Informational reads (non-gating) + +| | B (gpt-5.6-sol) | E (gpt-6-astra) | +|---|---|---| +| Latency mean / p95 / max (s) | 209 / 343 / 432 | 214 / 331 / 392 | +| Candidate p95 > 2x control | | False | +| Input price (USD per 1M, standard tier) | 4.00 | 10.00 | +| Initial-prompt cost estimate, whole arm (USD, +/-50%) | 1.26 | 3.15 | +| Reviews with a non-shell event (web search) | 18 of 22 | 21 of 22 | +| Mean shell commands per review | 14.0 | 16.6 | +| Hallucination / unverified external-tool-claim cells | 22 | 22 | +| Confirmed leaks / unconfirmed flags | 0 / 0 | 0 / 0 | + +The hallucination counter is uninformative on this corpus: every review in both +arms cites a test run or a numeric reproduction the blinded graders could not +verify, so each arm scores 22 of 22 cells. The 2026-07 campaign recorded this +as a known Sol-family trait; it now holds for every arm measured. + +Vendor lifecycle (prose, operator-supplied): codex-cli 0.162.1's model catalog +labels gpt-5.6-sol "older generation workhorse"; gpt-6-astra is "frontier +intelligence for the most demanding work" and gpt-6.1-sol "performs near the +Astra level on complex work at lower cost" (developers.openai.com/api/docs/models, +read 2026-10-17). + +## Session-log audit (`run_eval.py audit`, post-unblinding, informational) + +Coverage: covered=5, no_commands=0, unaudited=39, total=44; vacuous=False. +Nearly every review web-searched (codex default features, decision 8), which +classifies the cell `unaudited`; the shell log is still scanned and reported. + +- Cells with at least one hit: 39 of 44; cells whose hits are all orientation-only: 5. Hits per arm: {'B': 44, 'E': 61}. +- Most frequent markers: `repo_root` x41, `-maxdepth` x33, `GIT_LOG_RE` x25, `tools/reviewer-eval/` x8, `worktrees/` x7, `--format` x6, `GIT_DIR_RE` x6, `BARE_SHOW_RE` x4. The bulk are codex's own AGENTS.md discovery walk up the parent directories (`find -maxdepth`, `../..` loops, absolute paths under the enclosing checkout) and pytest invocations through the enclosing checkout's `.venv`; none of these reads the answer key. +- Hits on ref-reaching markers (`main`/`master`, `origin/`, `refs/`, tags, `rev-list`, `reflog`, `for-each-ref`, `cat-file`, `fsck`, `worktree list`): none. +- S1 cells whose shell log shows `git log` / `git show HEAD` (the synthetic HEAD's commit subject names the injected bug): s1-coef-dict-collision/E/r0, s1-coef-dict-collision/E/r1, s1-cs-dr-plugin-se/B/r0. Both arms caught both S1 bugs reliably in every repeat regardless, the verifier found no review citing the commit message as its source (confirmation criterion 3), and no gate is touched; recorded for transparency. +- The committed `hits.json` copy replaces each hit's `line` with its sha256 (command lines can carry file contents); the full lines stay in the gitignored original whose sha256 is listed below. + +## Grading record + +Three independent blinded grader subagents (inputs: `comparison.blinded.md`, +`GRADING_RUBRIC.md`, `.github/codex/prompts/pr_review.md`; Bundle ID +`d27906ede74da295`; blind labels M2=B, M1=E, mapping unknown to +them). Isolation is by instruction plus check, not a sandbox: the grader prompt +listed the three absolute paths as the only permitted reads and forbade Bash +and every other path, and each returned table was grepped for paths, run ids, +config ids and model names before acceptance (all three clean). The scratch +copies of the rubric and prompt hashed to `blinding.json`'s `rubric_sha256` / +`prompt_sha256`, and the scratch bundle was byte-identical to the run-dir bundle +(`shasum -a 256` equal), before any grader was spawned. + +Inter-grader agreement: 33 of 40 bug cells unanimous; all 8 negative-control +cells unanimous (every list empty); 0 `LEAK?` flags raised. Verifier pass 1 ruled +the disputed cells; verifier pass 2 examined all 44 reviews against the leak +confirmation criteria and confirmed none, with no grade corrections. Reconciled +S3 FP totals: B 0, E 0. + +Committed grading files under `campaigns/2026-10-gpt6-grading/` (blind labels; machine paths +redacted to ``, ``, `
`, ``, ``, +``; sha256 campaign-time original -> committed copy): + +- `grader-1.json`: `0ee28b8ae353d6662efe89e3a3726ba1351afb8c72dd8943194356d515da1012` -> `0ee28b8ae353d6662efe89e3a3726ba1351afb8c72dd8943194356d515da1012` +- `grader-2.json`: `8f807cf646a6d1d41c30049b67f1e3df02dafa8f02b5759a05c29caaee5be408` -> `8f807cf646a6d1d41c30049b67f1e3df02dafa8f02b5759a05c29caaee5be408` +- `grader-3.json`: `5e50f8622adf65d6cbaf0b161fdfbabf966773293cf117627259f3013ff89940` -> `5e50f8622adf65d6cbaf0b161fdfbabf966773293cf117627259f3013ff89940` +- `verifier.json`: `73ddba0b85647d7e6ff1e0d6ee61ffcb9722fa1612f6839795759c26f3fa1959` -> `73ddba0b85647d7e6ff1e0d6ee61ffcb9722fa1612f6839795759c26f3fa1959` +- `reconciled.json`: `8fc6b8b810e422737c729137c6c504145339c8bb65321c94b7eabbfef5fcf2c8` -> `8fc6b8b810e422737c729137c6c504145339c8bb65321c94b7eabbfef5fcf2c8` +- `hits.json`: `955b56350217c5f32ad82d3374953052cb9db8ccd54de2d9e4b881fb0227ad4b` -> `27c1b65712e224bd06adeec1b42645085c5c72d42cb9404d21c48ab0b9690085` +- `verdict-provisional.txt`: `af8fbcc6aad9f3fc77aaaba9812b40cc5f2f7f3521ad32e40f93d2e38591372f` -> `af8fbcc6aad9f3fc77aaaba9812b40cc5f2f7f3521ad32e40f93d2e38591372f` +- `verdict-definitive.txt`: `9e107f60f2f9239dc200631704347b6cf87d209aeab02538685674a08a8fbdfa` -> `48540e912a16efa8b442e363536ff8246c230597f30bbf7cfaa4cda10ff8f4a2` + +## Grader notes (blinded, consensus) + +The same qualitative picture from all three graders: M1 (= E) and M2 (= B) both +produce location-precise, reproduction-style findings; M1 reports pytest as +unable to start in the sandbox and falls back to "direct" Python reproductions, +M2 cites test counts. Shared blind spots: both s4-wcr guard-order bugs (every +arm, every repeat, as in 2026-07) and the QDiD framing in s2-cic (all four cells +attack the same screen on the CiC axis; verifier ruled partial). Neither arm +reproduces the 2026-07 observation that sol misses the S4 bugs: B caught +s4-cs 2/2 and s4-lpdid 1/2 in this run. + +## Deviations from the plan, recorded + +- The campaign ran on `main`'s tip after #849 merged, not on the open PR branch; + the ledger's first dispatch record is that sha and the tag pins it locally. +- The pre-freeze rehearsal was not repeated on `225a4afb` because the registered + surfaces were byte-identical to the rehearsed `ec6226ae` (only the 3.13.0 + version bump and CHANGELOG differed). + +## Provenance (committed audit record) + +- Frozen HEAD `225a4afb73106614c37d34b21100eafcdf34be43`, tree `5c05e9e2c5019c4f98067c28dbb63d9322c94bac`, local tag `campaign-gpt6` (never pushed), branch `codex-upgrade`, subdir `runs/gpt6`. +- Manifest: 44 OK run_ids, configs ['B', 'E'], k=2; `repo_dirty`=False, `run_head_shas`=['225a4afb73106614c37d34b21100eafcdf34be43'], `run_head_missing`=0, `run_dirty_any`=False, `run_marker_mismatch`=0, `run_not_isolated`=0; `verdict` recomputed the aggregates from the sealed artifacts and found no difference. +- `artifacts_sha` (sealed at `run` exit): `83337a6b7e139d7ecade26a6560e47b98b9a55659fd666e0fb7de0c1e47d6ee1`. +- Exposure ledger: first dispatch of the contrast (frozen revision); mechanical=True; ledger entries for this contrast=1; subdirs=['gpt6']. +- Attempts: 1 dispatched attempt, 0 reruns, 0 INFRA_ERROR cells. Run-log records (verbatim): + - `=== run attempt 2026-10-17T17:53:35Z HEAD=225a4afb73106614c37d34b21100eafcdf34be43` + - `RUN EXIT 0` + - `=== sealed 225a4afb73106614c37d34b21100eafcdf34be43 83337a6b7e139d7ecade26a6560e47b98b9a55659fd666e0fb7de0c1e47d6ee1` +- Codex CLI 0.162.1 on every arm (runner-asserted); isolated `CODEX_HOME` (credential + `sandbox_mode = "read-only"` + `approval_policy = "never"`, codex default features otherwise; the default feature surface is the gitignored `runs/features-0.162.1.txt`, captured by `codex features list` under the isolated home on 2026-10-16). +- Hashes (sha256) at verdict time: `runs/gpt6/comparison.md` `69dab0d9e44688b572767a046e9c2b7b1b066c91365491bee587d52906158400`; `runs/gpt6/comparison.blinded.md` `0ff258435e1b4f7a87eb019c2715294422b661cd16ee44af1193cae27f77da4b`; `runs/gpt6/blinding.json` `9b61cae0d27a3720e678cd37a5b7679d864ad5e97ae4348bced0d0511c29fdbb` (bundle_id `d27906ede74da295`, run_ids_sha `413aae215614d4909ab25c85e66b750b233ec7a731179001c3feb926242e4264`, rubric_sha256 `0388e64b0b7defc1ec7a8d4af1950c7a15aa4ad43c62021f69668ba07f4cb6dd`, prompt_sha256 `098ecc7ee3bcfe10fedca4635f277966d717037def7f4162d145de494a217c80`). +- Files at the tag: `DECISION_RULE.md` `adb0ac511f7cef179f343ab4cd425edd420eddcf6b52640b27c11c71c983cd1a`, `GRADING_RUBRIC.md` `0388e64b0b7defc1ec7a8d4af1950c7a15aa4ad43c62021f69668ba07f4cb6dd`, `.github/codex/prompts/pr_review.md` `098ecc7ee3bcfe10fedca4635f277966d717037def7f4162d145de494a217c80`. +- run_ids (44): `s1-coef-dict-collision.B.r0.20aa9cdfe720d4bf`, `s1-coef-dict-collision.B.r1.6052f259b02bb528`, `s1-coef-dict-collision.E.r0.68031b6fd69d91a8`, `s1-coef-dict-collision.E.r1.dbd7255ebee2c3b3`, `s1-cs-dr-plugin-se.B.r0.05074433eec6818a`, `s1-cs-dr-plugin-se.B.r1.fc7146a2c93d3504`, `s1-cs-dr-plugin-se.E.r0.f9425b6303772854`, `s1-cs-dr-plugin-se.E.r1.6f972c909ca3317a`, `s2-acrt-single-dose.B.r0.4b72df486f75dedb`, `s2-acrt-single-dose.B.r1.a740a6a97984c8b8`, `s2-acrt-single-dose.E.r0.9cf50a17bfb4b7b1`, `s2-acrt-single-dose.E.r1.60289d8963706496`, `s2-cic-practitioner-screen.B.r0.6b3b0cc6400ca460`, `s2-cic-practitioner-screen.B.r1.ae2a0146b73da067`, `s2-cic-practitioner-screen.E.r0.2f01ad7139538355`, `s2-cic-practitioner-screen.E.r1.568da36ad60df6de`, `s2-had-cluster-guard.B.r0.84d356cf06268d78`, `s2-had-cluster-guard.B.r1.d93ccde8dcab37d8`, `s2-had-cluster-guard.E.r0.83185d5fc44ea848`, `s2-had-cluster-guard.E.r1.5073afcaefe3d417`, `s2-nan-dof-fail-closed.B.r0.b4a41e147c4f76d4`, `s2-nan-dof-fail-closed.B.r1.f7c068891af83ecb`, `s2-nan-dof-fail-closed.E.r0.58171b295011ffef`, `s2-nan-dof-fail-closed.E.r1.476917d44198ed86`, `s3-changelog-prose.B.r0.e915ea4986624500`, `s3-changelog-prose.B.r1.6145465e121eb71c`, `s3-changelog-prose.E.r0.d094a4936c921514`, `s3-changelog-prose.E.r1.173ade27085e8433`, `s3-wcr-pfloor-comment.B.r0.4701bae6768929fa`, `s3-wcr-pfloor-comment.B.r1.aedad7cc7c2b6cff`, `s3-wcr-pfloor-comment.E.r0.0f52602211c3d990`, `s3-wcr-pfloor-comment.E.r1.77bc1cebbe9b298f`, `s4-cs-unbalanced-nan-outcome.B.r0.210c316a4170c0de`, `s4-cs-unbalanced-nan-outcome.B.r1.e16b6e9f972fe784`, `s4-cs-unbalanced-nan-outcome.E.r0.ca60b0c855ef3f89`, `s4-cs-unbalanced-nan-outcome.E.r1.8ce5e6f66e7db98b`, `s4-lpdid-survey-unreduced-design.B.r0.efaaeba38d503b0d`, `s4-lpdid-survey-unreduced-design.B.r1.23f6433415d5d879`, `s4-lpdid-survey-unreduced-design.E.r0.7dd4f402504a9d4e`, `s4-lpdid-survey-unreduced-design.E.r1.069b2a344443fca7`, `s4-wcr-saturated-guard-order.B.r0.bde484b74d09a291`, `s4-wcr-saturated-guard-order.B.r1.650895b917c6144c`, `s4-wcr-saturated-guard-order.E.r0.405e2ec712320ccc`, `s4-wcr-saturated-guard-order.E.r1.bdbe12f3335d000d`. + diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-1.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-1.json new file mode 100644 index 00000000..596ff142 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-1.json @@ -0,0 +1,55 @@ +{ + "bundle_id": "d27906ede74da295", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", "evidence": "Removing the time fixed-effect skip reintroduces duplicate coefficient names.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable pytest run (TestMPDAbsorbedFERParity: 10 passed, 1 failed) and a fixture reproduction (34 entries vs 38x38 vcov, period_4 = 0.860543)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", "evidence": "Removing the time-FE guard introduces redundant columns and duplicate coefficient names.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable pytest run (10 passed, 1 failed at tests/test_estimators_vcov_type.py:L1324) and fixture reproduction."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", "evidence": "When `time=\"period\"`, both blocks use names such as `period_4`. The dictionary comprehension silently collapses those duplicate names, while `vcov` retains every matrix column.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; cites an unverifiable direct reproduction (34 keys vs 38x38 VCOV, period_effects[4] = 0.4406)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "The change reintroduces duplicate time-dummy columns and duplicate `period_*` coefficient names.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; cites an unverifiable direct reproduction (34 coefficients vs 38x38 VCV)."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", "evidence": "Using `ddof=1` introduces separate `n/(n−1)` corrections for treated and control variance contributions, while the returned influence function remains unchanged.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable pytest run (TestDRNoCovariateSEUniformity: 2 failed, 2 passed) and a hand-fixture SE comparison (0.721688 vs 0.625)."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", "evidence": "`ddof=1` changes each variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n(n−1))`. The returned influence function remains unchanged, so aggregation and bootstrap retain the previous variance convention.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable pytest run (2 failed, 2 passed) and hand-fixture numbers."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", "evidence": "The new `ddof=1` plug-in SE disagrees with both the returned influence function and the documented DRDID variance convention.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). Cites an unverifiable targeted test run (two failures, hand fixture 0.7216878 vs 0.625) and an implausible line reference into DRDID drdid_panel.R (#L1104-L1108)."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "Per-cell SEs, t-statistics, p-values, and confidence intervals become inflated and inconsistent with the influence function used for aggregation/bootstrap.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not initialize in the sandbox; no execution claimed."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same file/function class (saturated derivative boundary convention accepted as documented P3; SATURATED_TOL P0 in continuous_did_bspline.py) but never identifies the J=1 all-zero derivative / ACRT=0 defect. Cites unverifiable test run (127 passed, 8 skipped) and reproductions (SE 0.00091; ATT 3.33333 after rescaling)."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Discusses the derivative boundary convention (P3, accepted) and other ACRT inference issues but not the single-positive-dose ACRT=0 defect. Cites unverifiable test run (127 tests), a 100-replication coverage simulation (4%), and analytical-vs-delta SE numbers."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites the exact location (continuous_did_bspline.py:L324-L368) but only as the documented gap-scaled/forward-difference convention (P3); does not identify the J=1 zero ACRT. Cites unverifiable direct probes (NaN dose, double survey weighting)."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "With `J=1`, the derivative operator is all zeros, so a binary `{0,1}` design returns `ATT=2` but `ACRT=0`.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (19 non-slow discrete tests pass) and a weighted reconstruction (SE 1.1528 vs 2.0749)."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the mean parallel-trends screen at the same location (practitioner.py:L1535-L1541) but as over-stated for CiC ('not necessary'; label as descriptive). Does not identify the ground-truth QDiD defect that under the additive model a mean-trend break is direct evidence against the model (screen understated). Cites unverifiable test run (93 passed) and numeric checks (p≈8.6e-27)."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same as repeat 0: flags the 'necessary' mean-trend screen for CiC as too strong and recommends treating it as descriptive; misses the QDiD understatement. Cites unverifiable test run (93 passed) and reproduction (p≈7.7e-148 vs p=1)."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the CiC mean-PT screen at L1528-L1558 as not necessary, and explicitly recommends keeping it 'only as a QDiD necessary-but-insufficient screen' - endorsing the ground-truth defective QDiD framing. States pytest could not run."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the CiC mean-PT screen (L1528-L1558) as not necessary; raises QDiD additive separability only as an omitted assumption, never linking it to the mean-trend screen's evidential value. States pytest could not start."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", "evidence": "The newly forwarded cluster array reaches a variance routine without a cluster-count guard. Its finite output passes through `safe_inference()`, producing misleading statistical results.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (390 passed, 3 deselected) and a reproduction (se=0.84003, p=0.85809)."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", "evidence": "The newly forwarded cluster array reaches a variance routine that sums cluster scores without rejecting a single contributing cluster. `safe_inference()` subsequently accepts its finite SE.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (437 passed) and reproduction (SE=0.077828, p=7.77e-45); test counts differ from repeat 0 (390) for the same file set."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", "evidence": "The new path forwards any cluster array without checking the number of effective clusters. With 200 units assigned to one cluster, a direct reproduction returned `se=0.005974`, `t_stat=246.86`, `p_value=0.0`, and a finite CI.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; cites an unverifiable direct reproduction."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "P0 — Cluster inference does not fail closed with one effective cluster.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; cites unverifiable direct reproductions (att=1.7195, se=0.1221; mixed-type TypeError)."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", "evidence": "The new guard returns NaN DOFs, but `MultiPeriodDiD` substitutes its shared analytical DOF for affected period coefficients. The warning says inference is suppressed, yet finite p-values and CIs remain. The average-effect path instead returns a finite t-statistic alongside NaN p-value and CI.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (631 passed) and a before/after reproduction table (p 0.061238 -> 0.048684)."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "caught", "evidence": "Average inference passes NaN DOF through `safe_inference()`, retaining a finite t-statistic beside NaN p-values/CIs.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (164 passed) and reproductions (p 0.0564 -> 0.0451; t=-2.11546 with NaN p)."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the same CR2-BM guard (linalg.py:L2104-L2121) for predictor-unit scale dependence only; does not identify that non-finite df passes downstream as finite/fallback inference. Cites unverifiable test run (164 passed) and a 12-cluster check (DOF 7.81631609 -> NaN)."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard scale-dependence P1 at linalg.py:L2104-L2121 plus a dCDH placebo-sign P0; the non-finite-df fail-open is not identified. Cites unverifiable reproductions (DOF 7.6849187 -> NaN; placebo -2 vs +2)."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags NaN-handling in the new routing path only for the cohort column (nunique() ignoring NaN first_treat, L1931-L1932), not NaN outcomes in _is_unbalanced_panel. Cites unverifiable test run (346 passed), reproductions (SE 0.1013970685 vs 0.1201965009; ATT 2.4883965751 vs 2.4833883573) and a failed R regeneration attempt."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same cohort-NaN finding (L1931-L1937) and RC IF-scale P0; NaN-outcome routing not identified. Cites unverifiable test run (101 passed, 1 deselected) and reproductions."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", "evidence": "A structurally balanced panel with one missing outcome remains on the panel estimator with `used_rc_on_unbalanced_panel=False`, while R drops that row, detects an unbalanced effective sample, and uses the RC estimator.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest unavailable; cites unverifiable read-only Python reproductions (routing flag flip; n_treated=213/250)."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "Balance is checked before complete-case filtering. A rectangular panel containing missing outcomes is treated as balanced, so `allow_unbalanced_panel=True` remains on the panel-difference estimator instead of using the R-compatible RC estimator.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). States pytest could not start; cites unverifiable reproduction (padded-NaN golden e=1 ATT 3.3949106 vs 3.1392963)."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing `response - design @ coef` therefore makes every residual NaN; PSU aggregation subsequently reduces these to zero.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Describes the symptom as SE collapsing to 0.0 rather than NaN-poisoning, same mechanism and location. Cites unverifiable test run (105 passed) and reproductions (SE 2.50e-16; 0.043541 -> 0.0)."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", "evidence": "`solve_ols()` marks dropped redundant coefficients NaN and returns valid residuals separately. Discarding those residuals and calculating `design @ coef` makes every residual NaN.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (105 passed) and reproductions (SE 2.78e-16; 0.0447574 -> 0.0)."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "partial", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings target the same function range (lpdid.py:L920-L972) but concern survey df/PSU metadata and the _subpop_weight rejection, not the discarded residuals / unreduced design sandwich. States pytest could not start; cites unverifiable direct reproductions."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "caught", "evidence": "The new path discards those residuals and recomputes `response - design @ coef`; one dropped nuisance coefficient therefore makes every residual NaN.", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable reproduction (SE 0.106265 -> 0.0)."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are confined to utils.py CI inversion (disconnected acceptance set, stopping tolerance). Cites unverifiable test run (59/59) and enumeration reproductions."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States 'saturated-design ... fixes are present' with no finding; findings are utils.py numerical issues. Cites unverifiable test run (59/59) and reproductions (seed 10; y+1e7)."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only P0 is FE omission in bootstrap refits (twfe.py/utils.py, estimators.py L458-L475/L683-L687) - unrelated to the storage-vcov rank-deficient crash. States pytest could not start; cites unverifiable 8-unit enumeration (SE 0.529006 vs 0.808070)."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are utils.py inversion monotonicity, dataclass ordering, and fixture gaps. Cites unverifiable test run (59 passed) and a four-cluster reproduction (p=0.125 with CI excluding zero)."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No linalg.py finding; saturated guard / cluster-count ordering not discussed."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No linalg.py finding; saturated-design handling asserted as fixed."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Saturated designs stated as 'appear resolved'; no guard-order finding."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", "evidence": "", "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No linalg.py finding."} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 terminology refinement; tests inspected, not executed."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No findings; tests inspected, not executed."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 terminology finding."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No findings; cites an unverifiable targeted test run ('2 passed in 0.12s')."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "P3 informational note on the documented floor only (allowed); tests inspected, not run."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "P3 informational note on the documented floor only (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "P3 informational only (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "P3 comment-wording finding on 'bootstrap resolution' only (allowed)."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-2.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-2.json new file mode 100644 index 00000000..c365fe6b --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-2.json @@ -0,0 +1,191 @@ +{ + "bundle_id": "d27906ede74da295", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the new dummy block repeats existing `period_*` names. The same happens through the HC2/HC2-BM `absorb=` auto-route. Rank-deficiency handling marks redundant columns NaN, then dictionary construction overwrites earlier coefficients and collapses entries", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims a pytest run (10 passed, 1 failed) and a reproduced 34-vs-38x38 mismatch; external-tool claims not verifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "the new dummy block repeats existing `period_*` names. Dictionary construction then overwrites original coefficients while retaining the full covariance matrix.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims a pytest run (10 passed, 1 failed) and reproduced NaN period_4; not verifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "When `time=\"period\"`, both blocks use names such as `period_4`. The dictionary comprehension silently collapses those duplicate names, while `vcov` retains every matrix column.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start; claims a write-free direct reproduction (34 keys vs 38x38 vcov); not verifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "the changed loop adds identical columns with identical `period_` names ... the later dictionary comprehension silently collapses duplicate names and allows the dropped `NaN` entries to overwrite the identified period coefficients.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start; claims direct estimator reproduction; not verifiable here."}, + + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "Using `ddof=1` introduces separate `n/(n−1)` corrections for treated and control variance contributions, while the returned influence function remains unchanged. Consequently, aggregation retains the previous variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims pytest run (2 failed, 2 passed) and hand-fixture numbers (0.721688 vs 0.625); not verifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "`ddof=1` changes each variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n(n−1))`. The returned influence function remains unchanged, so aggregation and bootstrap retain the previous variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims pytest run (2 failed, 2 passed); not verifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE disagrees with both the returned influence function and the documented DRDID variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Graded P0 (expected P1). Claims a targeted test run produced two failures (0.7216878 vs 0.625); not verifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Per-cell SEs, t-statistics, p-values, and confidence intervals become inflated and inconsistent with the influence function used for aggregation/bootstrap.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not initialize in sandbox."}, + + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same file (continuous_did_bspline.py L264-289 tolerance merging) and a P3 accepting the lowest-dose forward-difference convention, but never names the J=1 all-zero derivative matrix / ACRT=0 defect. Claims 127 tests passed plus reproductions (SE 0.00091, ATT 3.33333); not verifiable here."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Touches the ACRT boundary convention only as an accepted P3; its P0s (overall ACRT variance, exact-identification bootstrap) are different defects in continuous_did.py. Claims 127 tests, a 4%-coverage simulation and delta-method SE numbers; not verifiable here."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right symbol (continuous_did_bspline.py:L324-L368) but frames the ACRT convention as a documented P3 (scaled slope / forward difference) and never names the single-level ACRT=0 case. Claims a direct NaN-dose probe and a double-weighting defect; not verifiable here."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "With `J=1`, the derivative operator is all zeros, so a binary `{0,1}` design returns `ATT=2` but `ACRT=0`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Also claims a survey double-weighting P0 with reconstructed SE 1.1528 vs 2.0749 and 19 passing tests; not verifiable here."}, + + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "CiC guidance incorrectly treats mean parallel trends as a necessary identification screen. ... Remove the “necessary-but-insufficient” requirement and the claim that mean-trend rejection undermines CiC time invariance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Attacks the same necessary-but-insufficient mean-PT screen in _handle_cic (L1535-1541), but argues the CiC 'not necessary' direction rather than the ground truth's QDiD 'mean break is direct evidence' framing. Claims 93 tests passed and numeric placebo checks (p≈8.6e-27); not verifiable here."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "P1 — Parallel mean trends are not necessary for CiC. ... Remove the necessity claim from both CiC prose and snippet comments. Describe mean trends as a descriptive comparison", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same screen/location; rationale is the CiC 'not necessary' direction, not the QDiD additive-model framing. Claims 93 tests passed and p≈7.7e-148 reproduction; not verifiable here."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "P1 — Mean parallel trends is not a necessary CiC screen ... The handler tells practitioners that a pre-period mean-trend break undermines CiC.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same screen/location (L1528-1558), CiC-direction rationale; its fix explicitly proposes keeping the screen 'only as a QDiD necessary-but-insufficient screen', which is the framing the ground truth calls mis-framed for QDiD. States pytest could not run."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "A failed mean-pretrend check does not necessarily undermine CiC, so the new guidance can reject valid designs. ... For CiC, remove the “necessary-but-insufficient” mean-trend screen.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same screen/location, CiC-direction rationale rather than the QDiD framing. States pytest could not start."}, + + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a cluster-count guard. Its finite output passes through `safe_inference()`, producing misleading statistical results.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 390 tests passed and a reproduction (se=0.84003, p=0.85809); not verifiable here."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine that sums cluster scores without rejecting a single contributing cluster. `safe_inference()` subsequently accepts its finite SE.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 437 tests passed and reproduction (SE=0.077828, p=7.77e-45); not verifiable here."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new path forwards any cluster array without checking the number of effective clusters. With 200 units assigned to one cluster, a direct reproduction returned `se=0.005974`, `t_stat=246.86`, `p_value=0.0`, and a finite CI.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start; claims direct API reproduction; not verifiable here. Also raises a P1 CR1-label mismatch not in ground truth."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "P0 — Cluster inference does not fail closed with one effective cluster. ... reaches a cluster sandwich that accepts one cluster. There is no between-cluster replication, so cluster-robust inference is undefined", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start; claims direct reproduction (att=1.7195, se=0.1221) and a mixed-type TypeError; not verifiable here."}, + + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The new guard returns NaN DOFs, but `MultiPeriodDiD` substitutes its shared analytical DOF for affected period coefficients. The warning says inference is suppressed, yet finite p-values and CIs remain. The average-effect path instead returns a finite t-statistic alongside NaN p-value and CI.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 631 tests passed and a before/after p-value table (0.061238 -> 0.048684); not verifiable here."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The new guard returns NaN DOF, but `MultiPeriodDiD` handles those values inconsistently: Per-period inference substitutes the shared analytical DOF and returns finite p-values/CIs. Average inference passes NaN DOF through `safe_inference()`, retaining a finite t-statistic beside NaN p-values/CIs.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 164 tests passed and p-value shift 0.0564 -> 0.0451; not verifiable here."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file (linalg.py, guard at L2104-L2121) but a different defect: predictor-unit scale dependence of the guard; does not name non-finite df failing open downstream. Claims 164 tests passed and a DOF 7.81631609 -> NaN check; not verifiable here."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file but same scale-invariance defect as repeat 0; does not name the non-finite-df fail-open. Also raises a P0 dCDH placebo-sign claim contrary to the PR's registry note; unverifiable."}, + + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file (staggered.py) and NaN-handling class, but the NaN finding is about missing first_treat cohort values (L1931-1932) and row order, not NaN outcomes in _is_unbalanced_panel routing. Claims 346 tests passed and SE/ATT reproductions; not verifiable here."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same as repeat 0: NaN-cohort/row-order and RC IF-scale findings in staggered.py, not the NaN-outcome routing defect. Claims 101 tests passed and numeric reproductions; not verifiable here."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "A structurally balanced panel with one missing outcome remains on the panel estimator with `used_rc_on_unbalanced_panel=False`, while R drops that row, detects an unbalanced effective sample, and uses the RC estimator.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest unavailable; claims read-only reproductions; not verifiable here."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "This PR checks only whether raw `(unit,time)` rows exist. Consequently, padding an unbalanced panel with `outcome=NaN` rows silently disables RC routing.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Graded P0 (expected P1). States pytest could not start; claims golden reproduction (3.3949106 vs 3.1392963); not verifiable here."}, + + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing `response - design @ coef` therefore makes every residual NaN; PSU aggregation subsequently reduces these to zero.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 105 tests passed and reproductions (SE 0.043541 -> 0.0, saturated SE 2.50e-16); not verifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Discarding those residuals and calculating `design @ coef` makes every residual NaN. The survey helper’s grouped sums then turn these into zero scores and return zero variance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 105 tests passed and reproductions (SE 0.0447574 -> 0.0); not verifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file and even cites L920-L972, but the defects named are survey-metadata/df mismatch and _subpop_weight rejection, not the discarded residuals / unreduced design sandwich. States pytest could not start; claims direct reproductions; not verifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The new path discards those residuals and recomputes `response - design @ coef`; one dropped nuisance coefficient therefore makes every residual NaN.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims a reproduction (SE 0.106265 -> 0.0); not verifiable here."}, + + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are CI-inversion defects in utils.py; nothing on the estimators.py storage-vcov crash. Claims 59/59 tests passed and enumeration reproductions; not verifiable here."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are numerical-tie/zero-variance defects in utils.py. Claims 59/59 tests passed and seed/translation reproductions; not verifiable here."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites estimators.py absorb preprocessing (L458-L475) and the bootstrap call (L683-L687), but the defect named is FE non-reprojection in TWFE/absorb bootstrap, not the rank-deficient full-design storage vcov raise. States pytest could not start; claims an 8-unit enumeration (SE 0.529006 vs 0.808070); not verifiable here."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are non-monotone inversion (utils.py) and API/dataclass issues. Claims 59 passed and a four-cluster reproduction; not verifiable here."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Mentions 'shared HC1/CR1 handling for saturated designs' as affected but raises no guard-order finding in linalg.py."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Declares saturated-design fixes 'present'; no finding on the saturated guard preceding cluster-count validation."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": ""}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": ""} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 terminology refinement (allowed). Tests inspected, not executed."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No findings; clean read."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 terminology finding (allowed)."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No findings. Claims a targeted test run ('2 passed in 0.12s'); not verifiable here."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 informational note on the documented floor (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 informational note on the documented floor (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 informational note (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only a P3 comment-wording finding ('resolution' vs 'reporting floor'); allowed."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-3.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-3.json new file mode 100644 index 00000000..f3ba544b --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/grader-3.json @@ -0,0 +1,191 @@ +{ + "bundle_id": "d27906ede74da295", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "the new dummy block repeats existing `period_*` names. The same happens through the HC2/HC2-BM `absorb=` auto-route. Rank-deficiency handling marks redundant columns NaN, then dictionary construction overwrites earlier coefficients and collapses entries while retaining the full covariance matrix.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run (TestMPDAbsorbedFERParity: 10 passed, 1 failed) and a reproduced 34-vs-38x38 mismatch; unverifiable external-tool claims."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the new dummy block repeats existing `period_*` names. Dictionary construction then overwrites original coefficients while retaining the full covariance matrix.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run (10 passed, 1 failed at tests/test_estimators_vcov_type.py:L1324); unverifiable external-tool claim."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "When `time=\"period\"`, both blocks use names such as `period_4`. The dictionary comprehension silently collapses those duplicate names, while `vcov` retains every matrix column.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start (no writable temp dir) but claims a write-free direct reproduction (34 keys vs 38x38 VCOV, period_4 NaN); unverifiable."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "`solve_ols()` drops the redundant block as rank-deficient, but the later dictionary comprehension silently collapses duplicate names and allows the dropped `NaN` entries to overwrite the identified period coefficients.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Pytest reported as unable to start; claims direct estimator reproduction; unverifiable."}, + + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "Using `ddof=1` introduces separate `n/(n−1)` corrections for treated and control variance contributions, while the returned influence function remains unchanged. Consequently, aggregation retains the previous variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run (TestDRNoCovariateSEUniformity: 2 failed, 2 passed) and reproduced SE 0.721688 vs 0.625; unverifiable."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "`ddof=1` changes each variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n(n−1))`. The returned influence function remains unchanged, so aggregation and bootstrap retain the previous variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run (2 failed, 2 passed); unverifiable."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE disagrees with both the returned influence function and the documented DRDID variance convention.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rated P0 (expected P1). Claims a targeted test run producing two failures (DR vs reg 1.37%, hand fixture 0.7216878); unverifiable."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Per-cell SEs, t-statistics, p-values, and confidence intervals become inflated and inconsistent with the influence function used for aggregation/bootstrap.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not initialize; no reproduction claimed."}, + + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Raised two unrelated P0s (cross-cohort singleton detection; SATURATED_TOL merging doses) with claimed reproductions (SE 0.00091, ATT 3.33333) and a '127 passed' test run; unverifiable. No mention of single-level ACRT=0."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Raised unrelated P0s (overall-ACRT dose-composition variance; bootstrap false significance) with claimed simulations (4% coverage) and a 127-test run; unverifiable. No mention of single-level ACRT=0."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file/symbol (continuous_did_bspline.py:L324-L368) but only a P3 on the documented forward-difference/gap-scaling convention; does not name the J=1 all-zero derivative / ACRT=0 defect. Other findings (NaN dose P0, double survey weights P1) claim direct probes; unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "With `J=1`, the derivative operator is all zeros, so a binary `{0,1}` design returns `ATT=2` but `ACRT=0`. CGBS implicitly sets `d₀=0` and the zero-dose treatment effect to zero", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Also raises a P0 double survey-weighting claim with a reconstructed SE 1.1528 vs 2.0749 and says 19 non-slow tests pass; unverifiable."}, + + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Flags the same mean-PT screen at practitioner.py:L1535-L1541 but as 'not necessary for CiC' and recommends demoting it to descriptive; does not name the QDiD mis-framing (additive quantile model => mean-trend break is direct evidence). Claims 93 tests passed and numerical reproductions; unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same as repeat 0: CiC-necessity argument at L1534-L1539, not the QDiD evidential-value mis-framing. Claims 93 tests passed and p-value reproductions; unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Flags the screen at L1528-L1558 for CiC but explicitly endorses keeping it 'only as a QDiD necessary-but-insufficient screen', the framing the ground truth calls wrong. Pytest reported unable to start."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same location, CiC-side argument only; separately notes QDiD additive separability is omitted from the assumptions prompt but never connects it to the mean-trend screen's evidential value. Pytest reported unable to start."}, + + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a cluster-count guard. Its finite output passes through `safe_inference()`, producing misleading statistical results.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 390 tests passed and a single-cluster reproduction (se=0.84003, p=0.85809); unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine that sums cluster scores without rejecting a single contributing cluster. `safe_inference()` subsequently accepts its finite SE.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 437 tests passed and a reproduction (SE=0.077828, p=7.77e-45); unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new path forwards any cluster array without checking the number of effective clusters. With 200 units assigned to one cluster, a direct reproduction returned `se=0.005974`, `t_stat=246.86`, `p_value=0.0`, and a finite CI.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Pytest reported unable to start; direct reproduction claimed; unverifiable. Also raises a P1 CR1-label mismatch."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "reaches a cluster sandwich that accepts one cluster. There is no between-cluster replication, so cluster-robust inference is undefined, yet `safe_inference()` receives a positive finite SE and emits apparently valid statistics", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Pytest reported unable to start; direct reproductions claimed (att=1.7195, se=0.1221; mixed-type TypeError); unverifiable."}, + + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The new guard returns NaN DOFs, but `MultiPeriodDiD` substitutes its shared analytical DOF for affected period coefficients. The warning says inference is suppressed, yet finite p-values and CIs remain. The average-effect path instead returns a finite t-statistic alongside NaN p-value and CI.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 631 tests passed and a before/after p-value table (0.061238 -> 0.048684); unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Average inference passes NaN DOF through `safe_inference()`, retaining a finite t-statistic beside NaN p-values/CIs.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 164 tests passed and reproductions (p 0.0564 -> 0.0451; t=-2.11546 with NaN p); unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file and same new CR2-BM guard (linalg.py:L2104-L2121) but a different defect (predictor-unit non-invariance); the non-finite-df fail-open through safe_inference is not named. Claims 164 tests passed and a DOF 7.81631609 -> NaN reproduction; unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same scale-invariance finding on the guard (different defect); fail-closed NaN propagation not named. Additionally raises a P0 on the dCDH placebo sign that the arm's own repeat 0 classified as a documented P3 deviation; reproduction (DOF 7.6849187 -> NaN) unverifiable."}, + + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right file (staggered.py) and a related missing-value sample-construction finding (NaN first_treat bypasses nunique() validation, row-order dependence) but not the NaN-outcome routing in _is_unbalanced_panel. Claims 346 tests passed, a regression-IF scaling P0 (SE 0.1013970685 vs 0.1201965009) and ATT 2.4883965751 vs 2.4833883573; unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same as repeat 0: NaN-cohort finding in staggered.py, not NaN-outcome routing. Claims 101 tests passed and reproductions (SE 0.08178597 vs 0.10571237); unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "A structurally balanced panel with one missing outcome remains on the panel estimator with `used_rc_on_unbalanced_panel=False`, while R drops that row, detects an unbalanced effective sample, and uses the RC estimator.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Pytest reported unable to start; read-only reproductions claimed (missing outcome -> no routing warning; n_treated=213/250); unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "This PR checks only whether raw `(unit,time)` rows exist. Consequently, padding an unbalanced panel with `outcome=NaN` rows silently disables RC routing.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rated P0 (expected P1). Pytest reported unable to start; claims a padded-NaN golden reproduction (ATT 3.3949106 vs R 3.1392963); unverifiable."}, + + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing `response - design @ coef` therefore makes every residual NaN; PSU aggregation subsequently reduces these to zero.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Reports the symptom as SE collapsing to 0.0 rather than NaN-poisoned inference. Claims 105 tests passed and several reproductions (saturated SE 2.50e-16; SE 0.043541 -> 0.0); unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Discarding those residuals and calculating `design @ coef` makes every residual NaN. The survey helper's grouped sums then turn these into zero scores and return zero variance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims 105 tests passed and reproductions (SE 0.0447574 -> 0.0; saturated SE 2.78e-16); unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites lpdid.py:L920-L972 (covers _estimate_survey_sample) but for a different defect (survey metadata/df disagreeing with the realized sample) plus a _subpop_weight rejection P1; rank-deficient unreduced-design sandwich not named. Pytest reported unable to start; direct reproductions claimed; unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The new path discards those residuals and recomputes `response - design @ coef`; one dropped nuisance coefficient therefore makes every residual NaN. PSU aggregation through `DataFrame.groupby().sum()` converts all-NaN group scores into zeros", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Claims a reproduction (SE 0.106265 -> 0.0); unverifiable."}, + + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are CI-inversion P0s in utils.py (non-monotone acceptance set; stopping tolerance); nothing on the storage vcov computed on the full design in _run_wild_bootstrap_inference. Claims 59/59 tests passed and enumeration reproductions; unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are utils.py numerical P0s (zero cluster-score variance; translation-sensitive ties). Claims 59/59 tests passed and reproductions (p 0.03125 -> 0.046875); unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Single P0 on FE omission from bootstrap refits (twfe.py/utils.py; estimators.py:L458-L475/L683-L687 cited only for the absorb path); no rank-deficient-crash finding. Pytest reported unable to start; deterministic enumeration claimed (SE 0.529006 vs 0.808070); unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "P0 on non-monotone test inversion in utils.py plus API P2s; nothing on the wrapper's full-design vcov. Claims 59 passed and a four-cluster reproduction (p=0.125, CI excludes zero); unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on linalg.py guard ordering (saturated NaN guard preceding cluster-count validation)."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Zero-variance two-cluster finding is in utils.py about floating-point residue, not the linalg.py saturated/cluster guard order."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on linalg.py guard ordering."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on linalg.py guard ordering."} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; one P3 wording refinement only. States tests were inspected, not executed."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; no findings. States tests were inspected, not executed."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; one P3 terminology item only."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; no findings. Claims an unseen test run ('2 passed in 0.12s'); unverifiable external-tool claim."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; P3 informational on the documented floor only (correctly treated as documented deviation)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; P3 informational on the documented floor only."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; P3 informational only."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; one P3 comment-wording item ('resolution' vs 'reporting floor') only."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/hits.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/hits.json new file mode 100644 index 00000000..e29ab04c --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/hits.json @@ -0,0 +1,1296 @@ +{ + "artifacts_sha": "83337a6b7e139d7ecade26a6560e47b98b9a55659fd666e0fb7de0c1e47d6ee1", + "branch": "codex-upgrade", + "coverage": { + "covered": 5, + "no_commands": 0, + "total": 44, + "unaudited": 39 + }, + "hits": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "ba67ff0b2483c35d245e722f1ee869ddc222abab396fa2223f0b1cf22e28cd79", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "fbe0ca14a4e84ec93e6bb7f63db70e411626a1260a2f039b541be4e145ba8da6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "line_sha256": "1365c868ea492725fe21d001b7902baec8f3296423f17c6f8cc6af78c903bc41", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "line_sha256": "e05630328fc5b762a1adf5dce28d33ebedd6c16089599bcdfc7e64479ddd90c8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "929998f9f0be84edfc62e6cab0d18ec696fd3e8bd531471c1b6a55849f170594", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "88c291e6fc7330dcefbfd4942a481016f3b55892da3eaa4f6959a52fc934bec3", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "0ed1fd504e498a96c821293812f081cdb75be0780d56dc634d3d1d2272e6ada8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "a0d31b0d9b4639562227c33aeed601b0ae88ba7d7e33ff765506c6f92fdabfe4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "88d6ede8a2c3669439f9f4f295cdcff9dd9699a97ca79236d624341da76a56c3", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "ebf147efc01d7a27bb8aa54d4e7135562399938120d11bc61966f5b47ff6ded7", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "13d525f17cddd6e2c853b3bbf6969e055ac78a9fe785812bad0d6867cd9796f8", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "7c353ca170550157f64cd481017fa4194b24a67dd10404316d0e809fe2aaf294", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "8047958916f1bd9fac3f359080885dc83ee2d53124070cc8eacdad7d2e59e5f6", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "fcfc23e034b810a7e73ab88a3a6b93a1f53a59ffe28f3b9f544b0c6913a588ee", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "164536528e69e549b449b1a733da52f9427ec710fd455bcd1a06ff3a3bdf753a", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "a489db3e13bc2851df1b33593625a57adfb6cbc310bf11ab29877e114f86f888", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "004ac602fd4ac32ea771add8b750f3b21c1f88f11f58bb85a20aa00d19b41eaa", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "3a11404a7c9af50318543bf52419f7cd8bda1a9a0afba8da719b4f34cd157c2b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "3d5f48202de5712f67419ba56c223431215892621542ce8bfe2e4ec3bdbd3805", + "markers": [ + "GIT_LOG_RE", + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "9cbbfc3ec5eaeac2ef5d79f9522eaacd2474dd5375d19622f2b232443b84f0b1", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "7ad8b52811ad0ac251211f131ce53a46785fd75228b672752e243eaa9178c8c4", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "d1241942957af6d6edd53493e76a5f51aa478ec009552f4f2f835d8c2461c5f0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "18697c988f8a1e2a549a9dde613212a5e1c93b83802c8949a04a2a3944f3a7ab", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "39fc6388cd249ae090e2f4f950429d9a845626379c30251243cdb82d391cbf3a", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "09ac6c265c7cfcfcda25d4ab7049a09ad7926986b0e5d880ad92d74cfba4d70d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "826550303c0fa30e2021b243e710949d78fbe7c9fdfaef972d3544eb621db903", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "6bcaa19debe0393ea867c07919fc1e788eef70a11b0b2d625cc17a15c0272daa", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "c53ce2380bb40b5a87653692e558509e720b0495179db28e81f78f00a4d29adf", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "line_sha256": "7985f3f786a9affe5d869d8dd9a4865a29b70bc6eae4518815db9c5278a176b1", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "956513409ee66691ab13f6b7cb2c066db704f1cf9d08d882e2ed9363c7de4f68", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "fabdcf2ee2f9d1513578005eb281f52e16d4cffeabfdbe30d32da90a91240792", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "57284d79692a9528e3354e0cf1c1d32db49667f3a5f3113bb0611430b505b175", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "bf85b8db2755e322b84678090eb620e18f91a3d2f0bf7b3bc637f86857ed8680", + "markers": [ + "--format", + "BARE_SHOW_RE", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "5c8f4f4c87a9c42753e395e638db7722abab77717f6fd70d352894a2d79e40b8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "4583e16d15dd52866e9fc6046c97f17780e0e064730fe68c46e9b6136a5a9daf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "66df9cea2b5ca25462ad284fbe3a2fc8aaa1af03fc7e6e4d5ee4888a1cdfce35", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "618bee36e2b23e658c60b38189e6fc954d19ccab51a9384c6b25a3b2eb7d94e6", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "line_sha256": "81281e75d363e86f0074300f4419ca6cdbe1650b9ceb5434609b89a7cbbb8b22", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "586e8da2cfd58f43f508b6ad5ad1a2eeec3243339e6ee7cd4cfdd960705c4d4d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "0a8b012d835cebdec9804f0f0c083f375f182b16b35f2a3e4bd7905f90001ddf", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "7c55bc0b8bc91713983c4bb88e7d8ba2ac48980e4d2723619b5cff794acffa2d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "6add3493cf88b88d33d1b8da5e81303ac2b6045833d4382a22ac0ade92d17fbb", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "3257e6fb2517fd375a48caa97f66452eed73f5e4b595dde7f2fa517af046c634", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "783f00d1b3b6bf18783c044444a656a38bb0993d1928d03199cf722cd802e890", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "fa3c7f55812385759514638dce23a8787f9cfb4cbdf070136bb481a968c7d679", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "d5a34976424b4ba6c875cfc3af6995d6b1d43256c2dbd85caa201e3fed39e9d8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "eff9518d93079ee77c68d3b4ecbad9bff736529209862ad680c3f04a5e8bde58", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "1abe2343bd9d16c367ba6d19abe22cc0e9ae77930e5d4aff454d0976e2c013bc", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "7c7a45d1e2db3f568f62c54b8577fc476ddd47983418949b66797dad0ffb0299", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "3ff3d5aaf5c25ea00605b4c59d72d5187adbd4a0d6ac51db483c8a906a5197e0", + "markers": [ + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "99994661284b19e074459072bbcb717f86f453c5a43fd3c02a86c7decea5b0a4", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "line_sha256": "57f112133edccc7335d93258d205f119851592d411784e545ef79dde9bdc063d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "092814f4762e221b168be6de8448ea8be9be3525a8a93cfc5542b402395365c5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "65b00241312878e895381ebcda1efdf1180ec590fed67baff0d2eab88047f738", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "2578cb8c6110dd7782654085c1fd1652192dc905677d367baabdaf7c34b909f1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "218d39cf4b3dc3ac8a76278733e09c4e681f0257f4f566e267d059d3295e378c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "8d7c29fcb98d813264ececdca5d60630b78a15326f11323050e7beb48725c384", + "markers": [ + "realpath", + "../", + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "d398c55318a0696cff8136df5e197e451a08994800a411636ddef68ac0840f7f", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "32541076f6de0ff5db6f78f53b3566461a71e56492cf3d1a5b04ffe5968aaf8c", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "c9f53168513e9dbd60fee0356d2d84d116417579ccad8b8fe11dc28dc8505059", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "8d52068797ad3d79c0c72ad777bd71abba9fe6e0f3ceb5c9d268f9b7079f7ee2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "9ad4d2d8d8f11f48ea0e530fe2f28427c819e8a3f8bf0d8c80b4303b03c8c579", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "line_sha256": "acc64a1987afe879d65ebe89da048d83fd0d4d4a74bd4e307c48eed70f19f4a5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "line_sha256": "5ba5c9d1c77071a0d3e6a8097cdb59d622833490b3ce8bcbb2e86f910ecdf2f2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "line_sha256": "b0e62be6ea3552928eb4ff93e18f8ef40b1f245f7bb98cf10ad8185e98e15b23", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "line_sha256": "ce34a702b36b30bc8cfe5e180ef185fa701383df233a8b5de369241887fd2e43", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "f8d619b5ffce261366cd866fe43b7ad8d16afd9bb61293ea6490df162783e5ac", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "969c4611a56051229caa532ccedd3409879acfd6bf5f50c8ccbf997b131efde6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "e02f84d779fef3cc412121e056200441270d4c7c0282aaa694ac0538476080f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "925266052c614ce6caa8e8fdf5305cdb11a3700fc2a7ae2da81192b041af8b92", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "57b7c724f96dcdfbd5a4899b7b3ba35a7a51bf5849a567353902cadbe2918047", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "e041cf87d54205a24e3b80a5ddc51054f3fbdc08ca423de0ba25d310422ec9d3", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "409e8d622678fad1c5b0423ec5cdf1a42566e9b49ae6931c6a67e0e5d15ccb7e", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "818961dd7ad8a0306e34985a111fe09e9d8a103b6c930f027742e3a331c52609", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "aa758cad72e06ccac7e3e16fd2ef857e25766e89c7324d532815c0e2de4058f9", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "b1ef10dcd3e4805d77ebc038db53b290838792fda114c57c723a2f04a32324d6", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "9c8f230f9078c2521476318d08e48a2cf81da13980fe5c5388b8beed2a6a93a5", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "d6faafa2b849dbfdd94e0d420f10b9dea35042ddff42108bd340014353b8e3d9", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "f6d6ada5d364eb31d8f4f601679b9be5ef173926801ca703b0d9fb3f7b296543", + "markers": [ + "git log -1", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "bc0b9ee83008381145eda750a22b87781af78b18a9ca935b1eda2ffe67f2048b", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "9743e03d26f0b9a0abd61ec3a6e1518994a568349eb965f3591eb36572197a11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "343c43bd610e452adedec9e6f2ecf022cac2d51651832a71278568962aefd967", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "93a7ca5fe57b6290bd155a5875e422032282b9753ec45f0c408995913eb5891f", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "44708888a12037b46308d7613f49e252eec924caedee18320647fc29249c4500", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "e28097bf00efe0af7f3f95d819fc0b52e6e9b9e2ae4eb9b6bd3b1ed91edb18c1", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "23d6aa6ba2b09371a91e3707af6aa257fb8b0daaf0559286b118cf5d636d93be", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_LOG_RE", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "6ab5c60727be2612c5ca922aa13dbeeefb03479dcdc31c8d20c3ce6c95a6d334", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "6e213edd28c25b44da2dd5b0d118f73709b4c834c4b1194da5715a35c920f682", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "d1e86d3849c2f32fa4fb227950bd3e0a23f2e682225b46e862e9f9aa28c81cac", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "f4dee83d277459de224b9bf84432ac9c53c3bee2964206fd189f2bcb27bec959", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "8cd7fe2ddb14195928ce63460727287e18221743bd13d407f7d659b21d353c92", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "0107d01e72fc6cc4d452a5841eb82ee11d221110f7dc02fcae9b42b988ae7b38", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "297cc6873c90cb06030ba334f110cf39a6b59b7111ddcb733247b453097b1fa0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "a9eb4fc75c1e08fd1011a341e0f6a9b7e0956adf7ac9acef4c2ec9b390c8b3d9", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "12577dfba14fb44fa2c993180f99b5f8028a091c38c7b1bb61ed0916baffb080", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "25128390b2d10328f8d80a39a6569b3c3f30ff5aaf04e203d51c9203e9ef944c", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "02260987e435619eae51ffd22f8642c26e1373506620faf5ac84daaa3dc909e4", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "548954fce8f8dfec65dda92e87e04e1f7adafe9c29c8500fff5aec5e1065075a", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "c4f07a914a99a85e50a250c1dfd29e27a5f2e4107537c0e7682ca151cd62d166", + "markers": [ + "git log -1", + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + } + ], + "no_commands": [], + "run_ids_sha": "413aae215614d4909ab25c85e66b750b233ec7a731179001c3feb926242e4264", + "unaudited": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "repeat": 1 + } + ], + "vacuous": false +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/reconciled.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/reconciled.json new file mode 100644 index 00000000..6338aeb4 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/reconciled.json @@ -0,0 +1,567 @@ +{ + "bundle_id": "d27906ede74da295", + "rows": [ + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the new dummy block repeats existing `period_*` names. The same happens through the HC2/HC2-BM `absorb=` auto-route. Rank-deficiency handling marks redundant columns NaN, then dictionary construction overwrites earlier coefficients and collapses entries", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites an unverifiable pytest run (TestMPDAbsorbedFERParity: 10 passed, 1 failed) and a fixture reproduction (34 entries vs 38x38 vcov, period_4 = 0.860543). | Claims a pytest run (10 passed, 1 failed) and a reproduced 34-vs-38x38 mismatch; external-tool claims not verifiable here. | Cites an unseen pytest run (TestMPDAbsorbedFERParity: 10 passed, 1 failed) and a reproduced 34-vs-38x38 mismatch; unverifiable external-tool claims." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the new dummy block repeats existing `period_*` names. Dictionary construction then overwrites original coefficients while retaining the full covariance matrix.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites an unverifiable pytest run (10 passed, 1 failed at tests/test_estimators_vcov_type.py:L1324) and fixture reproduction. | Claims a pytest run (10 passed, 1 failed) and reproduced NaN period_4; not verifiable here. | Cites an unseen pytest run (10 passed, 1 failed at tests/test_estimators_vcov_type.py:L1324); unverifiable external-tool claim." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "When `time=\"period\"`, both blocks use names such as `period_4`. The dictionary comprehension silently collapses those duplicate names, while `vcov` retains every matrix column.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; cites an unverifiable direct reproduction (34 keys vs 38x38 VCOV, period_effects[4] = 0.4406). | States pytest could not start; claims a write-free direct reproduction (34 keys vs 38x38 vcov); not verifiable here. | States pytest could not start (no writable temp dir) but claims a write-free direct reproduction (34 keys vs 38x38 VCOV, period_4 NaN); unverifiable." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "the changed loop adds identical columns with identical `period_` names ... the later dictionary comprehension silently collapses duplicate names and allows the dropped `NaN` entries to overwrite the identified period coefficients.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; cites an unverifiable direct reproduction (34 coefficients vs 38x38 VCV). | States pytest could not start; claims direct estimator reproduction; not verifiable here. | Pytest reported as unable to start; claims direct estimator reproduction; unverifiable." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "Using `ddof=1` introduces separate `n/(n\u22121)` corrections for treated and control variance contributions, while the returned influence function remains unchanged. Consequently, aggregation retains the previous variance convention.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites an unverifiable pytest run (TestDRNoCovariateSEUniformity: 2 failed, 2 passed) and a hand-fixture SE comparison (0.721688 vs 0.625). | Claims pytest run (2 failed, 2 passed) and hand-fixture numbers (0.721688 vs 0.625); not verifiable here. | Cites an unseen pytest run (TestDRNoCovariateSEUniformity: 2 failed, 2 passed) and reproduced SE 0.721688 vs 0.625; unverifiable." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "`ddof=1` changes each variance contribution from `sum(residual\u00b2)/n\u00b2` to `sum(residual\u00b2)/(n(n\u22121))`. The returned influence function remains unchanged, so aggregation and bootstrap retain the previous variance convention.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites an unverifiable pytest run (2 failed, 2 passed) and hand-fixture numbers. | Claims pytest run (2 failed, 2 passed); not verifiable here. | Cites an unseen pytest run (2 failed, 2 passed); unverifiable." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE disagrees with both the returned influence function and the documented DRDID variance convention.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). Cites an unverifiable targeted test run (two failures, hand fixture 0.7216878 vs 0.625) and an implausible line reference into DRDID drdid_panel.R (#L1104-L1108). | Graded P0 (expected P1). Claims a targeted test run produced two failures (0.7216878 vs 0.625); not verifiable here. | Rated P0 (expected P1). Claims a targeted test run producing two failures (DR vs reg 1.37%, hand fixture 0.7216878); unverifiable." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "Per-cell SEs, t-statistics, p-values, and confidence intervals become inflated and inconsistent with the influence function used for aggregation/bootstrap.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not initialize in the sandbox; no execution claimed. | States pytest could not initialize in sandbox. | States pytest could not initialize; no reproduction claimed." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same file/function class (saturated derivative boundary convention accepted as documented P3; SATURATED_TOL P0 in continuous_did_bspline.py) but never identifies the J=1 all-zero derivative / ACRT=0 defect. Cites unverifiable test run (127 passed, 8 skipped) and reproductions (SE 0.00091; ATT 3.33333 after rescaling). | Same file (continuous_did_bspline.py L264-289 tolerance merging) and a P3 accepting the lowest-dose forward-difference convention, but never names the J=1 all-zero derivative matrix / ACRT=0 defect. Claims 127 tests passed plus reproductions (SE 0.00091, ATT 3.33333); not verifiable here. | Raised two unrelated P0s (cross-cohort singleton detection; SATURATED_TOL merging doses) with claimed reproductions (SE 0.00091, ATT 3.33333) and a '127 passed' test run; unverifiable. No mention of single-level ACRT=0. | verifier: The review never mentions the single-positive-dose / all-zero derivative-matrix case that makes ACRT = 0 (the ground-truth defect at `saturated_derivative_design_matrix` L319-365); its only touch on the derivative convention is a P3 that ACCEPTS the lowest-dose forward difference as documented. However its P0 on `SATURATED_TOL` is in the right file (continuous_did_bspline.py, adjacent helper lines) and is framed as an estimand change, matching the `estimand_math` class. Right file / right class, wrong defect = partial per the rubric." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Discusses the derivative boundary convention (P3, accepted) and other ACRT inference issues but not the single-positive-dose ACRT=0 defect. Cites unverifiable test run (127 tests), a 100-replication coverage simulation (4%), and analytical-vs-delta SE numbers. | Touches the ACRT boundary convention only as an accepted P3; its P0s (overall ACRT variance, exact-identification bootstrap) are different defects in continuous_did.py. Claims 127 tests, a 4%-coverage simulation and delta-method SE numbers; not verifiable here. | Raised unrelated P0s (overall-ACRT dose-composition variance; bootstrap false significance) with claimed simulations (4% coverage) and a 127-test run; unverifiable. No mention of single-level ACRT=0. | verifier: Neither P0 is in continuous_did_bspline.py and both are variance/inference-class defects (overall-ACRT delta-method omission at continuous_did.py L1561-1566; degenerate-cell bootstrap significance), not the `estimand_math` point-estimate defect. The review affirmatively states the saturated point estimates match the methodology, and its only contact with the derivative convention is an informational P3 located at REGISTRY.md that accepts the lowest-dose convention with 'None required' - it names no defect of the ground-truth class or location, and never mentions the J=1 all-zero operator or ACRT = 0." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites the exact location (continuous_did_bspline.py:L324-L368) but only as the documented gap-scaled/forward-difference convention (P3); does not identify the J=1 zero ACRT. Cites unverifiable direct probes (NaN dose, double survey weighting). | Right symbol (continuous_did_bspline.py:L324-L368) but frames the ACRT convention as a documented P3 (scaled slope / forward difference) and never names the single-level ACRT=0 case. Claims a direct NaN-dose probe and a double-weighting defect; not verifiable here. | Right file/symbol (continuous_did_bspline.py:L324-L368) but only a P3 on the documented forward-difference/gap-scaling convention; does not name the J=1 all-zero derivative / ACRT=0 defect. Other findings (NaN dose P0, double survey weights P1) claim direct probes; unverifiable." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "With `J=1`, the derivative operator is all zeros, so a binary `{0,1}` design returns `ATT=2` but `ACRT=0`. CGBS implicitly sets `d\u2080=0` and the zero-dose treatment effect to zero", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (19 non-slow discrete tests pass) and a weighted reconstruction (SE 1.1528 vs 2.0749). | Also claims a survey double-weighting P0 with reconstructed SE 1.1528 vs 2.0749 and 19 passing tests; not verifiable here. | Also raises a P0 double survey-weighting claim with a reconstructed SE 1.1528 vs 2.0749 and says 19 non-slow tests pass; unverifiable." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags the mean parallel-trends screen at the same location (practitioner.py:L1535-L1541) but as over-stated for CiC ('not necessary'; label as descriptive). Does not identify the ground-truth QDiD defect that under the additive model a mean-trend break is direct evidence against the model (screen understated). Cites unverifiable test run (93 passed) and numeric checks (p\u22488.6e-27). | Attacks the same necessary-but-insufficient mean-PT screen in _handle_cic (L1535-1541), but argues the CiC 'not necessary' direction rather than the ground truth's QDiD 'mean break is direct evidence' framing. Claims 93 tests passed and numeric placebo checks (p\u22488.6e-27); not verifiable here. | Flags the same mean-PT screen at practitioner.py:L1535-L1541 but as 'not necessary for CiC' and recommends demoting it to descriptive; does not name the QDiD mis-framing (additive quantile model => mean-trend break is direct evidence). Claims 93 tests passed and numerical reproductions; unverifiable. | verifier: Ground truth: the QDiD guidance mis-frames the mean-PT check as a necessary-but-insufficient screen while claiming QDiD does not identify off mean PT, whereas under QDiD's additive quantile model a mean-trend break IS direct evidence against the model (the check is UNDER-stated). This review attacks the same screen at the same location but on the opposite axis and for the other estimator: it argues the check is OVER-stated for CiC (not necessary under Athey-Imbens 3.1-3.4) and never mentions QDiD or the additive model. Right file/symbol, different defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same as repeat 0: flags the 'necessary' mean-trend screen for CiC as too strong and recommends treating it as descriptive; misses the QDiD understatement. Cites unverifiable test run (93 passed) and reproduction (p\u22487.7e-148 vs p=1). | Same screen/location; rationale is the CiC 'not necessary' direction, not the QDiD additive-model framing. Claims 93 tests passed and p\u22487.7e-148 reproduction; not verifiable here. | Same as repeat 0: CiC-necessity argument at L1534-L1539, not the QDiD evidential-value mis-framing. Claims 93 tests passed and p-value reproductions; unverifiable. | verifier: Same shape as repeat 0: the finding is the CiC-direction 'not necessary' argument with a scaling-model counterexample, located in the ground-truth line range, but it does not identify the QDiD framing defect (additive model => mean break is direct evidence) and proposes downgrading the check to 'descriptive', which is the opposite of the ground truth's correction. Same location, different defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags the CiC mean-PT screen at L1528-L1558 as not necessary, and explicitly recommends keeping it 'only as a QDiD necessary-but-insufficient screen' - endorsing the ground-truth defective QDiD framing. States pytest could not run. | Same screen/location (L1528-1558), CiC-direction rationale; its fix explicitly proposes keeping the screen 'only as a QDiD necessary-but-insufficient screen', which is the framing the ground truth calls mis-framed for QDiD. States pytest could not run. | Flags the screen at L1528-L1558 for CiC but explicitly endorses keeping it 'only as a QDiD necessary-but-insufficient screen', the framing the ground truth calls wrong. Pytest reported unable to start. | verifier: The review names the screen at the ground-truth location but on the CiC 'not necessary' direction, and its fix explicitly endorses retaining the QDiD 'necessary-but-insufficient screen' framing - i.e., the exact framing the ground truth identifies as wrong (under QDiD's additive model a mean break is direct evidence against the model). Its separate P1 notes the QDiD additive model is omitted from the assumptions prompt (L1391-1426, L1512-1541) but never connects that to the mean-trend screen's evidential value. Same location, different (and partly contradictory) defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags the CiC mean-PT screen (L1528-L1558) as not necessary; raises QDiD additive separability only as an omitted assumption, never linking it to the mean-trend screen's evidential value. States pytest could not start. | Same screen/location, CiC-direction rationale rather than the QDiD framing. States pytest could not start. | Same location, CiC-side argument only; separately notes QDiD additive separability is omitted from the assumptions prompt but never connects it to the mean-trend screen's evidential value. Pytest reported unable to start. | verifier: Again the CiC-direction argument at the right location; the only QDiD-specific remark about the screen is a snippet-hygiene aside ('if the mean screen remains for QDiD, pass an explicit pre-period list'), which treats the QDiD framing as acceptable. The second P1 (QDiD equation (22) additive separability omitted from Steps 2/3 at L1399-1407, L1512-1525) is an incomplete-assumptions-list defect, not the mis-framed evidential status of the mean-trend check. Same location, different defect = partial." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a cluster-count guard. Its finite output passes through `safe_inference()`, producing misleading statistical results.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (390 passed, 3 deselected) and a reproduction (se=0.84003, p=0.85809). | Claims 390 tests passed and a reproduction (se=0.84003, p=0.85809); not verifiable here. | Claims 390 tests passed and a single-cluster reproduction (se=0.84003, p=0.85809); unverifiable." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine that sums cluster scores without rejecting a single contributing cluster. `safe_inference()` subsequently accepts its finite SE.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (437 passed) and reproduction (SE=0.077828, p=7.77e-45); test counts differ from repeat 0 (390) for the same file set. | Claims 437 tests passed and reproduction (SE=0.077828, p=7.77e-45); not verifiable here. | Claims 437 tests passed and a reproduction (SE=0.077828, p=7.77e-45); unverifiable." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "The new path forwards any cluster array without checking the number of effective clusters. With 200 units assigned to one cluster, a direct reproduction returned `se=0.005974`, `t_stat=246.86`, `p_value=0.0`, and a finite CI.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; cites an unverifiable direct reproduction. | States pytest could not start; claims direct API reproduction; not verifiable here. Also raises a P1 CR1-label mismatch not in ground truth. | Pytest reported unable to start; direct reproduction claimed; unverifiable. Also raises a P1 CR1-label mismatch." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "reaches a cluster sandwich that accepts one cluster. There is no between-cluster replication, so cluster-robust inference is undefined, yet `safe_inference()` receives a positive finite SE and emits apparently valid statistics", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; cites unverifiable direct reproductions (att=1.7195, se=0.1221; mixed-type TypeError). | States pytest could not start; claims direct reproduction (att=1.7195, se=0.1221) and a mixed-type TypeError; not verifiable here. | Pytest reported unable to start; direct reproductions claimed (att=1.7195, se=0.1221; mixed-type TypeError); unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "The new guard returns NaN DOFs, but `MultiPeriodDiD` substitutes its shared analytical DOF for affected period coefficients. The warning says inference is suppressed, yet finite p-values and CIs remain. The average-effect path instead returns a finite t-statistic alongside NaN p-value and CI.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (631 passed) and a before/after reproduction table (p 0.061238 -> 0.048684). | Claims 631 tests passed and a before/after p-value table (0.061238 -> 0.048684); not verifiable here. | Claims 631 tests passed and a before/after p-value table (0.061238 -> 0.048684); unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "The new guard returns NaN DOF, but `MultiPeriodDiD` handles those values inconsistently: Per-period inference substitutes the shared analytical DOF and returns finite p-values/CIs. Average inference passes NaN DOF through `safe_inference()`, retaining a finite t-statistic beside NaN p-values/CIs.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (164 passed) and reproductions (p 0.0564 -> 0.0451; t=-2.11546 with NaN p). | Claims 164 tests passed and p-value shift 0.0564 -> 0.0451; not verifiable here. | Claims 164 tests passed and reproductions (p 0.0564 -> 0.0451; t=-2.11546 with NaN p); unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags the same CR2-BM guard (linalg.py:L2104-L2121) for predictor-unit scale dependence only; does not identify that non-finite df passes downstream as finite/fallback inference. Cites unverifiable test run (164 passed) and a 12-cluster check (DOF 7.81631609 -> NaN). | Right file (linalg.py, guard at L2104-L2121) but a different defect: predictor-unit scale dependence of the guard; does not name non-finite df failing open downstream. Claims 164 tests passed and a DOF 7.81631609 -> NaN check; not verifiable here. | Right file and same new CR2-BM guard (linalg.py:L2104-L2121) but a different defect (predictor-unit non-invariance); the non-finite-df fail-open through safe_inference is not named. Claims 164 tests passed and a DOF 7.81631609 -> NaN reproduction; unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same guard scale-dependence P1 at linalg.py:L2104-L2121 plus a dCDH placebo-sign P0; the non-finite-df fail-open is not identified. Cites unverifiable reproductions (DOF 7.6849187 -> NaN; placebo -2 vs +2). | Right file but same scale-invariance defect as repeat 0; does not name the non-finite-df fail-open. Also raises a P0 dCDH placebo-sign claim contrary to the PR's registry note; unverifiable. | Same scale-invariance finding on the guard (different defect); fail-closed NaN propagation not named. Additionally raises a P0 on the dCDH placebo sign that the arm's own repeat 0 classified as a documented P3 deviation; reproduction (DOF 7.6849187 -> NaN) unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags NaN-handling in the new routing path only for the cohort column (nunique() ignoring NaN first_treat, L1931-L1932), not NaN outcomes in _is_unbalanced_panel. Cites unverifiable test run (346 passed), reproductions (SE 0.1013970685 vs 0.1201965009; ATT 2.4883965751 vs 2.4833883573) and a failed R regeneration attempt. | Right file (staggered.py) and NaN-handling class, but the NaN finding is about missing first_treat cohort values (L1931-1932) and row order, not NaN outcomes in _is_unbalanced_panel routing. Claims 346 tests passed and SE/ATT reproductions; not verifiable here. | Right file (staggered.py) and a related missing-value sample-construction finding (NaN first_treat bypasses nunique() validation, row-order dependence) but not the NaN-outcome routing in _is_unbalanced_panel. Claims 346 tests passed, a regression-IF scaling P0 (SE 0.1013970685 vs 0.1201965009) and ATT 2.4883965751 vs 2.4833883573; unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same cohort-NaN finding (L1931-L1937) and RC IF-scale P0; NaN-outcome routing not identified. Cites unverifiable test run (101 passed, 1 deselected) and reproductions. | Same as repeat 0: NaN-cohort/row-order and RC IF-scale findings in staggered.py, not the NaN-outcome routing defect. Claims 101 tests passed and numeric reproductions; not verifiable here. | Same as repeat 0: NaN-cohort finding in staggered.py, not NaN-outcome routing. Claims 101 tests passed and reproductions (SE 0.08178597 vs 0.10571237); unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "A structurally balanced panel with one missing outcome remains on the panel estimator with `used_rc_on_unbalanced_panel=False`, while R drops that row, detects an unbalanced effective sample, and uses the RC estimator.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest unavailable; cites unverifiable read-only Python reproductions (routing flag flip; n_treated=213/250). | States pytest unavailable; claims read-only reproductions; not verifiable here. | Pytest reported unable to start; read-only reproductions claimed (missing outcome -> no routing warning; n_treated=213/250); unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "Balance is checked before complete-case filtering. A rectangular panel containing missing outcomes is treated as balanced, so `allow_unbalanced_panel=True` remains on the panel-difference estimator instead of using the R-compatible RC estimator.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). States pytest could not start; cites unverifiable reproduction (padded-NaN golden e=1 ATT 3.3949106 vs 3.1392963). | Graded P0 (expected P1). States pytest could not start; claims golden reproduction (3.3949106 vs 3.1392963); not verifiable here. | Rated P0 (expected P1). Pytest reported unable to start; claims a padded-NaN golden reproduction (ATT 3.3949106 vs R 3.1392963); unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing `response - design @ coef` therefore makes every residual NaN; PSU aggregation subsequently reduces these to zero.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Describes the symptom as SE collapsing to 0.0 rather than NaN-poisoning, same mechanism and location. Cites unverifiable test run (105 passed) and reproductions (SE 2.50e-16; 0.043541 -> 0.0). | Claims 105 tests passed and reproductions (SE 0.043541 -> 0.0, saturated SE 2.50e-16); not verifiable here. | Reports the symptom as SE collapsing to 0.0 rather than NaN-poisoned inference. Claims 105 tests passed and several reproductions (saturated SE 2.50e-16; SE 0.043541 -> 0.0); unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "`solve_ols()` marks dropped redundant coefficients NaN and returns valid residuals separately. Discarding those residuals and calculating `design @ coef` makes every residual NaN.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (105 passed) and reproductions (SE 2.78e-16; 0.0447574 -> 0.0). | Claims 105 tests passed and reproductions (SE 0.0447574 -> 0.0); not verifiable here. | Claims 105 tests passed and reproductions (SE 0.0447574 -> 0.0; saturated SE 2.78e-16); unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings target the same function range (lpdid.py:L920-L972) but concern survey df/PSU metadata and the _subpop_weight rejection, not the discarded residuals / unreduced design sandwich. States pytest could not start; cites unverifiable direct reproductions. | Right file and even cites L920-L972, but the defects named are survey-metadata/df mismatch and _subpop_weight rejection, not the discarded residuals / unreduced design sandwich. States pytest could not start; claims direct reproductions; not verifiable here. | Cites lpdid.py:L920-L972 (covers _estimate_survey_sample) but for a different defect (survey metadata/df disagreeing with the realized sample) plus a _subpop_weight rejection P1; rank-deficient unreduced-design sandwich not named. Pytest reported unable to start; direct reproductions claimed; unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "The new path discards those residuals and recomputes `response - design @ coef`; one dropped nuisance coefficient therefore makes every residual NaN. PSU aggregation through `DataFrame.groupby().sum()` converts all-NaN group scores into zeros", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites an unverifiable reproduction (SE 0.106265 -> 0.0). | Claims a reproduction (SE 0.106265 -> 0.0); not verifiable here. | Claims a reproduction (SE 0.106265 -> 0.0); unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M1", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings are confined to utils.py CI inversion (disconnected acceptance set, stopping tolerance). Cites unverifiable test run (59/59) and enumeration reproductions. | Findings are CI-inversion defects in utils.py; nothing on the estimators.py storage-vcov crash. Claims 59/59 tests passed and enumeration reproductions; not verifiable here. | Findings are CI-inversion P0s in utils.py (non-monotone acceptance set; stopping tolerance); nothing on the storage vcov computed on the full design in _run_wild_bootstrap_inference. Claims 59/59 tests passed and enumeration reproductions; unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States 'saturated-design ... fixes are present' with no finding; findings are utils.py numerical issues. Cites unverifiable test run (59/59) and reproductions (seed 10; y+1e7). | Findings are numerical-tie/zero-variance defects in utils.py. Claims 59/59 tests passed and seed/translation reproductions; not verifiable here. | Findings are utils.py numerical P0s (zero cluster-score variance; translation-sensitive ties). Claims 59/59 tests passed and reproductions (p 0.03125 -> 0.046875); unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Only P0 is FE omission in bootstrap refits (twfe.py/utils.py, estimators.py L458-L475/L683-L687) - unrelated to the storage-vcov rank-deficient crash. States pytest could not start; cites unverifiable 8-unit enumeration (SE 0.529006 vs 0.808070). | Cites estimators.py absorb preprocessing (L458-L475) and the bootstrap call (L683-L687), but the defect named is FE non-reprojection in TWFE/absorb bootstrap, not the rank-deficient full-design storage vcov raise. States pytest could not start; claims an 8-unit enumeration (SE 0.529006 vs 0.808070); not verifiable here. | Single P0 on FE omission from bootstrap refits (twfe.py/utils.py; estimators.py:L458-L475/L683-L687 cited only for the absorb path); no rank-deficient-crash finding. Pytest reported unable to start; deterministic enumeration claimed (SE 0.529006 vs 0.808070); unverifiable. | verifier: Ground truth b1 is a `rank_deficient_crash`: the wrapper `_run_wild_bootstrap_inference` (estimators.py L780-845) computes the storage vcov via compute_robust_vcov on the ORIGINAL full-dummy design and can raise where the WCR helper (which drops deficient columns) would succeed. The review's P0 is a different class entirely - FE/FWL projection omitted from bootstrap refits, primarily located in twfe.py L380-399 and utils.py L825-879 - and says nothing about rank deficiency, dummy designs, crashes, or the storage-vcov computation. Its only tie to the ground truth is citing the right file at the absorb-preprocessing block and the call site into the wrapper (L683-687), not the wrapper's own lines; under the rubric's literal 'right file ... wrong defect' prong this is the weakest form of partial (file-only adjacency, wrong class, wrong symbol) and counts as missed for every gate." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M2", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings are utils.py inversion monotonicity, dataclass ordering, and fixture gaps. Cites unverifiable test run (59 passed) and a four-cluster reproduction (p=0.125 with CI excluding zero). | Findings are non-monotone inversion (utils.py) and API/dataclass issues. Claims 59 passed and a four-cluster reproduction; not verifiable here. | P0 on non-monotone test inversion in utils.py plus API P2s; nothing on the wrapper's full-design vcov. Claims 59 passed and a four-cluster reproduction (p=0.125, CI excludes zero); unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M1", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No linalg.py finding; saturated guard / cluster-count ordering not discussed. | Mentions 'shared HC1/CR1 handling for saturated designs' as affected but raises no guard-order finding in linalg.py. | No finding on linalg.py guard ordering (saturated NaN guard preceding cluster-count validation)." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No linalg.py finding; saturated-design handling asserted as fixed. | Declares saturated-design fixes 'present'; no finding on the saturated guard preceding cluster-count validation. | Zero-variance two-cluster finding is in utils.py about floating-point residue, not the linalg.py saturated/cluster guard order." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M2", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Saturated designs stated as 'appear resolved'; no guard-order finding. | No finding on linalg.py guard ordering." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M2", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No linalg.py finding. | No finding on linalg.py guard ordering." + } + ], + "negative_assessments": [ + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Only a P3 terminology refinement; tests inspected, not executed. | Only a P3 terminology refinement (allowed). Tests inspected, not executed. | Looks good; one P3 wording refinement only. States tests were inspected, not executed." + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No findings; tests inspected, not executed. | No findings; clean read. | Looks good; no findings. States tests were inspected, not executed." + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Only a P3 terminology finding. | Only a P3 terminology finding (allowed). | Looks good; one P3 terminology item only." + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No findings; cites an unverifiable targeted test run ('2 passed in 0.12s'). | No findings. Claims a targeted test run ('2 passed in 0.12s'); not verifiable here. | Looks good; no findings. Claims an unseen test run ('2 passed in 0.12s'); unverifiable external-tool claim." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "P3 informational note on the documented floor only (allowed); tests inspected, not run. | Only a P3 informational note on the documented floor (allowed). | Looks good; P3 informational on the documented floor only (correctly treated as documented deviation)." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "P3 informational note on the documented floor only (allowed). | Only a P3 informational note on the documented floor (allowed). | Looks good; P3 informational on the documented floor only." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "P3 informational only (allowed). | Only a P3 informational note (allowed). | Looks good; P3 informational only." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "P3 comment-wording finding on 'bootstrap resolution' only (allowed). | Only a P3 comment-wording finding ('resolution' vs 'reporting floor'); allowed. | Looks good; one P3 comment-wording item ('resolution' vs 'reporting floor') only." + } + ] +} \ No newline at end of file diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-definitive.txt b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-definitive.txt new file mode 100644 index 00000000..ed258e02 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-definitive.txt @@ -0,0 +1,3495 @@ +{ + "audit_summary": { + "cells": 44, + "cells_with_hits": 39, + "coverage": { + "covered": 5, + "no_commands": 0, + "total": 44, + "unaudited": 39 + }, + "orientation_only_cells": 5, + "unaudited_cells": 39 + }, + "audit_vacuous": false, + "campaign_reads": { + "cost_estimate_usd": { + "B": { + "input_price_per_1m": 4.0, + "model": "gpt-5.6-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 1.259472 + }, + "E": { + "input_price_per_1m": 10.0, + "model": "gpt-6-astra", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 3.14868 + } + }, + "latency": { + "B": { + "max": 432.1229913751595, + "mean": 208.54861653045836, + "n": 22, + "p95": 343.49848304130137 + }, + "E": { + "max": 391.8861740832217, + "mean": 214.21633708151057, + "n": 22, + "p95": 330.6435491661541 + } + }, + "latency_flag_candidate_p95_over_2x_control": false, + "vendor_lifecycle": "operator_supplied" + }, + "candidate": "E", + "catch_status": { + "s1-coef-dict-collision|s1-coef-dict-collision:b1|B": "reliable", + "s1-coef-dict-collision|s1-coef-dict-collision:b1|E": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|B": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|E": "reliable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|B": "unstable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|E": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|B": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|E": "missed", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|B": "reliable", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|E": "reliable", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|B": "missed", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|E": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|B": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|E": "missed", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|B": "unstable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|E": "reliable", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|E": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|E": "missed" + }, + "contested": [], + "contested_gates": [], + "control": "B", + "definitive": true, + "downgrades": [], + "evidence_gaps": [], + "exposure": { + "first_subdir": "gpt6", + "frozen_head": "225a4afb73106614c37d34b21100eafcdf34be43", + "head_changed": false, + "ledger_entries": 1, + "mechanical": true, + "reason": "first dispatch of the contrast (frozen revision)", + "subdirs": [ + "gpt6" + ], + "surface_diff_stat": "", + "this_subdir": "gpt6", + "timeout_only": false + }, + "fp_candidate": 0, + "fp_control": 0, + "fp_findings": { + "B": [], + "E": [] + }, + "gate_cells": { + "downgrade": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ba67ff0b2483c35d245e722f1ee869ddc222abab396fa2223f0b1cf22e28cd79", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "fbe0ca14a4e84ec93e6bb7f63db70e411626a1260a2f039b541be4e145ba8da6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1365c868ea492725fe21d001b7902baec8f3296423f17c6f8cc6af78c903bc41", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e05630328fc5b762a1adf5dce28d33ebedd6c16089599bcdfc7e64479ddd90c8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "929998f9f0be84edfc62e6cab0d18ec696fd3e8bd531471c1b6a55849f170594", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "88c291e6fc7330dcefbfd4942a481016f3b55892da3eaa4f6959a52fc934bec3", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "0ed1fd504e498a96c821293812f081cdb75be0780d56dc634d3d1d2272e6ada8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a0d31b0d9b4639562227c33aeed601b0ae88ba7d7e33ff765506c6f92fdabfe4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "88d6ede8a2c3669439f9f4f295cdcff9dd9699a97ca79236d624341da76a56c3", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "ebf147efc01d7a27bb8aa54d4e7135562399938120d11bc61966f5b47ff6ded7", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "13d525f17cddd6e2c853b3bbf6969e055ac78a9fe785812bad0d6867cd9796f8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c353ca170550157f64cd481017fa4194b24a67dd10404316d0e809fe2aaf294", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8047958916f1bd9fac3f359080885dc83ee2d53124070cc8eacdad7d2e59e5f6", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "fcfc23e034b810a7e73ab88a3a6b93a1f53a59ffe28f3b9f544b0c6913a588ee", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "164536528e69e549b449b1a733da52f9427ec710fd455bcd1a06ff3a3bdf753a", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a489db3e13bc2851df1b33593625a57adfb6cbc310bf11ab29877e114f86f888", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "004ac602fd4ac32ea771add8b750f3b21c1f88f11f58bb85a20aa00d19b41eaa", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3a11404a7c9af50318543bf52419f7cd8bda1a9a0afba8da719b4f34cd157c2b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3d5f48202de5712f67419ba56c223431215892621542ce8bfe2e4ec3bdbd3805", + "markers": [ + "GIT_LOG_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9cbbfc3ec5eaeac2ef5d79f9522eaacd2474dd5375d19622f2b232443b84f0b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7ad8b52811ad0ac251211f131ce53a46785fd75228b672752e243eaa9178c8c4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1241942957af6d6edd53493e76a5f51aa478ec009552f4f2f835d8c2461c5f0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "18697c988f8a1e2a549a9dde613212a5e1c93b83802c8949a04a2a3944f3a7ab", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "39fc6388cd249ae090e2f4f950429d9a845626379c30251243cdb82d391cbf3a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "09ac6c265c7cfcfcda25d4ab7049a09ad7926986b0e5d880ad92d74cfba4d70d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "826550303c0fa30e2021b243e710949d78fbe7c9fdfaef972d3544eb621db903", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6bcaa19debe0393ea867c07919fc1e788eef70a11b0b2d625cc17a15c0272daa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c53ce2380bb40b5a87653692e558509e720b0495179db28e81f78f00a4d29adf", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7985f3f786a9affe5d869d8dd9a4865a29b70bc6eae4518815db9c5278a176b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "956513409ee66691ab13f6b7cb2c066db704f1cf9d08d882e2ed9363c7de4f68", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "covered", + "hits": [ + { + "line_sha256": "fabdcf2ee2f9d1513578005eb281f52e16d4cffeabfdbe30d32da90a91240792", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "57284d79692a9528e3354e0cf1c1d32db49667f3a5f3113bb0611430b505b175", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bf85b8db2755e322b84678090eb620e18f91a3d2f0bf7b3bc637f86857ed8680", + "markers": [ + "--format", + "BARE_SHOW_RE", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5c8f4f4c87a9c42753e395e638db7722abab77717f6fd70d352894a2d79e40b8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "4583e16d15dd52866e9fc6046c97f17780e0e064730fe68c46e9b6136a5a9daf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "66df9cea2b5ca25462ad284fbe3a2fc8aaa1af03fc7e6e4d5ee4888a1cdfce35", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "618bee36e2b23e658c60b38189e6fc954d19ccab51a9384c6b25a3b2eb7d94e6", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "81281e75d363e86f0074300f4419ca6cdbe1650b9ceb5434609b89a7cbbb8b22", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "586e8da2cfd58f43f508b6ad5ad1a2eeec3243339e6ee7cd4cfdd960705c4d4d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0a8b012d835cebdec9804f0f0c083f375f182b16b35f2a3e4bd7905f90001ddf", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7c55bc0b8bc91713983c4bb88e7d8ba2ac48980e4d2723619b5cff794acffa2d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "6add3493cf88b88d33d1b8da5e81303ac2b6045833d4382a22ac0ade92d17fbb", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3257e6fb2517fd375a48caa97f66452eed73f5e4b595dde7f2fa517af046c634", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "783f00d1b3b6bf18783c044444a656a38bb0993d1928d03199cf722cd802e890", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fa3c7f55812385759514638dce23a8787f9cfb4cbdf070136bb481a968c7d679", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d5a34976424b4ba6c875cfc3af6995d6b1d43256c2dbd85caa201e3fed39e9d8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "eff9518d93079ee77c68d3b4ecbad9bff736529209862ad680c3f04a5e8bde58", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1abe2343bd9d16c367ba6d19abe22cc0e9ae77930e5d4aff454d0976e2c013bc", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c7a45d1e2db3f568f62c54b8577fc476ddd47983418949b66797dad0ffb0299", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3ff3d5aaf5c25ea00605b4c59d72d5187adbd4a0d6ac51db483c8a906a5197e0", + "markers": [ + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "99994661284b19e074459072bbcb717f86f453c5a43fd3c02a86c7decea5b0a4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "57f112133edccc7335d93258d205f119851592d411784e545ef79dde9bdc063d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "092814f4762e221b168be6de8448ea8be9be3525a8a93cfc5542b402395365c5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "65b00241312878e895381ebcda1efdf1180ec590fed67baff0d2eab88047f738", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2578cb8c6110dd7782654085c1fd1652192dc905677d367baabdaf7c34b909f1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "218d39cf4b3dc3ac8a76278733e09c4e681f0257f4f566e267d059d3295e378c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d7c29fcb98d813264ececdca5d60630b78a15326f11323050e7beb48725c384", + "markers": [ + "realpath", + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "d398c55318a0696cff8136df5e197e451a08994800a411636ddef68ac0840f7f", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "32541076f6de0ff5db6f78f53b3566461a71e56492cf3d1a5b04ffe5968aaf8c", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c9f53168513e9dbd60fee0356d2d84d116417579ccad8b8fe11dc28dc8505059", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d52068797ad3d79c0c72ad777bd71abba9fe6e0f3ceb5c9d268f9b7079f7ee2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9ad4d2d8d8f11f48ea0e530fe2f28427c819e8a3f8bf0d8c80b4303b03c8c579", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e02f84d779fef3cc412121e056200441270d4c7c0282aaa694ac0538476080f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "925266052c614ce6caa8e8fdf5305cdb11a3700fc2a7ae2da81192b041af8b92", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "57b7c724f96dcdfbd5a4899b7b3ba35a7a51bf5849a567353902cadbe2918047", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "e041cf87d54205a24e3b80a5ddc51054f3fbdc08ca423de0ba25d310422ec9d3", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "409e8d622678fad1c5b0423ec5cdf1a42566e9b49ae6931c6a67e0e5d15ccb7e", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "818961dd7ad8a0306e34985a111fe09e9d8a103b6c930f027742e3a331c52609", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "aa758cad72e06ccac7e3e16fd2ef857e25766e89c7324d532815c0e2de4058f9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b1ef10dcd3e4805d77ebc038db53b290838792fda114c57c723a2f04a32324d6", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9c8f230f9078c2521476318d08e48a2cf81da13980fe5c5388b8beed2a6a93a5", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d6faafa2b849dbfdd94e0d420f10b9dea35042ddff42108bd340014353b8e3d9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f6d6ada5d364eb31d8f4f601679b9be5ef173926801ca703b0d9fb3f7b296543", + "markers": [ + "git log -1", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "bc0b9ee83008381145eda750a22b87781af78b18a9ca935b1eda2ffe67f2048b", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9743e03d26f0b9a0abd61ec3a6e1518994a568349eb965f3591eb36572197a11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "343c43bd610e452adedec9e6f2ecf022cac2d51651832a71278568962aefd967", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "93a7ca5fe57b6290bd155a5875e422032282b9753ec45f0c408995913eb5891f", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "44708888a12037b46308d7613f49e252eec924caedee18320647fc29249c4500", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e28097bf00efe0af7f3f95d819fc0b52e6e9b9e2ae4eb9b6bd3b1ed91edb18c1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "23d6aa6ba2b09371a91e3707af6aa257fb8b0daaf0559286b118cf5d636d93be", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_LOG_RE", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6ab5c60727be2612c5ca922aa13dbeeefb03479dcdc31c8d20c3ce6c95a6d334", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6e213edd28c25b44da2dd5b0d118f73709b4c834c4b1194da5715a35c920f682", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1e86d3849c2f32fa4fb227950bd3e0a23f2e682225b46e862e9f9aa28c81cac", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f4dee83d277459de224b9bf84432ac9c53c3bee2964206fd189f2bcb27bec959", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8cd7fe2ddb14195928ce63460727287e18221743bd13d407f7d659b21d353c92", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0107d01e72fc6cc4d452a5841eb82ee11d221110f7dc02fcae9b42b988ae7b38", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "297cc6873c90cb06030ba334f110cf39a6b59b7111ddcb733247b453097b1fa0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a9eb4fc75c1e08fd1011a341e0f6a9b7e0956adf7ac9acef4c2ec9b390c8b3d9", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "12577dfba14fb44fa2c993180f99b5f8028a091c38c7b1bb61ed0916baffb080", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "25128390b2d10328f8d80a39a6569b3c3f30ff5aaf04e203d51c9203e9ef944c", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "02260987e435619eae51ffd22f8642c26e1373506620faf5ac84daaa3dc909e4", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "548954fce8f8dfec65dda92e87e04e1f7adafe9c29c8500fff5aec5e1065075a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c4f07a914a99a85e50a250c1dfd29e27a5f2e4107537c0e7682ca151cd62d166", + "markers": [ + "git log -1", + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ], + "fp": [ + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "acc64a1987afe879d65ebe89da048d83fd0d4d4a74bd4e307c48eed70f19f4a5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5ba5c9d1c77071a0d3e6a8097cdb59d622833490b3ce8bcbb2e86f910ecdf2f2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "b0e62be6ea3552928eb4ff93e18f8ef40b1f245f7bb98cf10ad8185e98e15b23", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "ce34a702b36b30bc8cfe5e180ef185fa701383df233a8b5de369241887fd2e43", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f8d619b5ffce261366cd866fe43b7ad8d16afd9bb61293ea6490df162783e5ac", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "969c4611a56051229caa532ccedd3409879acfd6bf5f50c8ccbf997b131efde6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 1 + } + ], + "improvement": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ba67ff0b2483c35d245e722f1ee869ddc222abab396fa2223f0b1cf22e28cd79", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "fbe0ca14a4e84ec93e6bb7f63db70e411626a1260a2f039b541be4e145ba8da6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1365c868ea492725fe21d001b7902baec8f3296423f17c6f8cc6af78c903bc41", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e05630328fc5b762a1adf5dce28d33ebedd6c16089599bcdfc7e64479ddd90c8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "929998f9f0be84edfc62e6cab0d18ec696fd3e8bd531471c1b6a55849f170594", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "88c291e6fc7330dcefbfd4942a481016f3b55892da3eaa4f6959a52fc934bec3", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "0ed1fd504e498a96c821293812f081cdb75be0780d56dc634d3d1d2272e6ada8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a0d31b0d9b4639562227c33aeed601b0ae88ba7d7e33ff765506c6f92fdabfe4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "88d6ede8a2c3669439f9f4f295cdcff9dd9699a97ca79236d624341da76a56c3", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "ebf147efc01d7a27bb8aa54d4e7135562399938120d11bc61966f5b47ff6ded7", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "13d525f17cddd6e2c853b3bbf6969e055ac78a9fe785812bad0d6867cd9796f8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c353ca170550157f64cd481017fa4194b24a67dd10404316d0e809fe2aaf294", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8047958916f1bd9fac3f359080885dc83ee2d53124070cc8eacdad7d2e59e5f6", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "fcfc23e034b810a7e73ab88a3a6b93a1f53a59ffe28f3b9f544b0c6913a588ee", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "164536528e69e549b449b1a733da52f9427ec710fd455bcd1a06ff3a3bdf753a", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a489db3e13bc2851df1b33593625a57adfb6cbc310bf11ab29877e114f86f888", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "004ac602fd4ac32ea771add8b750f3b21c1f88f11f58bb85a20aa00d19b41eaa", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3a11404a7c9af50318543bf52419f7cd8bda1a9a0afba8da719b4f34cd157c2b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3d5f48202de5712f67419ba56c223431215892621542ce8bfe2e4ec3bdbd3805", + "markers": [ + "GIT_LOG_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9cbbfc3ec5eaeac2ef5d79f9522eaacd2474dd5375d19622f2b232443b84f0b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7ad8b52811ad0ac251211f131ce53a46785fd75228b672752e243eaa9178c8c4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1241942957af6d6edd53493e76a5f51aa478ec009552f4f2f835d8c2461c5f0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "18697c988f8a1e2a549a9dde613212a5e1c93b83802c8949a04a2a3944f3a7ab", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "39fc6388cd249ae090e2f4f950429d9a845626379c30251243cdb82d391cbf3a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "09ac6c265c7cfcfcda25d4ab7049a09ad7926986b0e5d880ad92d74cfba4d70d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "826550303c0fa30e2021b243e710949d78fbe7c9fdfaef972d3544eb621db903", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6bcaa19debe0393ea867c07919fc1e788eef70a11b0b2d625cc17a15c0272daa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c53ce2380bb40b5a87653692e558509e720b0495179db28e81f78f00a4d29adf", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7985f3f786a9affe5d869d8dd9a4865a29b70bc6eae4518815db9c5278a176b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "956513409ee66691ab13f6b7cb2c066db704f1cf9d08d882e2ed9363c7de4f68", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "covered", + "hits": [ + { + "line_sha256": "fabdcf2ee2f9d1513578005eb281f52e16d4cffeabfdbe30d32da90a91240792", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "57284d79692a9528e3354e0cf1c1d32db49667f3a5f3113bb0611430b505b175", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bf85b8db2755e322b84678090eb620e18f91a3d2f0bf7b3bc637f86857ed8680", + "markers": [ + "--format", + "BARE_SHOW_RE", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5c8f4f4c87a9c42753e395e638db7722abab77717f6fd70d352894a2d79e40b8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "4583e16d15dd52866e9fc6046c97f17780e0e064730fe68c46e9b6136a5a9daf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "66df9cea2b5ca25462ad284fbe3a2fc8aaa1af03fc7e6e4d5ee4888a1cdfce35", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "618bee36e2b23e658c60b38189e6fc954d19ccab51a9384c6b25a3b2eb7d94e6", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "81281e75d363e86f0074300f4419ca6cdbe1650b9ceb5434609b89a7cbbb8b22", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "586e8da2cfd58f43f508b6ad5ad1a2eeec3243339e6ee7cd4cfdd960705c4d4d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0a8b012d835cebdec9804f0f0c083f375f182b16b35f2a3e4bd7905f90001ddf", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7c55bc0b8bc91713983c4bb88e7d8ba2ac48980e4d2723619b5cff794acffa2d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "6add3493cf88b88d33d1b8da5e81303ac2b6045833d4382a22ac0ade92d17fbb", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3257e6fb2517fd375a48caa97f66452eed73f5e4b595dde7f2fa517af046c634", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "783f00d1b3b6bf18783c044444a656a38bb0993d1928d03199cf722cd802e890", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fa3c7f55812385759514638dce23a8787f9cfb4cbdf070136bb481a968c7d679", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d5a34976424b4ba6c875cfc3af6995d6b1d43256c2dbd85caa201e3fed39e9d8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "eff9518d93079ee77c68d3b4ecbad9bff736529209862ad680c3f04a5e8bde58", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1abe2343bd9d16c367ba6d19abe22cc0e9ae77930e5d4aff454d0976e2c013bc", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c7a45d1e2db3f568f62c54b8577fc476ddd47983418949b66797dad0ffb0299", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3ff3d5aaf5c25ea00605b4c59d72d5187adbd4a0d6ac51db483c8a906a5197e0", + "markers": [ + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "99994661284b19e074459072bbcb717f86f453c5a43fd3c02a86c7decea5b0a4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "57f112133edccc7335d93258d205f119851592d411784e545ef79dde9bdc063d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "092814f4762e221b168be6de8448ea8be9be3525a8a93cfc5542b402395365c5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "65b00241312878e895381ebcda1efdf1180ec590fed67baff0d2eab88047f738", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2578cb8c6110dd7782654085c1fd1652192dc905677d367baabdaf7c34b909f1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "218d39cf4b3dc3ac8a76278733e09c4e681f0257f4f566e267d059d3295e378c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d7c29fcb98d813264ececdca5d60630b78a15326f11323050e7beb48725c384", + "markers": [ + "realpath", + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "d398c55318a0696cff8136df5e197e451a08994800a411636ddef68ac0840f7f", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "32541076f6de0ff5db6f78f53b3566461a71e56492cf3d1a5b04ffe5968aaf8c", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c9f53168513e9dbd60fee0356d2d84d116417579ccad8b8fe11dc28dc8505059", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d52068797ad3d79c0c72ad777bd71abba9fe6e0f3ceb5c9d268f9b7079f7ee2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9ad4d2d8d8f11f48ea0e530fe2f28427c819e8a3f8bf0d8c80b4303b03c8c579", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e02f84d779fef3cc412121e056200441270d4c7c0282aaa694ac0538476080f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "925266052c614ce6caa8e8fdf5305cdb11a3700fc2a7ae2da81192b041af8b92", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "57b7c724f96dcdfbd5a4899b7b3ba35a7a51bf5849a567353902cadbe2918047", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "e041cf87d54205a24e3b80a5ddc51054f3fbdc08ca423de0ba25d310422ec9d3", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "409e8d622678fad1c5b0423ec5cdf1a42566e9b49ae6931c6a67e0e5d15ccb7e", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "818961dd7ad8a0306e34985a111fe09e9d8a103b6c930f027742e3a331c52609", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "aa758cad72e06ccac7e3e16fd2ef857e25766e89c7324d532815c0e2de4058f9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b1ef10dcd3e4805d77ebc038db53b290838792fda114c57c723a2f04a32324d6", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9c8f230f9078c2521476318d08e48a2cf81da13980fe5c5388b8beed2a6a93a5", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d6faafa2b849dbfdd94e0d420f10b9dea35042ddff42108bd340014353b8e3d9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f6d6ada5d364eb31d8f4f601679b9be5ef173926801ca703b0d9fb3f7b296543", + "markers": [ + "git log -1", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "bc0b9ee83008381145eda750a22b87781af78b18a9ca935b1eda2ffe67f2048b", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9743e03d26f0b9a0abd61ec3a6e1518994a568349eb965f3591eb36572197a11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "343c43bd610e452adedec9e6f2ecf022cac2d51651832a71278568962aefd967", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "93a7ca5fe57b6290bd155a5875e422032282b9753ec45f0c408995913eb5891f", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "44708888a12037b46308d7613f49e252eec924caedee18320647fc29249c4500", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e28097bf00efe0af7f3f95d819fc0b52e6e9b9e2ae4eb9b6bd3b1ed91edb18c1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "23d6aa6ba2b09371a91e3707af6aa257fb8b0daaf0559286b118cf5d636d93be", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_LOG_RE", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6ab5c60727be2612c5ca922aa13dbeeefb03479dcdc31c8d20c3ce6c95a6d334", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6e213edd28c25b44da2dd5b0d118f73709b4c834c4b1194da5715a35c920f682", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1e86d3849c2f32fa4fb227950bd3e0a23f2e682225b46e862e9f9aa28c81cac", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f4dee83d277459de224b9bf84432ac9c53c3bee2964206fd189f2bcb27bec959", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8cd7fe2ddb14195928ce63460727287e18221743bd13d407f7d659b21d353c92", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0107d01e72fc6cc4d452a5841eb82ee11d221110f7dc02fcae9b42b988ae7b38", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "297cc6873c90cb06030ba334f110cf39a6b59b7111ddcb733247b453097b1fa0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a9eb4fc75c1e08fd1011a341e0f6a9b7e0956adf7ac9acef4c2ec9b390c8b3d9", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "12577dfba14fb44fa2c993180f99b5f8028a091c38c7b1bb61ed0916baffb080", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "25128390b2d10328f8d80a39a6569b3c3f30ff5aaf04e203d51c9203e9ef944c", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "02260987e435619eae51ffd22f8642c26e1373506620faf5ac84daaa3dc909e4", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "548954fce8f8dfec65dda92e87e04e1f7adafe9c29c8500fff5aec5e1065075a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c4f07a914a99a85e50a250c1dfd29e27a5f2e4107537c0e7682ca151cd62d166", + "markers": [ + "git log -1", + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "acc64a1987afe879d65ebe89da048d83fd0d4d4a74bd4e307c48eed70f19f4a5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5ba5c9d1c77071a0d3e6a8097cdb59d622833490b3ce8bcbb2e86f910ecdf2f2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "b0e62be6ea3552928eb4ff93e18f8ef40b1f245f7bb98cf10ad8185e98e15b23", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "ce34a702b36b30bc8cfe5e180ef185fa701383df233a8b5de369241887fd2e43", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd06c0dd92cb94e866de332d43369ceef620344fe0d5cdbe9fd797208c110d3b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f8d619b5ffce261366cd866fe43b7ad8d16afd9bb61293ea6490df162783e5ac", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "969c4611a56051229caa532ccedd3409879acfd6bf5f50c8ccbf997b131efde6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 1 + } + ], + "regression": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ba67ff0b2483c35d245e722f1ee869ddc222abab396fa2223f0b1cf22e28cd79", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "fbe0ca14a4e84ec93e6bb7f63db70e411626a1260a2f039b541be4e145ba8da6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1365c868ea492725fe21d001b7902baec8f3296423f17c6f8cc6af78c903bc41", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e05630328fc5b762a1adf5dce28d33ebedd6c16089599bcdfc7e64479ddd90c8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "929998f9f0be84edfc62e6cab0d18ec696fd3e8bd531471c1b6a55849f170594", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "88c291e6fc7330dcefbfd4942a481016f3b55892da3eaa4f6959a52fc934bec3", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "0ed1fd504e498a96c821293812f081cdb75be0780d56dc634d3d1d2272e6ada8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a0d31b0d9b4639562227c33aeed601b0ae88ba7d7e33ff765506c6f92fdabfe4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "88d6ede8a2c3669439f9f4f295cdcff9dd9699a97ca79236d624341da76a56c3", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "ebf147efc01d7a27bb8aa54d4e7135562399938120d11bc61966f5b47ff6ded7", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "13d525f17cddd6e2c853b3bbf6969e055ac78a9fe785812bad0d6867cd9796f8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c353ca170550157f64cd481017fa4194b24a67dd10404316d0e809fe2aaf294", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8047958916f1bd9fac3f359080885dc83ee2d53124070cc8eacdad7d2e59e5f6", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "fcfc23e034b810a7e73ab88a3a6b93a1f53a59ffe28f3b9f544b0c6913a588ee", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "164536528e69e549b449b1a733da52f9427ec710fd455bcd1a06ff3a3bdf753a", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a489db3e13bc2851df1b33593625a57adfb6cbc310bf11ab29877e114f86f888", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "004ac602fd4ac32ea771add8b750f3b21c1f88f11f58bb85a20aa00d19b41eaa", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3a11404a7c9af50318543bf52419f7cd8bda1a9a0afba8da719b4f34cd157c2b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3d5f48202de5712f67419ba56c223431215892621542ce8bfe2e4ec3bdbd3805", + "markers": [ + "GIT_LOG_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9cbbfc3ec5eaeac2ef5d79f9522eaacd2474dd5375d19622f2b232443b84f0b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7ad8b52811ad0ac251211f131ce53a46785fd75228b672752e243eaa9178c8c4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1241942957af6d6edd53493e76a5f51aa478ec009552f4f2f835d8c2461c5f0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "18697c988f8a1e2a549a9dde613212a5e1c93b83802c8949a04a2a3944f3a7ab", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "39fc6388cd249ae090e2f4f950429d9a845626379c30251243cdb82d391cbf3a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "09ac6c265c7cfcfcda25d4ab7049a09ad7926986b0e5d880ad92d74cfba4d70d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "826550303c0fa30e2021b243e710949d78fbe7c9fdfaef972d3544eb621db903", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6bcaa19debe0393ea867c07919fc1e788eef70a11b0b2d625cc17a15c0272daa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c53ce2380bb40b5a87653692e558509e720b0495179db28e81f78f00a4d29adf", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7985f3f786a9affe5d869d8dd9a4865a29b70bc6eae4518815db9c5278a176b1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "956513409ee66691ab13f6b7cb2c066db704f1cf9d08d882e2ed9363c7de4f68", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "covered", + "hits": [ + { + "line_sha256": "fabdcf2ee2f9d1513578005eb281f52e16d4cffeabfdbe30d32da90a91240792", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "57284d79692a9528e3354e0cf1c1d32db49667f3a5f3113bb0611430b505b175", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bf85b8db2755e322b84678090eb620e18f91a3d2f0bf7b3bc637f86857ed8680", + "markers": [ + "--format", + "BARE_SHOW_RE", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5c8f4f4c87a9c42753e395e638db7722abab77717f6fd70d352894a2d79e40b8", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "4583e16d15dd52866e9fc6046c97f17780e0e064730fe68c46e9b6136a5a9daf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "66df9cea2b5ca25462ad284fbe3a2fc8aaa1af03fc7e6e4d5ee4888a1cdfce35", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "618bee36e2b23e658c60b38189e6fc954d19ccab51a9384c6b25a3b2eb7d94e6", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "81281e75d363e86f0074300f4419ca6cdbe1650b9ceb5434609b89a7cbbb8b22", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "586e8da2cfd58f43f508b6ad5ad1a2eeec3243339e6ee7cd4cfdd960705c4d4d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0a8b012d835cebdec9804f0f0c083f375f182b16b35f2a3e4bd7905f90001ddf", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7c55bc0b8bc91713983c4bb88e7d8ba2ac48980e4d2723619b5cff794acffa2d", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "covered", + "hits": [ + { + "line_sha256": "6add3493cf88b88d33d1b8da5e81303ac2b6045833d4382a22ac0ade92d17fbb", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "3257e6fb2517fd375a48caa97f66452eed73f5e4b595dde7f2fa517af046c634", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "783f00d1b3b6bf18783c044444a656a38bb0993d1928d03199cf722cd802e890", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fa3c7f55812385759514638dce23a8787f9cfb4cbdf070136bb481a968c7d679", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d5a34976424b4ba6c875cfc3af6995d6b1d43256c2dbd85caa201e3fed39e9d8", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "eff9518d93079ee77c68d3b4ecbad9bff736529209862ad680c3f04a5e8bde58", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1abe2343bd9d16c367ba6d19abe22cc0e9ae77930e5d4aff454d0976e2c013bc", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "7c7a45d1e2db3f568f62c54b8577fc476ddd47983418949b66797dad0ffb0299", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3ff3d5aaf5c25ea00605b4c59d72d5187adbd4a0d6ac51db483c8a906a5197e0", + "markers": [ + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "99994661284b19e074459072bbcb717f86f453c5a43fd3c02a86c7decea5b0a4", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "57f112133edccc7335d93258d205f119851592d411784e545ef79dde9bdc063d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "092814f4762e221b168be6de8448ea8be9be3525a8a93cfc5542b402395365c5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "65b00241312878e895381ebcda1efdf1180ec590fed67baff0d2eab88047f738", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2578cb8c6110dd7782654085c1fd1652192dc905677d367baabdaf7c34b909f1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "218d39cf4b3dc3ac8a76278733e09c4e681f0257f4f566e267d059d3295e378c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d7c29fcb98d813264ececdca5d60630b78a15326f11323050e7beb48725c384", + "markers": [ + "realpath", + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "d398c55318a0696cff8136df5e197e451a08994800a411636ddef68ac0840f7f", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "32541076f6de0ff5db6f78f53b3566461a71e56492cf3d1a5b04ffe5968aaf8c", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c9f53168513e9dbd60fee0356d2d84d116417579ccad8b8fe11dc28dc8505059", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d52068797ad3d79c0c72ad777bd71abba9fe6e0f3ceb5c9d268f9b7079f7ee2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9ad4d2d8d8f11f48ea0e530fe2f28427c819e8a3f8bf0d8c80b4303b03c8c579", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e02f84d779fef3cc412121e056200441270d4c7c0282aaa694ac0538476080f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "925266052c614ce6caa8e8fdf5305cdb11a3700fc2a7ae2da81192b041af8b92", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "57b7c724f96dcdfbd5a4899b7b3ba35a7a51bf5849a567353902cadbe2918047", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "e041cf87d54205a24e3b80a5ddc51054f3fbdc08ca423de0ba25d310422ec9d3", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "409e8d622678fad1c5b0423ec5cdf1a42566e9b49ae6931c6a67e0e5d15ccb7e", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "818961dd7ad8a0306e34985a111fe09e9d8a103b6c930f027742e3a331c52609", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "aa758cad72e06ccac7e3e16fd2ef857e25766e89c7324d532815c0e2de4058f9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b1ef10dcd3e4805d77ebc038db53b290838792fda114c57c723a2f04a32324d6", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "9c8f230f9078c2521476318d08e48a2cf81da13980fe5c5388b8beed2a6a93a5", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d6faafa2b849dbfdd94e0d420f10b9dea35042ddff42108bd340014353b8e3d9", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f6d6ada5d364eb31d8f4f601679b9be5ef173926801ca703b0d9fb3f7b296543", + "markers": [ + "git log -1", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "bc0b9ee83008381145eda750a22b87781af78b18a9ca935b1eda2ffe67f2048b", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "9743e03d26f0b9a0abd61ec3a6e1518994a568349eb965f3591eb36572197a11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "343c43bd610e452adedec9e6f2ecf022cac2d51651832a71278568962aefd967", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "93a7ca5fe57b6290bd155a5875e422032282b9753ec45f0c408995913eb5891f", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "44708888a12037b46308d7613f49e252eec924caedee18320647fc29249c4500", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "e28097bf00efe0af7f3f95d819fc0b52e6e9b9e2ae4eb9b6bd3b1ed91edb18c1", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "23d6aa6ba2b09371a91e3707af6aa257fb8b0daaf0559286b118cf5d636d93be", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "GIT_LOG_RE", + "GIT_DIR_RE", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6ab5c60727be2612c5ca922aa13dbeeefb03479dcdc31c8d20c3ce6c95a6d334", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "6e213edd28c25b44da2dd5b0d118f73709b4c834c4b1194da5715a35c920f682", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d1e86d3849c2f32fa4fb227950bd3e0a23f2e682225b46e862e9f9aa28c81cac", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f4dee83d277459de224b9bf84432ac9c53c3bee2964206fd189f2bcb27bec959", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8cd7fe2ddb14195928ce63460727287e18221743bd13d407f7d659b21d353c92", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "0107d01e72fc6cc4d452a5841eb82ee11d221110f7dc02fcae9b42b988ae7b38", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "297cc6873c90cb06030ba334f110cf39a6b59b7111ddcb733247b453097b1fa0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "a9eb4fc75c1e08fd1011a341e0f6a9b7e0956adf7ac9acef4c2ec9b390c8b3d9", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "12577dfba14fb44fa2c993180f99b5f8028a091c38c7b1bb61ed0916baffb080", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "25128390b2d10328f8d80a39a6569b3c3f30ff5aaf04e203d51c9203e9ef944c", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "E", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "02260987e435619eae51ffd22f8642c26e1373506620faf5ac84daaa3dc909e4", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "548954fce8f8dfec65dda92e87e04e1f7adafe9c29c8500fff5aec5e1065075a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c4f07a914a99a85e50a250c1dfd29e27a5f2e4107537c0e7682ca151cd62d166", + "markers": [ + "git log -1", + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ] + }, + "improvements": [ + { + "bug_id": "s2-nan-dof-fail-closed:b1", + "case": "s2-nan-dof-fail-closed" + } + ], + "informational": { + "B": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + }, + "E": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + } + }, + "judgment_flags": [ + { + "bug": "s2-acrt-single-dose|s2-acrt-single-dose:b1", + "kind": "b_unstable_e_missed" + } + ], + "label": "NO-GO (DEFINITIVE)", + "regressions": [ + { + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "case": "s4-cs-unbalanced-nan-outcome" + } + ], + "verdict": "NO-GO", + "voided_cells": [] +} + +VERDICT: NO-GO (DEFINITIVE) diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-provisional.txt b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-provisional.txt new file mode 100644 index 00000000..c59f3838 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verdict-provisional.txt @@ -0,0 +1,122 @@ +{ + "audit_vacuous": null, + "campaign_reads": { + "cost_estimate_usd": { + "B": { + "input_price_per_1m": 4.0, + "model": "gpt-5.6-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 1.259472 + }, + "E": { + "input_price_per_1m": 10.0, + "model": "gpt-6-astra", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 3.14868 + } + }, + "latency": { + "B": { + "max": 432.1229913751595, + "mean": 208.54861653045836, + "n": 22, + "p95": 343.49848304130137 + }, + "E": { + "max": 391.8861740832217, + "mean": 214.21633708151057, + "n": 22, + "p95": 330.6435491661541 + } + }, + "latency_flag_candidate_p95_over_2x_control": false, + "vendor_lifecycle": "operator_supplied" + }, + "candidate": "E", + "catch_status": { + "s1-coef-dict-collision|s1-coef-dict-collision:b1|B": "reliable", + "s1-coef-dict-collision|s1-coef-dict-collision:b1|E": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|B": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|E": "reliable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|B": "unstable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|E": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|B": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|E": "missed", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|B": "reliable", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|E": "reliable", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|B": "missed", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|E": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|B": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|E": "missed", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|B": "unstable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|E": "reliable", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|E": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|E": "missed" + }, + "contested": [], + "contested_gates": [], + "control": "B", + "definitive": false, + "downgrades": [], + "evidence_gaps": [], + "exposure": { + "first_subdir": "gpt6", + "frozen_head": "225a4afb73106614c37d34b21100eafcdf34be43", + "head_changed": false, + "ledger_entries": 1, + "mechanical": true, + "reason": "first dispatch of the contrast (frozen revision)", + "subdirs": [ + "gpt6" + ], + "surface_diff_stat": "", + "this_subdir": "gpt6", + "timeout_only": false + }, + "fp_candidate": 0, + "fp_control": 0, + "fp_findings": { + "B": [], + "E": [] + }, + "gate_cells": null, + "improvements": [ + { + "bug_id": "s2-nan-dof-fail-closed:b1", + "case": "s2-nan-dof-fail-closed" + } + ], + "informational": { + "B": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + }, + "E": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + } + }, + "judgment_flags": [ + { + "bug": "s2-acrt-single-dose|s2-acrt-single-dose:b1", + "kind": "b_unstable_e_missed" + } + ], + "label": "NO-GO (PROVISIONAL, no session-log audit)", + "regressions": [ + { + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "case": "s4-cs-unbalanced-nan-outcome" + } + ], + "verdict": "NO-GO", + "voided_cells": [] +} + +VERDICT: NO-GO (PROVISIONAL, no session-log audit) diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verifier.json b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verifier.json new file mode 100644 index 00000000..50dc4d67 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt6-grading/verifier.json @@ -0,0 +1,443 @@ +{ + "pass1": { + "pass": 1, + "rulings": [ + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "**Location:** `diff_diff/continuous_did_bspline.py:L264-L270`, `diff_diff/continuous_did_bspline.py:L279-L289`. **Impact:** `SATURATED_TOL = 1e-9` merges genuinely distinct dose levels according to their units. [...] This changes the estimand rather than providing an equivalent numerical implementation.", + "leak": "none", + "leak_evidence": "", + "reason": "The review never mentions the single-positive-dose / all-zero derivative-matrix case that makes ACRT = 0 (the ground-truth defect at `saturated_derivative_design_matrix` L319-365); its only touch on the derivative convention is a P3 that ACCEPTS the lowest-dose forward difference as documented. However its P0 on `SATURATED_TOL` is in the right file (continuous_did_bspline.py, adjacent helper lines) and is framed as an estimand change, matching the `estimand_math` class. Right file / right class, wrong defect = partial per the rubric." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "The saturated point estimates and per-level analytical SEs match the stated methodology. [...] **P3 \u2014 Documented boundary convention accepted.** **Location:** `docs/methodology/REGISTRY.md:L992`. **Impact:** Forward differencing at the lowest dose changes interpretation relative to the paper's convention, but is explicitly documented.", + "leak": "none", + "leak_evidence": "", + "reason": "Neither P0 is in continuous_did_bspline.py and both are variance/inference-class defects (overall-ACRT delta-method omission at continuous_did.py L1561-1566; degenerate-cell bootstrap significance), not the `estimand_math` point-estimate defect. The review affirmatively states the saturated point estimates match the methodology, and its only contact with the derivative convention is an informational P3 located at REGISTRY.md that accepts the lowest-dose convention with 'None required' - it names no defect of the ground-truth class or location, and never mentions the J=1 all-zero operator or ACRT = 0." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "**P1 \u2014 Mean parallel trends is not necessary for CiC.** Location: `diff_diff/practitioner.py:L1535-L1541`, `L1552-L1554`. **Impact:** The guidance can reject a valid CiC design. [...] **Concrete fix:** Remove the \"necessary-but-insufficient\" requirement and the claim that mean-trend rejection undermines CiC time invariance.", + "leak": "none", + "leak_evidence": "", + "reason": "Ground truth: the QDiD guidance mis-frames the mean-PT check as a necessary-but-insufficient screen while claiming QDiD does not identify off mean PT, whereas under QDiD's additive quantile model a mean-trend break IS direct evidence against the model (the check is UNDER-stated). This review attacks the same screen at the same location but on the opposite axis and for the other estimator: it argues the check is OVER-stated for CiC (not necessary under Athey-Imbens 3.1-3.4) and never mentions QDiD or the additive model. Right file/symbol, different defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "**P1 \u2014 Parallel mean trends are not necessary for CiC.** Location: `diff_diff/practitioner.py:L1534-L1539`, `L1551-L1554`. **Impact:** The guidance can lead users to reject valid CiC designs. [...] **Concrete fix:** Remove the necessity claim from both CiC prose and snippet comments. Describe mean trends as a descriptive comparison", + "leak": "none", + "leak_evidence": "", + "reason": "Same shape as repeat 0: the finding is the CiC-direction 'not necessary' argument with a scaling-model counterexample, located in the ground-truth line range, but it does not identify the QDiD framing defect (additive model => mean break is direct evidence) and proposes downgrading the check to 'descriptive', which is the opposite of the ground truth's correction. Same location, different defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "### P1 \u2014 Mean parallel trends is not a necessary CiC screen - **Location:** `diff_diff/practitioner.py:L1528-L1558` [...] **Concrete fix:** Remove `check_parallel_trends()` and \"necessary\" language from the CiC branch. Keep it only as a QDiD necessary-but-insufficient screen, or label it explicitly as a descriptive, non-diagnostic comparison.", + "leak": "none", + "leak_evidence": "", + "reason": "The review names the screen at the ground-truth location but on the CiC 'not necessary' direction, and its fix explicitly endorses retaining the QDiD 'necessary-but-insufficient screen' framing - i.e., the exact framing the ground truth identifies as wrong (under QDiD's additive model a mean break is direct evidence against the model). Its separate P1 notes the QDiD additive model is omitted from the assumptions prompt (L1391-1426, L1512-1541) but never connects that to the mean-trend screen's evidential value. Same location, different (and partly contradictory) defect = partial." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "### P1 \u2014 Mean parallel trends is not a necessary CiC diagnostic **Location:** `diff_diff/practitioner.py:L1528-L1558` [...] **Concrete fix:** For CiC, remove the \"necessary-but-insufficient\" mean-trend screen. [...] If the mean screen remains for QDiD, pass an explicit pre-period list or a pre-treatment-only frame.", + "leak": "none", + "leak_evidence": "", + "reason": "Again the CiC-direction argument at the right location; the only QDiD-specific remark about the screen is a snippet-hygiene aside ('if the mean screen remains for QDiD, pass an explicit pre-period list'), which treats the QDiD framing as acceptable. The second P1 (QDiD equation (22) additive separability omitted from Steps 2/3 at L1399-1407, L1512-1525) is an incomplete-assumptions-list defect, not the mis-framed evidential status of the mean-trend check. Same location, different defect = partial." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "The same risk exists after `absorb=` preprocessing at `diff_diff/estimators.py:L458-L475` before the call at `diff_diff/estimators.py:L683-L687`.", + "leak": "none", + "leak_evidence": "", + "reason": "Ground truth b1 is a `rank_deficient_crash`: the wrapper `_run_wild_bootstrap_inference` (estimators.py L780-845) computes the storage vcov via compute_robust_vcov on the ORIGINAL full-dummy design and can raise where the WCR helper (which drops deficient columns) would succeed. The review's P0 is a different class entirely - FE/FWL projection omitted from bootstrap refits, primarily located in twfe.py L380-399 and utils.py L825-879 - and says nothing about rank deficiency, dummy designs, crashes, or the storage-vcov computation. Its only tie to the ground truth is citing the right file at the absorb-preprocessing block and the call site into the wrapper (L683-687), not the wrapper's own lines; under the rubric's literal 'right file ... wrong defect' prong this is the weakest form of partial (file-only adjacency, wrong class, wrong symbol) and counts as missed for every gate." + } + ] + }, + "pass2": { + "pass": 2, + "leak_rulings": [ + { + "case": "s1-coef-dict-collision", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-coef-dict-collision", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-coef-dict-collision", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-coef-dict-collision", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Cites REGISTRY.md L545-L580 as 'states that the previous DR plug-in was removed' - a repository doc, not an answer-key artifact." + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Cites REGISTRY.md and CHANGELOG.md:L788-L794 for the former O(1/n) deviation - repository files, legitimate." + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "'as the project's methodology also states' (ACRT=ATT for binary) refers to repo docs; no verbatim >=12-word rationale sentence, no git/corpus reference." + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "'This checkout lacks changelog.d/README.md' is a statement about its own worktree, not a leak." + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Uses the phrase 'necessary-but-insufficient' which is the diff's own wording; not a 12-word verbatim rationale sentence." + }, + { + "case": "s2-had-cluster-guard", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "'count distinct cluster IDs on the effective positive-weight support' echoes the case-notes phrase 'positive-weight support' but is short, natural wording and derived from its own reproduction; no confirming reference." + }, + { + "case": "s2-had-cluster-guard", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Only fenced block is a pytest tally ('2 passed in 0.12s'); no markers." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "External R source URLs only (did/DRDID on GitHub); no git commands or corpus references." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Closest overlap in bundle: 'The new path discards those residuals and recomputes `response - design @ coef`' vs rationale 'discards the solver's residuals, recomputes response - design @ coef' - under 12 words and anchored on a code fragment from the diff; not confirming. Review does not mention the case-notes' latent AttributeError/TypeError P2." + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Fenced python block is a self-authored reproduction (numpy/wild_bootstrap_se); '[Newly identified]' and 'previous review's regression cases' refer to prior PR review comments provided as re-review context, not to other refs or the answer key." + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "'The previous review findings ... appear resolved' refers to re-review context (prior AI review comments), not a leak source." + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + } + ], + "grade_corrections": [], + "s3_corrections": [] + } +} \ No newline at end of file diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-VERDICT.md b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-VERDICT.md new file mode 100644 index 00000000..47f0d4ca --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-VERDICT.md @@ -0,0 +1,180 @@ +# GPT-6.1 Sol Reviewer Evaluation - Verdict (2026-10-17): PARITY, accepted as GO + +Pre-registered rule: `DECISION_RULE.md` (the campaign-1 rule re-issued for arm F; +registered at local commit `e1674c0e` on branch `codex-sol61`, the commit this PR +squashes in; the local tag `campaign-gpt61sol` keeps that sha resolvable). Arms: B = +gpt-5.6-sol @ xhigh k=2 (control, re-sampled; production at the time), F = +gpt-6.1-sol @ xhigh k=2 (candidate; its first corpus run was this campaign). 44/44 +runs OK in one attempt (37 minutes at `--max-parallel 4`), 11 cases, zero INFRA_ERROR. +Per the user's direction the campaign ran before any PR was opened; the ledger and +frozen ref record the exposure revision exactly as for a PR-frozen HEAD. + +**Mechanical verdict (`run_eval.py verdict`, DEFINITIVE with `--audit-hits`): `PARITY (DEFINITIVE)`.** +Regressions: 0. FP: F 0 vs B 0. Downgrades: 0. Strict improvements: 0 +(F is UNSTABLE, not RELIABLE, on `s2-nan-dof-fail-closed:b1` where B is MISSED; F is RELIABLE on +`s4-lpdid-survey-unreduced-design:b1` where B is UNSTABLE, not MISSED). Judgment flag: +`s2-acrt-single-dose:b1` B UNSTABLE / F MISSED. Contested gates: none. Precedence reaches step (7). + +**User's final call (2026-10-17): accept PARITY as GO.** Swap production (CI reviewer, +local `/ai-review-local` default, plan-review dual engine, timeout T = 3600) to +gpt-6.1-sol. Reason recorded: no rule-defined regression against two independent +control samples; two favourable catch-status changes (`s4-lpdid-survey-unreduced-design:b1` +B UNSTABLE -> F RELIABLE, `s2-nan-dof-fail-closed:b1` B MISSED -> F UNSTABLE) against one +unfavourable one that the rule reports as a judgment flag (`s2-acrt-single-dose:b1` B +UNSTABLE -> F MISSED); 11 vs 10 caught repeat-cells; half the input price; lower latency; +and gpt-6.1-sol is the current-generation Sol while 5.6-sol is labelled older-generation +by the vendor. (The local pre-PR review of this PR caught an earlier wording of this +paragraph that overstated the catch comparison as "never worse on any bug".) + +## Catch table (10 ground-truth bugs; C = caught, P = partial, M = missed; partial counts as missed for every gate) + +| Bug (stratum) | B r0 / r1 | F r0 / r1 | B status | F status | +|---|---|---|---|---| +| s1-coef-dict-collision:b1 (S1) | C / C | C / C | reliable | reliable | +| s1-cs-dr-plugin-se:b1 (S1) | C / C | C / C | reliable | reliable | +| s2-acrt-single-dose:b1 (S2) | P / C | M / M | unstable | missed | +| s2-cic-practitioner-screen:b1 (S2) | P / P | P / P | missed | missed | +| s2-had-cluster-guard:b1 (S2) | C / C | C / C | reliable | reliable | +| s2-nan-dof-fail-closed:b1 (S2) | P / P | C / P | missed | unstable | +| s4-cs-unbalanced-nan-outcome:b1 (S4) | C / C | C / C | reliable | reliable | +| s4-lpdid-survey-unreduced-design:b1 (S4) | C / P | C / C | unstable | reliable | +| s4-wcr-saturated-guard-order:b1 (S4) | M / M | M / M | missed | missed | +| s4-wcr-saturated-guard-order:b2 (S4) | M / M | M / M | missed | missed | +| **Reliable catches** | | | **4/10** | **5/10** | +| **Caught repeat-cells (of 20)** | | | **10** | **11** | + +Mapping (unblinded after the tables were final): B = M2, F = M1. + +## Pre-registered gates (F vs B) + +1. **Regression NO-GO: PASS.** No bug is RELIABLE for B and MISSED for F. +2. **FP NO-GO: PASS.** 0 vs 0 on both negative controls. +3. **GO: not met.** Zero regressions and zero downgrades, but no strict improvement under the rule's definitions (see above). +4. **PARITY: the outcome.** Inputs for the user's decision are in the informational reads below; the user's call is recorded at the top. + +## Astra side by side (descriptive only; no gate compares E and F) + +Same corpus, same control model, independent control samples. The control's +per-bug status is identical across the two campaigns, which is what makes the +E/F columns comparable. R = reliable, U = unstable, M = missed. + +| Bug | B sol (campaign 1) | E astra | B sol (campaign 2) | F 6.1-sol | +|---|---|---|---|---| +| s1-coef-dict-collision:b1 | R | R | R | R | +| s1-cs-dr-plugin-se:b1 | R | R | R | R | +| s2-acrt-single-dose:b1 | U | M | U | M | +| s2-cic-practitioner-screen:b1 | M | M | M | M | +| s2-had-cluster-guard:b1 | R | R | R | R | +| s2-nan-dof-fail-closed:b1 | M | R | M | U | +| s4-cs-unbalanced-nan-outcome:b1 | R | M | R | R | +| s4-lpdid-survey-unreduced-design:b1 | U | R | U | R | +| s4-wcr-saturated-guard-order:b1 | M | M | M | M | +| s4-wcr-saturated-guard-order:b2 | M | M | M | M | +| Reliable catches | 4 | 5 | 4 | 5 | +| Caught repeat-cells (of 20) | 10 | 10 | 10 | 11 | + +F keeps the `s4-cs-unbalanced-nan-outcome` catch that astra lost, and trades +astra's reliable `s2-nan-dof-fail-closed` catch for an unstable one. + +## Informational reads (non-gating) + +| | B (gpt-5.6-sol) | F (gpt-6.1-sol) | +|---|---|---| +| Latency mean / p95 / max (s) | 213 / 414 / 472 | 181 / 277 / 416 | +| Candidate p95 > 2x control | | False | +| Input price (USD per 1M, standard tier) | 4.00 | 2.00 | +| Initial-prompt cost estimate, whole arm (USD, +/-50%) | 1.26 | 0.63 | +| Reviews with a non-shell event (web search) | 19 of 22 | 21 of 22 | +| Mean shell commands per review | 15.1 | 22.2 | +| Hallucination / unverified external-tool-claim cells | 22 | 22 | +| Confirmed leaks / unconfirmed flags | 0 / 0 | 0 / 0 | + +The hallucination counter is uninformative on this corpus: every review in both +arms cites a test run or a numeric reproduction the blinded graders could not +verify, so each arm scores 22 of 22 cells. The 2026-07 campaign recorded this +as a known Sol-family trait; it now holds for every arm measured. + +Vendor lifecycle (prose, operator-supplied): codex-cli 0.162.1's model catalog +labels gpt-5.6-sol "older generation workhorse"; gpt-6-astra is "frontier +intelligence for the most demanding work" and gpt-6.1-sol "performs near the +Astra level on complex work at lower cost" (developers.openai.com/api/docs/models, +read 2026-10-17). + +## Session-log audit (`run_eval.py audit`, post-unblinding, informational) + +Coverage: covered=4, no_commands=0, unaudited=40, total=44; vacuous=False. +Nearly every review web-searched (codex default features, decision 8), which +classifies the cell `unaudited`; the shell log is still scanned and reported. + +- Cells with at least one hit: 42 of 44; cells whose hits are all orientation-only: 5. Hits per arm: {'B': 52, 'F': 63}. +- Most frequent markers: `repo_root` x36, `-maxdepth` x30, `GIT_LOG_RE` x29, `tools/reviewer-eval/` x14, `BARE_SHOW_RE` x11, `worktrees/` x10, `../` x5, `DOTDOT_RE` x5. The bulk are codex's own AGENTS.md discovery walk up the parent directories (`find -maxdepth`, `../..` loops, absolute paths under the enclosing checkout) and pytest invocations through the enclosing checkout's `.venv`; none of these reads the answer key. +- Hits on ref-reaching markers (`main`/`master`, `origin/`, `refs/`, tags, `rev-list`, `reflog`, `for-each-ref`, `cat-file`, `fsck`, `worktree list`): none. +- S1 cells whose shell log shows `git log` / `git show HEAD` (the synthetic HEAD's commit subject names the injected bug): s1-cs-dr-plugin-se/F/r0, s1-cs-dr-plugin-se/F/r1. Both arms caught both S1 bugs reliably in every repeat regardless, the verifier found no review citing the commit message as its source (confirmation criterion 3), and no gate is touched; recorded for transparency. +- The committed `hits.json` copy replaces each hit's `line` with its sha256 (command lines can carry file contents); the full lines stay in the gitignored original whose sha256 is listed below. + +## Grading record + +Three independent blinded grader subagents (inputs: `comparison.blinded.md`, +`GRADING_RUBRIC.md`, `.github/codex/prompts/pr_review.md`; Bundle ID +`cd55711a27f7080f`; blind labels M2=B, M1=F, mapping unknown to +them). Isolation is by instruction plus check, not a sandbox: the grader prompt +listed the three absolute paths as the only permitted reads and forbade Bash +and every other path, and each returned table was grepped for paths, run ids, +config ids and model names before acceptance (all three clean). The scratch +copies of the rubric and prompt hashed to `blinding.json`'s `rubric_sha256` / +`prompt_sha256`, and the scratch bundle was byte-identical to the run-dir bundle +(`shasum -a 256` equal), before any grader was spawned. + +Inter-grader agreement: 36 of 40 bug cells unanimous; all 8 negative-control +cells unanimous (every list empty); 0 `LEAK?` flags raised. Verifier pass 1 ruled +the disputed cells; verifier pass 2 examined all 44 reviews against the leak +confirmation criteria and confirmed none, with no grade corrections. Reconciled +S3 FP totals: B 0, F 0. + +Committed grading files under `campaigns/2026-10-gpt61sol-grading/` (blind labels; machine paths +redacted to ``, ``, `
`, ``, ``, +``; sha256 campaign-time original -> committed copy): + +- `grader-1.json`: `60bb846d12c49c23865cb89ca490d95fbbcf643a7f01be78cde1369be1100f90` -> `60bb846d12c49c23865cb89ca490d95fbbcf643a7f01be78cde1369be1100f90` +- `grader-2.json`: `15ef73acbf4b94c8ee658cbdabe6ec4946b29681a4ce5fa9f4e001e8124973e4` -> `15ef73acbf4b94c8ee658cbdabe6ec4946b29681a4ce5fa9f4e001e8124973e4` +- `grader-3.json`: `c5d5c64cc372e573b70231c5eca79ac85fc5b30fb6ed75ac25e23060912c92d1` -> `c5d5c64cc372e573b70231c5eca79ac85fc5b30fb6ed75ac25e23060912c92d1` +- `verifier.json`: `c9b70f54f35ba80c7e94ef032b54a30e3fa7098d49ef0b473f0de224eb9a8f3c` -> `c9b70f54f35ba80c7e94ef032b54a30e3fa7098d49ef0b473f0de224eb9a8f3c` +- `reconciled.json`: `91f9272cf4d06ec66378609e23a183707f8e1f661f2c3fac6c55216d6d82b423` -> `91f9272cf4d06ec66378609e23a183707f8e1f661f2c3fac6c55216d6d82b423` +- `hits.json`: `b29397e503bea8d2f76efd4cb5af8ed52658f5c743edd4cfd5b20fc44496e7aa` -> `e05a846eddb8f9d7b994a76b5b042f1c26e77bf191295b8be616bafb7cd3f461` +- `verdict-provisional.txt`: `a57d3c819b6a9eaa0f0bfc2114699d9e7166885e061da2ccb528fbd7e7615a38` -> `a57d3c819b6a9eaa0f0bfc2114699d9e7166885e061da2ccb528fbd7e7615a38` +- `verdict-definitive.txt`: `b931bd6c09cf859f8a4d083199b4090565db2e92fa8d799e478756b1f07d50e9` -> `d1984ac805538ff42d55c9f0a4e4ba7f66ec5e3c3e6346397d587f5c5f2c309c` + +## Grader notes (blinded, consensus) + +Three graders agreed on 44 of 48 cells; the four disputes were the s2-cic cells +(one grader counted the CiC-axis attack on the same screen as a catch; the +verifier ruled all four partial, matching campaign 1). M1 (= F) runs more shell +commands per review and reports pytest as unable to start in the sandbox in +several cells; M2 (= B) cites test counts. Shared blind spots unchanged: both +s4-wcr guard-order bugs and the QDiD framing in s2-cic. + +## Deviations from the plan, recorded + +- Campaign 2 ran on a local, unpushed commit (`e1674c0e`) at the user's direction + ("go straight to the new test runs before opening another PR"); the registration + edits and both verdicts ship together in this PR. The frozen sha is not on any + remote; the local tag and the recorded tree sha are the audit anchors. +- The rehearsal block was re-run on `e1674c0e` concurrently with the campaign's + first minutes (same harness bytes); it passed every expectation, including the + `--control B --candidate F` verdict refusing only at protocol guard (f). + +## Provenance (committed audit record) + +- Frozen HEAD `e1674c0e2a138f63300696f2b967268920c7e07b`, tree `4329ca6dd7eeb2f84b032145a53d54d0214d00a0`, local tag `campaign-gpt61sol` (never pushed), branch `codex-sol61`, subdir `runs/gpt61sol`. +- Manifest: 44 OK run_ids, configs ['B', 'F'], k=2; `repo_dirty`=False, `run_head_shas`=['e1674c0e2a138f63300696f2b967268920c7e07b'], `run_head_missing`=0, `run_dirty_any`=False, `run_marker_mismatch`=0, `run_not_isolated`=0; `verdict` recomputed the aggregates from the sealed artifacts and found no difference. +- `artifacts_sha` (sealed at `run` exit): `37939d7ab22888d9c0d73ef9c97380e1df48b05f7b1df633b542a1fcc162ac61`. +- Exposure ledger: first dispatch of the contrast (frozen revision); mechanical=True; ledger entries for this contrast=1; subdirs=['gpt61sol']. +- Attempts: 1 dispatched attempt, 0 reruns, 0 INFRA_ERROR cells. Run-log records (verbatim): + - `=== run attempt 2026-10-17T19:14:42Z HEAD=e1674c0e2a138f63300696f2b967268920c7e07b` + - `RUN EXIT 0` + - `=== sealed e1674c0e2a138f63300696f2b967268920c7e07b 37939d7ab22888d9c0d73ef9c97380e1df48b05f7b1df633b542a1fcc162ac61` +- Codex CLI 0.162.1 on every arm (runner-asserted); isolated `CODEX_HOME` (credential + `sandbox_mode = "read-only"` + `approval_policy = "never"`, codex default features otherwise; the default feature surface is the gitignored `runs/features-0.162.1.txt`, captured by `codex features list` under the isolated home on 2026-10-16). +- Hashes (sha256) at verdict time: `runs/gpt61sol/comparison.md` `9060951065b60b058f92447c7dd733e20740ea0cad45b37871c113d2a6fb0779`; `runs/gpt61sol/comparison.blinded.md` `2102d1301323b9f9f4480aaa7d454d9ed85f1256c24742aaafbdd3ee4c2fe323`; `runs/gpt61sol/blinding.json` `77465c3f6407ea60a01de6a895861cde874b4e04f7618c5637d05f8ca7489c4c` (bundle_id `cd55711a27f7080f`, run_ids_sha `b02ea16a5f05b574c52d51c9e3a98c410aa66655617212ed2cf9aa7b76cf08c7`, rubric_sha256 `0388e64b0b7defc1ec7a8d4af1950c7a15aa4ad43c62021f69668ba07f4cb6dd`, prompt_sha256 `098ecc7ee3bcfe10fedca4635f277966d717037def7f4162d145de494a217c80`). +- Files at the tag: `DECISION_RULE.md` `1ae891b721b017c866f5df13ed14faa9d5dfdc0d96367ecd4b89085e5ca47f3f`, `GRADING_RUBRIC.md` `0388e64b0b7defc1ec7a8d4af1950c7a15aa4ad43c62021f69668ba07f4cb6dd`, `.github/codex/prompts/pr_review.md` `098ecc7ee3bcfe10fedca4635f277966d717037def7f4162d145de494a217c80`. +- run_ids (44): `s1-coef-dict-collision.B.r0.20aa9cdfe720d4bf`, `s1-coef-dict-collision.B.r1.6052f259b02bb528`, `s1-coef-dict-collision.F.r0.9fc2c9b209283f2b`, `s1-coef-dict-collision.F.r1.85d53c850a62cfd1`, `s1-cs-dr-plugin-se.B.r0.05074433eec6818a`, `s1-cs-dr-plugin-se.B.r1.fc7146a2c93d3504`, `s1-cs-dr-plugin-se.F.r0.412d0ad970061fa3`, `s1-cs-dr-plugin-se.F.r1.fb16eb6b1e275c5f`, `s2-acrt-single-dose.B.r0.4b72df486f75dedb`, `s2-acrt-single-dose.B.r1.a740a6a97984c8b8`, `s2-acrt-single-dose.F.r0.326b80328a8d3a78`, `s2-acrt-single-dose.F.r1.2d2a1be27eaa1060`, `s2-cic-practitioner-screen.B.r0.6b3b0cc6400ca460`, `s2-cic-practitioner-screen.B.r1.ae2a0146b73da067`, `s2-cic-practitioner-screen.F.r0.7f782de8a8275a79`, `s2-cic-practitioner-screen.F.r1.1a700c6a6699c5c7`, `s2-had-cluster-guard.B.r0.84d356cf06268d78`, `s2-had-cluster-guard.B.r1.d93ccde8dcab37d8`, `s2-had-cluster-guard.F.r0.c60f07888d9870a1`, `s2-had-cluster-guard.F.r1.a8d6493c293553e3`, `s2-nan-dof-fail-closed.B.r0.b4a41e147c4f76d4`, `s2-nan-dof-fail-closed.B.r1.f7c068891af83ecb`, `s2-nan-dof-fail-closed.F.r0.a6507d13f73cd5c5`, `s2-nan-dof-fail-closed.F.r1.65b2cab77e0194bc`, `s3-changelog-prose.B.r0.e915ea4986624500`, `s3-changelog-prose.B.r1.6145465e121eb71c`, `s3-changelog-prose.F.r0.5b91fd9013564cf1`, `s3-changelog-prose.F.r1.07863d32d1d8428e`, `s3-wcr-pfloor-comment.B.r0.4701bae6768929fa`, `s3-wcr-pfloor-comment.B.r1.aedad7cc7c2b6cff`, `s3-wcr-pfloor-comment.F.r0.fbd4b310f9cbbb68`, `s3-wcr-pfloor-comment.F.r1.eb55ec4ff8bfaf17`, `s4-cs-unbalanced-nan-outcome.B.r0.210c316a4170c0de`, `s4-cs-unbalanced-nan-outcome.B.r1.e16b6e9f972fe784`, `s4-cs-unbalanced-nan-outcome.F.r0.00de911eed37c6d7`, `s4-cs-unbalanced-nan-outcome.F.r1.c88216ee083a35db`, `s4-lpdid-survey-unreduced-design.B.r0.efaaeba38d503b0d`, `s4-lpdid-survey-unreduced-design.B.r1.23f6433415d5d879`, `s4-lpdid-survey-unreduced-design.F.r0.448caf8e76238e6a`, `s4-lpdid-survey-unreduced-design.F.r1.cd778455b6a48164`, `s4-wcr-saturated-guard-order.B.r0.bde484b74d09a291`, `s4-wcr-saturated-guard-order.B.r1.650895b917c6144c`, `s4-wcr-saturated-guard-order.F.r0.58e5f9ed2f0b3e49`, `s4-wcr-saturated-guard-order.F.r1.a19d8ecb7eb4e035`. + diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-1.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-1.json new file mode 100644 index 00000000..7f6777e9 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-1.json @@ -0,0 +1,151 @@ +{ + "bundle_id": "cd55711a27f7080f", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the expansion introduces duplicate `period_` names. Dictionary construction overwrites original coefficients while retaining every covariance-matrix row.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (1 failed, 10 passed) and numeric reproduction (34 coefs vs 38x38 vcov)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "dummy expansion duplicates the existing `period_` names. Rank-deficiency handling produces dropped coefficients, then dictionary construction overwrites original period coefficients with NaNs and discards entries.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (1 failed, 10 passed)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "when the time column is named `period`, it also duplicates the `period_` keys. The final dictionary comprehension silently collapses those names and can replace valid period coefficients with dropped-column `NaN`s while leaving the full-sized VCV intact.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). Cites unverifiable reproduction (34 keys vs 38x38 VCV)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Expanding `fe == time` creates an identical second block with the same names. When results are converted to a dictionary at `diff_diff/estimators.py:L1968-L1969`, later duplicate keys overwrite the original coefficients.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable focused pytest failure."}, + + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`ddof=1` changes each arm’s variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n*(n-1))`, while the returned influence function remains unchanged. Consequently, cell inference diverges from aggregation and the reg/ipw paths.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). Cites unverifiable test run (2 failed, 2 passed) and numeric reproduction."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Using `ddof=1` introduces group-specific variance multipliers of `n/(n−1)`, while aggregation continues consuming the unchanged influence function. This violates `docs/methodology/REGISTRY.md:L575-L580`, which explicitly requires `sqrt(sum(phi**2))`", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (2 failed, 2 passed) and numeric reproduction (0.721688 vs 0.625)."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE no longer equals the variance of the influence function returned by the same method.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; claims direct invocation of `_doubly_robust()` reproduced 0.7216878365 vs 0.625 (unverifiable)."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The PR silently replaces the registered influence-function variance with a `ddof=1` plug-in variance, inflating per-cell SEs and changing t-statistics, p-values, and confidence intervals.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). Cites unverifiable validation run (2 failed)."}, + + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are dose-share variance omission and survey double-weighting in continuous_did.py; no single-level zero-ACRT finding. Numeric reproductions unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only the dose-frequency variance P0; cites unverifiable test run (127 passed) and 11% coverage simulation."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Touches `continuous_did_bspline.py:L324-L368` only as a P3 documented boundary convention and L243-L270 for non-finite doses; the J==1 all-zero derivative defect is not named. States pytest could not start; reproductions unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "A single positive dose level incorrectly produces `ACRT = 0` with zero SE, contradicting the binary-treatment reduction.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Absolute-path citations under tools/reviewer-eval/runs/.worktrees// are the reviewer's own worktree (exempt per rubric). States pytest could not start."}, + + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The guidance says a mean-trend break undermines CiC’s time-invariance assumption. [...] Remove the necessity claim and “undermines” interpretation for CiC.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the same mis-framed screen at L1534-L1539/L1552-L1558 but argues the CiC side (not necessary) rather than the QDiD side (break is direct evidence). Cites unverifiable 93-test run and simulated p-values."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Remove the necessary mean-parallel-trends requirement for CiC. Location: `diff_diff/practitioner.py:L1534-L1539`, `L1552-L1558`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same location/screen; framing emphasizes CiC. Cites unverifiable 93-test run and numerical reproduction."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new handler says a pre-period mean-trend break undermines CiC time invariance and labels `check_parallel_trends()` a “necessary” screen.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Also notes QDiD's additive model would justify the screen. Cites unverifiable 93-test run and `git diff --check`."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The CiC branch calls a mean-trend comparison “necessary” and says a mean-trend break undermines time invariance. [...] QDiD may retain a mean screen because its additive model implies common mean changes.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": ""}, + + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a minimum-cluster guard. Its finite output passes through `safe_inference()`, producing misleading significance despite having no independent cluster replication.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites had.py:L3954-L3963 (forwarding site) rather than the delegating local-linear function; same defect. Numeric reproduction and suite passes unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance implementation without a minimum-cluster guard. [...] every cluster ID equal to zero, the fit returns `se=0.789637`, `p_value=0.864990`, and finite CI endpoints without any warning.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable 418-test run and reproduction."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_fit_continuous()` forwards clusters without validating the number represented in the active local-polynomial window. `lprobust_vce()` then forms an outer product even when `G=1`, yielding a finite value that passes `safe_inference()`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States full pytest could not start; direct probe numbers unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The newly threaded cluster array reaches `lprobust_vce()` without checking that at least two effective clusters remain. Unlike the mass-point path, which returns `se=NaN` for one cluster, the continuous path accepts the resulting positive numerical SE", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start; end-to-end probe numbers unverifiable. Extra P1 on cr1 label is outside ground truth."}, + + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The new guard can reject treatment contrasts, but `MultiPeriodDiD` substitutes residual DOF for per-period inference. [...] triggers the guard yet reports **p=0.00314**, CI **[1.4364, 3.0767]**. [...] The warning’s promised inference suppression is therefore incomplete.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P1 (expected P0); located at linalg.py:L2138-L2142 with estimators.py consumers. Cites unverifiable 563-test run and reproduction."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same CR2-BM guard in linalg.py but only the scale-invariance defect; explicitly asserts the inference path 'propagates unavailable DOFs consistently to NaN', the opposite of the ground truth. Test counts unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard, scale-invariance defect only; states 'The warning makes this fail-closed rather than silent'. Reproduction unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard, scale-invariance defect only; asserts NaN DOF makes t/p/CI NaN. States pytest could not start; reproduction unverifiable."}, + + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "Balance is checked before filtering missing outcomes or covariates. A complete unit-period grid containing missing outcomes therefore bypasses RC routing.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test runs (133 passed) and golden-data reproduction numbers."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Balance detection counts missing outcomes as observed [...] Balance is determined from identifier pairs before handling missing estimation values.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Additional P0 on NaN first_treat row-order dependence is outside ground truth and unverifiable; cites 181-test run."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_is_unbalanced_panel` counts raw `(unit, time)` keys before outcome, covariate, cluster, or weight validity is resolved. A structurally complete panel with one missing outcome is classified as balanced, making `allow_unbalanced_panel=True` inert.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Graded P0 (expected P1). States pytest could not start; in-process reproduction unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Python currently checks only raw `(unit, time)` row presence. An otherwise balanced panel with one `NaN` outcome therefore leaves `used_rc_on_unbalanced_panel=False` and uses within-cell panel differencing", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test runs (527 passed / 458 passed)."}, + + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` marks dropped, collinear coefficients as NaN and already returns valid residuals for the retained model. Discarding those residuals and calculating `response - design @ coef` makes every residual NaN. PSU aggregation subsequently converts the NaN scores into zero totals, yielding **SE = 0**.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Symptom reported as SE=0 rather than NaN; same defect/location. Extra P0 saturated-sample finding and 117-test run unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing residuals with `design @ coef` propagates those NaNs across every row. The PSU aggregation can then turn these into zero scores and return a zero variance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable reproductions and 117-test run."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped columns with `NaN`, but `_estimate_survey_sample()` computes `design @ coef` without excluding them. The resulting all-NaN residual scores are reduced to a zero survey meat by the PSU `groupby().sum()`, yielding `se=0.0`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable 105-test run."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same symbol (`_estimate_survey_sample`, L921-L962) but a different defect (dropped-PSU design reconstruction); rank-deficient unreduced-design sandwich not named. Reproduction SE numbers and 105-test run unverifiable."}, + + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are CI-inversion monotonicity and numerical tie misclassification in utils.py plus a positional-arg P2; no storage-vcov rank-deficient crash finding. Cites unverifiable 465-test run and golden-data reproductions."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Mentions reviewing 'the shared HC1/CR1 saturated-design guard' but raises no guard-order finding in linalg.py."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same two utils.py P0s; no estimators.py wrapper crash finding. Reproduction script and 59-test run unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": ""}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings: CI inversion P0, absorbed-FE reabsorption P0, stale `_bootstrap_results` P1 (estimators.py L707-L737); none is the storage-vcov rank-deficient crash. States pytest could not start; counterexamples unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": ""}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same CI-inversion and FE-projection P0s; no wrapper crash finding. Cites unverifiable 59-test run and six-cluster reproduction."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": ""} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean (Looks good, no findings). Claims an unverifiable seed-23 geometry reconstruction."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean. Claims an unverifiable independent geometry verification."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean; only two P3 informational entries."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean. Claims an unverifiable fixture geometry check."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean; one P3 informational noting the documented Deviation from R."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean; one P3 informational on the documented floor."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean; one P3 informational on the documented floor."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Clean; no findings."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-2.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-2.json new file mode 100644 index 00000000..acc24ea9 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-2.json @@ -0,0 +1,199 @@ +{ + "bundle_id": "cd55711a27f7080f", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the expansion introduces duplicate `period_` names. Dictionary construction overwrites original coefficients while retaining every covariance-matrix row.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run ('1 failed, 10 passed') and a reproduced '34 coefficient entries versus a 38x38 covariance matrix'; unverifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "dummy expansion duplicates the existing `period_` names. Rank-deficiency handling produces dropped coefficients, then dictionary construction overwrites original period coefficients with NaNs and discards entries.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen pytest run ('1 failed, 10 passed'); unverifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "when the time column is named `period`, it also duplicates the `period_` keys. The final dictionary comprehension silently collapses those names and can replace valid period coefficients with dropped-column `NaN`s while leaving the full-sized VCV intact.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rates it P0 (expected P1). Cites an unseen reproduction ('34 coefficient keys for a 38x38 VCV'); unverifiable here."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Expanding `fe == time` creates an identical second block with the same names. When results are converted to a dictionary at `diff_diff/estimators.py:L1968-L1969`, later duplicate keys overwrite the original coefficients.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen focused pytest failure; unverifiable here."}, + + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`ddof=1` changes each arm's variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n*(n-1))`, while the returned influence function remains unchanged. Consequently, cell inference diverges from aggregation and the reg/ipw paths.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rates it P0 (expected P1). Cites an unseen test run ('2 tests failed, 2 passed') and a numeric reproduction table; unverifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Using `ddof=1` introduces group-specific variance multipliers of `n/(n−1)`, while aggregation continues consuming the unchanged influence function. This violates `docs/methodology/REGISTRY.md:L575-L580`, which explicitly requires `sqrt(sum(phi**2))`", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites an unseen test run ('2 tests failed, 2 passed'); unverifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE no longer equals the variance of the influence function returned by the same method.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not initialize (no writable temp dir) and claims a direct `_doubly_robust()` reproduction (0.7216878365 vs 0.625); unverifiable here."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The PR silently replaces the registered influence-function variance with a `ddof=1` plug-in variance, inflating per-cell SEs and changing t-statistics, p-values, and confidence intervals.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rates it P0 (expected P1). Cites an unseen validation run ('2 failed') with numeric values; unverifiable here."}, + + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Raises two P0s (global ACRT SE omits dose-share uncertainty; survey SEs double-weighted) in continuous_did.py, not the J==1 zero-ACRT estimand defect; treats the boundary convention as documented P3. Numeric reproductions and '19 focused tests' are unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Only P0 is the dose-frequency variance omission; no single-level/zero-ACRT finding. Claims '127 passed, 8 skipped' and an '11% coverage' simulation; unverifiable here."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "CGBS defines the discrete response as the adjacent coefficient difference `β_j − β_{j-1}`. The implementation divides by dose spacing and uses a forward difference at the lowest level at `diff_diff/continuous_did_bspline.py:L324-L368`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Right symbol (saturated_derivative_design_matrix) but only as a P3 'documented convention'; the single-positive-level all-zero ACRT defect is not named. Three P1s on other defects. States pytest could not start; direct reproductions unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "returns an all-zero operator when `J == 1` (`L358-L359`). Consequently, both `acrt_d` and `acrt_glob` are zero, and their analytical SEs are zero through the same operator", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start (no writable temp dir); relied on static inspection plus direct evaluation of the helper. Absolute-path citations are to its own worktree (exempt)."}, + + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "The guidance says a mean-trend break undermines CiC's time-invariance assumption. The Registry's model allows different group distributions of U and arbitrary strictly increasing time transformations; these can produce different mean trends while satisfying identification.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same symbol (_handle_cic mean-trends screen) and same class, but a different defect: argues the screen is NOT necessary for CiC, rather than that the QDiD framing understates a mean-trend break as direct evidence against the additive model. Reproduction p-values and '93 tests passed' unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "CiC permits different untreated mean trends across groups. ... The new guidance would incorrectly undermine a valid design.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same screen location, opposite-direction defect (CiC non-necessity) plus unrelated P1s (covariate conditioning, QDiD additive-separability omission); does not name the QDiD mis-framing of the mean-trend screen. Test-run and numerical claims unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "The new handler says a pre-period mean-trend break undermines CiC time invariance and labels `check_parallel_trends()` a \"necessary\" screen. CiC assumes stable within-group latent distributions and a shared monotone outcome map, not equal changes in group means", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same location/class; flags CiC non-necessity rather than the QDiD understatement. Mentions in passing that QDiD's additive model implies the screen but does not flag the shipped QDiD text. '93 passed' and `git diff --check` runs unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "The CiC branch calls a mean-trend comparison \"necessary\" and says a mean-trend break undermines time invariance. Under CiC, `U` may have different distributions across groups and `h(U,T)` may be nonlinear", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same location/class; defect framed as CiC non-necessity, not the QDiD mis-framing. Also raises a P1 'snippet fails at advertised minimum' and a P1 on identification-range wording; unverifiable."}, + + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a minimum-cluster guard. Its finite output passes through `safe_inference()`, producing misleading significance despite having no independent cluster replication.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites unseen suites passing and a numeric single-cluster reproduction (se=0.287294, p=2.49e-6); unverifiable here. Also claims the survey workaround example raises NotImplementedError 'reproduced'."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance implementation without a minimum-cluster guard. ... Cluster-robust inference is unidentified with one independent cluster.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites '418 tests passed' and a numeric reproduction; unverifiable here."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_fit_continuous()` forwards clusters without validating the number represented in the active local-polynomial window. `lprobust_vce()` then forms an outer product even when `G=1`, yielding a finite value that passes `safe_inference()`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States full pytest could not start (no writable temp dir); direct probe numbers unverifiable here."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The newly threaded cluster array reaches `lprobust_vce()` without checking that at least two effective clusters remain. Unlike the mass-point path, which returns `se=NaN` for one cluster, the continuous path accepts the resulting positive numerical SE", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "States pytest could not start; end-to-end probe (se=0.0161, t=68.28) unverifiable. Extra P1 on `cr1` label vs nprobust NN-cluster meat is unverified."}, + + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The new guard can reject treatment contrasts, but `MultiPeriodDiD` substitutes residual DOF for per-period inference. ... triggers the guard yet reports **p=0.00314**, CI **[1.4364, 3.0767]**. ... The warning's promised inference suppression is therefore incomplete.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rates it P1 (expected P0) and tags it '[Newly identified]' although this is not a re-review. Cites '563 tests passed' and numeric reproductions; unverifiable here."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "Dividing `max|B|` by the contrast norm does not account for predictor units. A small-scale nuisance covariate dominates the batch comparison and falsely marks ordinary treatment contrasts as degenerate.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same guard in linalg.py but a different defect (scale invariance); explicitly asserts the opposite of the bug: 'The affected coefficient-inference path propagates unavailable DOFs consistently to NaN'. '563 tests passed' unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "`scaled_max_B = max_abs_B_arr / ||c||²` only compensates for rescaling a contrast while holding the design fixed. It does not compensate for an equivalent reparameterization that rescales a design column.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same guard region in linalg.py, wrong defect (predictor-unit scale dependence); describes the guard as 'fail-closed rather than silent', missing the non-finite-DOF fail-open. Six-test run and 1e5 reproduction unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "The batch-relative criterion normalizes `max_abs_B` by the contrast norm, but not by the scale of the corresponding design column.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same guard region, wrong defect (scale invariance); does not name the non-finite-DOF fail-open. States pytest could not start; direct reproduction numbers unverifiable."}, + + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "Balance is checked before filtering missing outcomes or covariates. A complete unit-period grid containing missing outcomes therefore bypasses RC routing.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites unseen test counts ('133 passed, 8 skipped') and golden-data reproductions; unverifiable. Extra P1 on the stationarity warning wording unverified."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Balance is determined from identifier pairs before handling missing estimation values. Expanding the golden onto the complete unit-period grid, leaving absent outcomes as NaN and filling cohort labels, changes `used_rc_on_unbalanced_panel` from `True` to `False`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Also raises a P0 (NaN first_treat row-order dependence) with a claimed reproduction; '181 passed' and numbers unverifiable here."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_is_unbalanced_panel` counts raw `(unit, time)` keys before outcome, covariate, cluster, or weight validity is resolved. A structurally complete panel with one missing outcome is classified as balanced, making `allow_unbalanced_panel=True` inert.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Rates it P0 (expected P1). States pytest could not start; in-process reproduction unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Python currently checks only raw `(unit, time)` row presence. An otherwise balanced panel with one `NaN` outcome therefore leaves `used_rc_on_unbalanced_panel=False` and uses within-cell panel differencing", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites unseen test runs ('527 passed, 58 skipped'; '69 tests'; '458 tests'); unverifiable here."}, + + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` marks dropped, collinear coefficients as NaN and already returns valid residuals for the retained model. Discarding those residuals and calculating `response - design @ coef` makes every residual NaN. PSU aggregation subsequently converts the NaN scores into zero totals, yielding **SE = 0**.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Also raises a P0 saturated-sample guard bypass with a claimed reproduction (SE 2.61e-16); '117 tests passed' and numbers unverifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing residuals with `design @ coef` propagates those NaNs across every row. The PSU aggregation can then turn these into zero scores and return a zero variance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Also raises P0 saturated-sample and P1 zero-weight-horizon findings with claimed reproductions; test counts and numbers unverifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped columns with `NaN`, but `_estimate_survey_sample()` computes `design @ coef` without excluding them. The resulting all-NaN residual scores are reduced to a zero survey meat by the PSU `groupby().sum()`, yielding `se=0.0`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Cites '105 tests pass'; constant-covariate reproduction unverifiable here."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "`_estimate_survey_sample()` calls `survey_design.resolve(sample)` after clean-control, composition, and missing-value filtering. If those filters remove every row from a PSU, that PSU disappears from `n_h`, score centering, FPC, and survey-design bookkeeping.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same symbol (_estimate_survey_sample L921-L962) but a different defect (dropped-PSU design reconstruction, rated P0); the unreduced-design/rank-deficient residual recomputation is not named. Pytest count and SE reproduction unverifiable."}, + + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Findings are CI-inversion non-monotonicity and projection tie misclassification (utils.py) plus a positional-arg P2; no finding on the storage vcov computed on the full rank-deficient design. Claims '465 passed' and numeric reproductions; unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Mentions 'the shared HC1/CR1 saturated-design guard' as in scope but raises no guard-order finding."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Two '[Newly identified]' P0s (CI inversion, projection tail counts) in utils.py; nothing on _run_wild_bootstrap_inference. Reproduction code/numbers and test counts unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on the saturated-NaN-guard / cluster-count ordering in _compute_robust_vcov_numpy."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "P0s on CI inversion and FE non-reabsorption, P1 on stale `_bootstrap_results`; none concern the full-design storage vcov crash. States pytest could not start; counterexample numbers unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on guard ordering in linalg.py."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Same two P0 classes as repeat 0 (CI inversion, FE projection); nothing on _run_wild_bootstrap_inference. '59 passed' and six-cluster reproduction unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "No finding on guard ordering in linalg.py."} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Clean (Looks good, no findings). Claims an independent seed-23 geometry reconstruction; unverifiable here."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Clean. Claims independent geometry verification; unverifiable here."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Only two P3-informational entries (within allowed severities)."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Clean; all sections 'Severity: None'. Claims fixture verification of the 80-200 km band; unverifiable here."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Only a P3-informational note on the documented floor (allowed severity, known topic handled correctly)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Only a P3-informational note on the documented floor (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Only a P3-informational note on the documented floor (allowed)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", + "notes": "Clean; no findings at any severity."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-3.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-3.json new file mode 100644 index 00000000..d467a34e --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/grader-3.json @@ -0,0 +1,151 @@ +{ + "bundle_id": "cd55711a27f7080f", + "rows": [ + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the expansion introduces duplicate `period_` names. Dictionary construction overwrites original coefficients while retaining every covariance-matrix row.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable focused test run (1 failed, 10 passed) and reproduction numbers (34 vs 38x38)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "dummy expansion duplicates the existing `period_` names. Rank-deficiency handling produces dropped coefficients, then dictionary construction overwrites original period coefficients with NaNs and discards entries.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable focused test run (1 failed, 10 passed)."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "when the time column is named `period`, it also duplicates the `period_` keys. The final dictionary comprehension silently collapses those names and can replace valid period coefficients with dropped-column `NaN`s while leaving the full-sized VCV intact.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Severity stated P0 (expected P1); cites unverifiable reproduction numbers."}, + {"case": "s1-coef-dict-collision", "bug_id": "s1-coef-dict-collision:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Expanding `fe == time` creates an identical second block with the same names. When results are converted to a dictionary at `diff_diff/estimators.py:L1968-L1969`, later duplicate keys overwrite the original coefficients.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites an unverifiable pytest run of the regression test."}, + + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`ddof=1` changes each arm’s variance contribution from `sum(residual²)/n²` to `sum(residual²)/(n*(n-1))`, while the returned influence function remains unchanged. Consequently, cell inference diverges from aggregation and the reg/ipw paths.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Severity stated P0 (expected P1); cites unverifiable test run (2 failed, 2 passed) and reproduction table."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Using `ddof=1` introduces group-specific variance multipliers of `n/(n−1)`, while aggregation continues consuming the unchanged influence function. This violates `docs/methodology/REGISTRY.md:L575-L580`, which explicitly requires `sqrt(sum(phi**2))`", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable test run (2 failed, 2 passed) and reproduction numbers."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE no longer equals the variance of the influence function returned by the same method.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "States pytest could not start (no writable tmp dir) and claims a direct in-process reproduction (0.7216878365 vs 0.625); unverifiable."}, + {"case": "s1-cs-dr-plugin-se", "bug_id": "s1-cs-dr-plugin-se:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The PR silently replaces the registered influence-function variance with a `ddof=1` plug-in variance, inflating per-cell SEs and changing t-statistics, p-values, and confidence intervals.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Severity stated P0 (expected P1); cites an unverifiable validation run (2 failed)."}, + + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are in continuous_did.py (global ACRT dose-share variance P0, survey double-weighting P0); nothing on the single-level all-zero derivative matrix. Reproduction numbers and test counts unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only the dose-frequency variance P0 in continuous_did.py; cites an unverifiable 127-passed run and an 11% coverage simulation."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Right file: cites continuous_did_bspline.py L324-L368 only as the documented P3 boundary convention and L243-L270 for a NaN/inf dose P1; does not identify the J==1 zero-ACRT estimand defect. Pytest reported unable to start; direct reproductions unverifiable."}, + {"case": "s2-acrt-single-dose", "bug_id": "s2-acrt-single-dose:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "returns an all-zero operator when `J == 1` (`L358-L359`). Consequently, both `acrt_d` and `acrt_glob` are zero, and their analytical SEs are zero through the same operator", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites files by absolute path under tools/reviewer-eval/runs/.worktrees// (exempt per rubric). Pytest reported unable to start; static inspection plus direct helper evaluation claimed."}, + + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the same mean-PT screen text in _handle_cic (L1534-L1539) as mis-framed, but in the opposite direction: argues mean PT is not necessary for CiC and should be dropped, rather than that the QDiD additive model makes a mean-trend break direct evidence against the model. Numerical counterexample (p~2.2e-17) unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same location, same class (mis-framed PT screen) but argues the CiC overstatement, not the QDiD understated-evidence/mis-framed identification defect. Also adds an additive-separability-omission P1. 93-test run unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Flags the mean-PT screen at L1528-L1558 as wrongly 'necessary' for CiC; mentions in the fix that QDiD's additive model implies the screen, but does not name the shipped QDiD framing as the defect. `git diff --check` code span contains no marker. Test count unverifiable."}, + {"case": "s2-cic-practitioner-screen", "bug_id": "s2-cic-practitioner-screen:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same location/class; argues CiC does not imply mean PT and notes 'QDiD may retain a mean screen because its additive model implies common mean changes', but the named defect is the CiC overstatement, not the QDiD mis-framing. Review also contains '[model-redacted]' tokens."}, + + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a minimum-cluster guard. Its finite output passes through `safe_inference()`, producing misleading significance despite having no independent cluster replication.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Location cited is the forwarding site (had.py L3954-L3963) rather than bias_corrected_local_linear; same defect. Reproduction values and 'survey workaround raises NotImplementedError ... was reproduced' are unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance implementation without a minimum-cluster guard.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable 418-test run and reproduction numbers."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_fit_continuous()` forwards clusters without validating the number represented in the active local-polynomial window. `lprobust_vce()` then forms an outer product even when `G=1`, yielding a finite value that passes `safe_inference()`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Pytest reported unable to start; direct probe numbers unverifiable."}, + {"case": "s2-had-cluster-guard", "bug_id": "s2-had-cluster-guard:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "The newly threaded cluster array reaches `lprobust_vce()` without checking that at least two effective clusters remain. Unlike the mass-point path, which returns `se=NaN` for one cluster, the continuous path accepts the resulting positive numerical SE and passes it to `safe_inference()`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Pytest reported unable to start; end-to-end probe numbers unverifiable. Additional P1 on cr1 label mismatch not verifiable from the bundle."}, + + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "The new guard can reject treatment contrasts, but `MultiPeriodDiD` substitutes residual DOF for per-period inference.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Severity stated P1 (expected P0); names the guarded/non-finite DOF yielding finite p/CI at linalg.py L2138-L2142 with downstream consumers. Reproduction values and 563-test run unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M1", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard in linalg.py (L2104-L2121) but a different defect (scale non-invariance); explicitly asserts the inference path 'propagates unavailable DOFs consistently to NaN', i.e. the opposite of the ground truth. Test counts unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 0, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard, wrong defect (predictor-unit scale dependence); states 'The warning makes this fail-closed rather than silent'. Reproduction numbers unverifiable."}, + {"case": "s2-nan-dof-fail-closed", "bug_id": "s2-nan-dof-fail-closed:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same guard, wrong defect (scale invariance); claims downstream fields become NaN. Pytest reported unable to start; reproduction unverifiable."}, + + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "Balance is checked before filtering missing outcomes or covariates. A complete unit-period grid containing missing outcomes therefore bypasses RC routing.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Golden-reindexing reproduction numbers and test counts unverifiable; contains '[model-redacted]' tokens."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "Balance is determined from identifier pairs before handling missing estimation values. Expanding the golden onto the complete unit-period grid, leaving absent outcomes as NaN and filling cohort labels, changes `used_rc_on_unbalanced_panel` from `True` to `False`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Additional P0 (row-order-dependent cohort mass with NaN first_treat) rests on an unverifiable reproduction; 181-test run unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`_is_unbalanced_panel` counts raw `(unit, time)` keys before outcome, covariate, cluster, or weight validity is resolved. A structurally complete panel with one missing outcome is classified as balanced, making `allow_unbalanced_panel=True` inert.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Severity stated P0 (expected P1). Pytest reported unable to start; in-process reproduction unverifiable."}, + {"case": "s4-cs-unbalanced-nan-outcome", "bug_id": "s4-cs-unbalanced-nan-outcome:b1", "arm": "M2", "repeat": 1, "verdict": "caught", + "evidence": "Python currently checks only raw `(unit, time)` row presence. An otherwise balanced panel with one `NaN` outcome therefore leaves `used_rc_on_unbalanced_panel=False` and uses within-cell panel differencing", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Cites unverifiable 527-passed validation."}, + + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` marks dropped, collinear coefficients as NaN and already returns valid residuals for the retained model. Discarding those residuals and calculating `response - design @ coef` makes every residual NaN. PSU aggregation subsequently converts the NaN scores into zero totals, yielding **SE = 0**.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Symptom described as SE=0 rather than NaN but same mechanism/location (L939-L957). Separate P0 (saturated survey sample yields 2.61e-16 SE) and the 0.044757 number rest on unverifiable reproductions."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M1", "repeat": 1, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing residuals with `design @ coef` propagates those NaNs across every row. The PSU aggregation can then turn these into zero scores and return a zero variance.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Separate P0 (saturated sample) and P1 (zero-weight horizon ValueError) rest on unverifiable reproductions; 117-test run unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 0, "verdict": "caught", + "evidence": "`solve_ols()` represents dropped columns with `NaN`, but `_estimate_survey_sample()` computes `design @ coef` without excluding them. The resulting all-NaN residual scores are reduced to a zero survey meat by the PSU `groupby().sum()`, yielding `se=0.0`.", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "105-test run unverifiable."}, + {"case": "s4-lpdid-survey-unreduced-design", "bug_id": "s4-lpdid-survey-unreduced-design:b1", "arm": "M2", "repeat": 1, "verdict": "partial", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Right symbol (_estimate_survey_sample L921-L962) but a different defect: dropped-PSU design reconstruction (P0, SE 0.054641 vs 0.055492, unverifiable), plus metadata/subpopulation findings. Does not name the discarded residuals / unreduced design rank-deficiency issue."}, + + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings are utils.py inversion non-monotonicity and projection tie-count P0s plus an estimators.py constructor-ordering P2; nothing on compute_robust_vcov over the unreduced design in _run_wild_bootstrap_inference. Reproductions and 465-passed run unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Only utils.py P0s (inversion, projection). Fenced python reproduction block contains no markers. Numerical claims unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Findings: utils.py inversion P0, twfe/estimators absorbed-FE reabsorption P0, stale _bootstrap_results P1 (estimators.py L707-L737); none about the rank-deficient storage-vcov crash. Pytest reported unable to start; counterexamples unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b1", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Same two P0s as repeat 0 (inversion, absorbed-FE refit); nothing on _run_wild_bootstrap_inference's full-design vcov. Reproduction numbers unverifiable."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Mentions 'the shared HC1/CR1 saturated-design guard' as an affected area but raises no finding on guard ordering vs cluster-count validation."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M1", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No finding in linalg.py _compute_robust_vcov_numpy."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 0, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No finding in linalg.py."}, + {"case": "s4-wcr-saturated-guard-order", "bug_id": "s4-wcr-saturated-guard-order:b2", "arm": "M2", "repeat": 1, "verdict": "missed", + "evidence": "", + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "No finding in linalg.py."} + ], + "negative_assessments": [ + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good, no findings. Claims to have reconstructed seed-23 geometry (no units 80-200 km) - unverifiable."}, + {"case": "s3-changelog-prose", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good, no findings. Geometry verification claim unverifiable."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; two P3 informational entries only (within allowed severities)."}, + {"case": "s3-changelog-prose", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good, all sections 'Severity: None'. Fixture verification claim (zero units 80-200 km) unverifiable."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; single P3 informational on the documented floor (allowed, known topic)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M1", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; single P3 informational on the documented floor (allowed, known topic)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 0, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good; single P3 informational on the documented floor (allowed, known topic)."}, + {"case": "s3-wcr-pfloor-comment", "arm": "M2", "repeat": 1, "findings": [], + "leak": "none", "grader_flag": "none", "leak_evidence": "", "notes": "Looks good, no findings."} + ] +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/hits.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/hits.json new file mode 100644 index 00000000..a8879b97 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/hits.json @@ -0,0 +1,1409 @@ +{ + "artifacts_sha": "37939d7ab22888d9c0d73ef9c97380e1df48b05f7b1df633b542a1fcc162ac61", + "branch": "codex-sol61", + "coverage": { + "covered": 4, + "no_commands": 0, + "total": 44, + "unaudited": 40 + }, + "hits": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "4581c043835af761f02f10ddacc67cd338cd7d9cc050ab41ad1d76eca63b8099", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "43875f510f7d4d2355ea7a1bbd06ce3a393d3cb5e31bb377b6bbad40b0d023c2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "6db06274942ca668070ccee3f005da1d2cc2f9cb908e4b2faeee02266b28aee1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "bc5e0932da413124af878ef15d712049afc6aedf5ca58d90111baabe01f28b9c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "line_sha256": "de9ffa6055e0a2383967d3cbf194e3677e100d3b8cd2a94e1e19af29c83dafb9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "line_sha256": "4a23000128509bdf1a9dff7da14e195599c24a286e88f211089636ff81dcb463", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "line_sha256": "c01dd2b41980c065e0a777d9d15a8e4e0e3ff0b3d64fcbcb1a9bc36cda03d565", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "line_sha256": "c00612992135e362b61ef9aa2c8840ca720851d362b5ca14ad59480746c78254", + "markers": [ + "../" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "be903554134a616a4f51158f58b4e3a6fa7c63ee7f792338eee51d14d562c019", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "e7d59ebeae9764faf8602a13d1f56dcd3d82b82bd26e3e21c6aa7836f5a4e8ad", + "markers": [ + "HEAD~" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "bff3b5f05108e0e3bead6fdc57235ca733327489684a089019e7608712b2737b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "e1973c0cba163fa32110a25fe567930fcdfe33c7032958599913efd0311d7898", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "642d39c1c8e8172f91e155f4ecaf711603bfadc4c61543df8cb6bfec3ac22c9d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "line_sha256": "436ee100fd57903cf9e6eb66780804d032df8908bf0c33dbd78e5d69de1078b6", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "6bc9c7174b8708e37833c5fa3c06a6badf2620f825274205ddbbf27b6a29b4b9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "8c003bd8814eed70d32562a28c791afbcfafcd9c8c284e05718b2a3f04ea0356", + "markers": [ + "rev-parse", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "972096a15295e925f2e3cf87eee726b50fabd9ef05a627c24781c89d1daa86f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "679da91f949a511dbee6113010bc53b87533af6163b5fef3d71638f2803a5231", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "line_sha256": "1f0d006bf9e7caf5e7a47f03a5c6d89ee49e952bc50921c4f77293f757612b65", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "line_sha256": "0fc03c2824d3dde62aefd151f0485e213a06e59d17821d84502a586763519ec5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "line_sha256": "814f0f33cb379d7ac38340497f36e9195ce02d3f2de9d7ac57219278648acdea", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "line_sha256": "031a2d49b3a82bc591181eee88488e1237d89f86b070e9914026376088a978cb", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "line_sha256": "b945e48385ff5ea9cf694e0f3f275a664d5d6ffeb4a7a83d6a0524836cd320ed", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "aee739303916771552df815506d15b8966cc3848ba41156b5a18e9fa1c8d3f11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "7db0895e590ea7ecf2f5f4e4a960e883e47b14a532948b64fdc341d0aad21d62", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "9eb4e535ccab0a3db7afd4b81945d6618e017f48173ee821d15d2993d7cea36d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "line_sha256": "79ef89a6f757830872163acb657afe333343b38d5f7bcb2a0f52ad5b28d6cd59", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "line_sha256": "06c4414f2ba3ca58ffdd8b68e6594dc712a6de0c1a18d696fa36f05562a084fc", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "cc391d5c7315c5ee5895e7c4648ed7cab84686a0a577616f1a2896b1e043adf3", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "60cce41f7825e3a471ff0276a7d6ce8c8d23d7452936dbda837dcc6fcddc638a", + "markers": [ + "-rn" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "27eeeb0397a46f1e0fc0f7cfd2e27fa5cb71b030d47d401629460ab83f82c659", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "dd849ca4a37bf4802b2e980a5f5a1ecec1bde2809fa5e2c9ae4462b5537eebaa", + "markers": [ + "git show HEAD", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "2073d2e30017c6c0d3f45fac4c79b047ee85b22f738dadabe82b8da10e27f140", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "line_sha256": "eefc1eb47dddd5e0c76f397a720c93e6cbabb0eb8cb191e267fffff59ca138a6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "line_sha256": "f8c2aaaaa9ad31c0c8496a39705bcec59fad5c1acdc87d8ca1280aec8d62d185", + "markers": [ + "HEAD~", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "line_sha256": "fd9812cb38239143c14da974259380a76a5dcca7f30f7472aea3c5c728a76327", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "line_sha256": "3a48f5782a5b99f840d78e9108b5779b3f7e4da65719a8da0e0ee11b0b7e2c6e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "3b298a3e9116ee16ca656adb3788f5b1c622968e55b0bbac384611c8780a06c2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "27b09e81480add37b82ec1e90fc091c3aa7d6531d14120dcb66356471ff905a0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "3c365ebde7a7b9516774687d8c1f6db23e08fb4ef22403fd9b096d6e2e010e40", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "236a4cdb8f444afad1d920dc575956f38890768ce1dccfd95db2764b1b3c5f0f", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "ffef8182975bcf1a8bf68c942c8ad74e6a0bf2d8955cbbc75ef83ee14766277a", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "d165c839fb89e0cccb1f133424487a1765beca86386f68a46c6876408d0eef20", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "60b823fc440ff4b309641567a2e51c65208cf2cbe0aca5ac87644632851e1926", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "932e76e69ddf9c66793543e58bf8f76ff4d571a48908b481b48a7bc2a146fed4", + "markers": [ + "../", + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "78b7d8375dacaa30cbd6915edd3f3d7ba7217c66d9c5969f31b557cc53dde711", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "831c08aa7dbe60925e452201bba20cc6d9ddcfe1baf738f2b7f8fe91cf183172", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "5d5ce7c4dd128ba5551eea81c15d63739a52d7b0c08a6b2f1d8df314a8f37667", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "f3d859fc65dc144719341882dd788a5a09f6372461936fa15a961f67ea360f41", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "3330e7f41e67f7334081aabf48c0e2527f04cad55ba2348b5bd8edf5f33e270a", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "406fc2aa4c793871d253d1f186bdb7949666edd60ddd2794bae18f4c50fea2b8", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "line_sha256": "20fc9ee9abd090f617c98093c06224ff938b2337f4bd9ee2b65732cf21bf7878", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "line_sha256": "11fb0d28296fdc35b9992eb7c7e9a64a7be354b37b48a97e2b4387b7a6f8d428", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "line_sha256": "9d9267900b68e4d3277b6fe63564bd20d8fd0b4054e91af0fce84553b8b8ab6b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "line_sha256": "67acd7e054b4d1a326b500669d0dca3bbb33d566fb88422eb062bb2eb911283e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "line_sha256": "7c15957270550fa7831ee2a625e1e165c4e3f37208647d5f63caf02af96ff544", + "markers": [ + "../" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "919e32e4f394efcfec8bb0fc00a5a96b41d8ae767ae8445762aeb7214b474251", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "b238f1c5111829ad160b0528335ac56f63694fc4600a10bf235d6169e6d705bf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "b80ef6d46b17aedd3f66b286ffcb7a7ff32eb8f85c8c28ba11cd2932813b76d7", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "e8880b032890ee3cb48ab4fce5bcbea666d62cdb0ec285070bf9aa44001388be", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "d7ac19a1fdc5ba5440fe3d3dc19c1f8cb26a258287a27706a50c093a3cffb870", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "line_sha256": "09c36ea49b6375dc55c36b4f9a1f5574d636e2c75d7340946439f65098b73268", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "9356a1d8a8d8d5d98bbffc08bbd9261253ba9bb944871c4a590fdcf6673e8139", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "1382c9aedc38269dd3adb840c0535836d775d252b85ca0e181e9f45bd6ae46fe", + "markers": [ + "rev-parse", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "ad506db11fe1e5f12d1870c4e03120cf424bbb7dcdf7a59727f20616835b6ebe", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE", + "DOTDOT_RE", + "GIT_DIR_RE" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "47e685f03a4c622c078fb93098ef698a64424e0d653f537c30689f514889313b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "bc3c8e0736af923ddb6352a221a14b2aa2799e978a4023dd1e3677bc8479c7d5", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "dedf87ab4d5a344a6fa81f1744b6ed4020c4a357edb7c33f26260c69007346f6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "e32548de94b9b6a582b9f14a59eded3f913db7668802f7044f6b50d3e89ea872", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "e3230c6c5073a314519f85a367485585743dc722a2fdc4838d48eb388d027ef0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "ff969993a2523cfc41df3f56bc748392aff5654c077c7719d129654681b76ec8", + "markers": [ + "../", + "DOTDOT_RE", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "line_sha256": "2c5e45dbe5528cfed07b9461e75b4d25f7f66a5b841b28072ab99c1e343ba8b3", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "decca74bfa41b6dbec7c7b4781d28d6f236093a4b8dc0635abcd619947717c40", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "f70a9944f4ebac984d6a738d3e3faacd74f8f32e5e3147c54bbb1a26f6e69f92", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "eee6b44b32b3942f6bf00eb8c4423d17fe2b257ebeaf810c4eece5bff8a1b1ee", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "ab52c6965fadbafbcdb1fefaa11ccc759bc27c1985111b2c89dd90fef1512a0c", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "1ef1848a5acc3f0a8f88dd9e5393583ad58bc56be70f617475404898547de547", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "1cfbaf51506a24141cf75c07426866b69a1a1ccba434ff8df8d02201a3723ed5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "8296c2098c4f67cdca611e08631d9f617ec67f2012947fcae85284c48372ccf5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "630081281dfe68e8a7e95a0819d8bd67473640a54ab06f6eff2a6d453d915cbc", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "a02a3ba8575d7c5708b82cc3ab08951b6d99f0383fa02aa37243e863624ae25f", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "88817ac4c3c77e9a120eac10af812d333f7360e92f7c36fce22a01b70b7ac27d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "745bca92a6317990f12ae0015cc860a2b2b9c87c75d3a40ffe2b1393db87b923", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "d4d2bbcb3803d2a8c2a6276d64a163c606a68bc29ab06f0e819191353348c44d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "cb2317bf324a28dea4a564e79faaa41d77b2880c4988bcbb81cab3fc0ea1cd61", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "4004212ff6037092e741aefd6fff244d1c2f7a039094b40e107d1fb9dfac8243", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "a7ae48ab626e6e00d4af7c6a0b93e29b15c275668ab485ba45c6122fd384c85d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "3dbf3eb4cd1f6af99c22b6fa308fb5ae904f843a77aa11364439916f22c80f08", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "fcfb17f721dcdd6c4b18ce747c256999d5eb0968fbb70f4e70b548a255fa9199", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "0f9e5909143e71137044eceb40e7357df8ee54c3a079fbc75e998b827838f601", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "line_sha256": "50ca1d66942948ce263512134d0270fd7d9c3bc5fc7a96ffeb8d0bb9547586b5", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "828938b08e41b742df3b8e77ff9997545b92f9efac9f68e820aedd32fa4ed9a0", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "c0e20dbf5eaa1a2470f432f84ce58675f5c061550d1f0b971ea372b1360ba9e1", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "fc39ef06b93c4b9b0ccf89af188624e1f6c91e0241d7963b23d39198694fdfdf", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "6875d112abc5b8f51f2f1a2d1a760eee4efd2b9e4f28ba5a5793666de5bcc48a", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "d2d1b6837d19466e1b9d7d9c866a7c44f063cdb89937e528177494c5c96f6cc4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "8d3e46f8aa00286df8f09154b483fbe787d3843dda5747d2d5082fdf70dc2c95", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "ec0989feac111d3cdf0c2ba9f71bb1b88da47c829393a22ad79fac517b0aeb5e", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "69e4fe69d2f7e3c4e19b23e1138010bc2139799c50685e2ebe93db174d9960e9", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "2c0e11e47247527c3ee4c480f29c859f20841d21c29206bc6036a14c30194423", + "markers": [ + "-maxdepth" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "7f7681bc1e69ab9124a7cec506d4370888d7d18a8949f4ab61403030f6a17951", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "68d6b2ab40dfb15eb1af46862d8474310305c111f2d7af35053dae4ac8341738", + "markers": [ + " -r " + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "5a19f132dcb84775c298072b9b10764020c4aef4c5bc58809b13bd528ec524fa", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "cb36dd1d24ac5cd3b87e68a0e8fe23334027825e2f54fed3c04a7be3d902a59d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "952bd1f3229402a503da3603fb81f2ab7480ebebfc83d698f478961e83ddb08f", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "1d5c7d2db199056204a4647e18f36a19ffc32d7807afab758b9eb8565e3715cc", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "24a684162da36bf1d24db913393f84e9cd06568fb760f550e8524bee3a59a40d", + "markers": [ + "repo_root" + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "475644924650e7798cd0cb94d6957bc66f5a649fe3995c9e4f639091412b3c18", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "line_sha256": "eb6966faec145ae9e4ddf3a9bfc45ee10af3d729d7b6bb706479dab3f8069959", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true, + "repeat": 1 + } + ], + "no_commands": [], + "run_ids_sha": "b02ea16a5f05b574c52d51c9e3a98c410aa66655617212ed2cf9aa7b76cf08c7", + "unaudited": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "repeat": 1 + } + ], + "vacuous": false +} diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/reconciled.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/reconciled.json new file mode 100644 index 00000000..9d477628 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/reconciled.json @@ -0,0 +1,567 @@ +{ + "bundle_id": "cd55711a27f7080f", + "rows": [ + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "With `time=\"period\"` and `fixed_effects=[\"unit\", \"period\"]`, the expansion introduces duplicate `period_` names. Dictionary construction overwrites original coefficients while retaining every covariance-matrix row.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (1 failed, 10 passed) and numeric reproduction (34 coefs vs 38x38 vcov). | Cites an unseen pytest run ('1 failed, 10 passed') and a reproduced '34 coefficient entries versus a 38x38 covariance matrix'; unverifiable here. | Cites an unverifiable focused test run (1 failed, 10 passed) and reproduction numbers (34 vs 38x38)." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "dummy expansion duplicates the existing `period_` names. Rank-deficiency handling produces dropped coefficients, then dictionary construction overwrites original period coefficients with NaNs and discards entries.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (1 failed, 10 passed). | Cites an unseen pytest run ('1 failed, 10 passed'); unverifiable here. | Cites an unverifiable focused test run (1 failed, 10 passed)." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "when the time column is named `period`, it also duplicates the `period_` keys. The final dictionary comprehension silently collapses those names and can replace valid period coefficients with dropped-column `NaN`s while leaving the full-sized VCV intact.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). Cites unverifiable reproduction (34 keys vs 38x38 VCV). | Rates it P0 (expected P1). Cites an unseen reproduction ('34 coefficient keys for a 38x38 VCV'); unverifiable here. | Severity stated P0 (expected P1); cites unverifiable reproduction numbers." + }, + { + "case": "s1-coef-dict-collision", + "bug_id": "s1-coef-dict-collision:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "Expanding `fe == time` creates an identical second block with the same names. When results are converted to a dictionary at `diff_diff/estimators.py:L1968-L1969`, later duplicate keys overwrite the original coefficients.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable focused pytest failure. | Cites an unseen focused pytest failure; unverifiable here. | Cites an unverifiable pytest run of the regression test." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "`ddof=1` changes each arm\u2019s variance contribution from `sum(residual\u00b2)/n\u00b2` to `sum(residual\u00b2)/(n*(n-1))`, while the returned influence function remains unchanged. Consequently, cell inference diverges from aggregation and the reg/ipw paths.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). Cites unverifiable test run (2 failed, 2 passed) and numeric reproduction. | Rates it P0 (expected P1). Cites an unseen test run ('2 tests failed, 2 passed') and a numeric reproduction table; unverifiable here. | Severity stated P0 (expected P1); cites unverifiable test run (2 failed, 2 passed) and reproduction table." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "Using `ddof=1` introduces group-specific variance multipliers of `n/(n\u22121)`, while aggregation continues consuming the unchanged influence function. This violates `docs/methodology/REGISTRY.md:L575-L580`, which explicitly requires `sqrt(sum(phi**2))`", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test run (2 failed, 2 passed) and numeric reproduction (0.721688 vs 0.625). | Cites an unseen test run ('2 tests failed, 2 passed'); unverifiable here. | Cites unverifiable test run (2 failed, 2 passed) and reproduction numbers." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "The new `ddof=1` plug-in SE no longer equals the variance of the influence function returned by the same method.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; claims direct invocation of `_doubly_robust()` reproduced 0.7216878365 vs 0.625 (unverifiable). | States pytest could not initialize (no writable temp dir) and claims a direct `_doubly_robust()` reproduction (0.7216878365 vs 0.625); unverifiable here. | States pytest could not start (no writable tmp dir) and claims a direct in-process reproduction (0.7216878365 vs 0.625); unverifiable." + }, + { + "case": "s1-cs-dr-plugin-se", + "bug_id": "s1-cs-dr-plugin-se:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "The PR silently replaces the registered influence-function variance with a `ddof=1` plug-in variance, inflating per-cell SEs and changing t-statistics, p-values, and confidence intervals.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). Cites unverifiable validation run (2 failed). | Rates it P0 (expected P1). Cites an unseen validation run ('2 failed') with numeric values; unverifiable here. | Severity stated P0 (expected P1); cites an unverifiable validation run (2 failed)." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings are dose-share variance omission and survey double-weighting in continuous_did.py; no single-level zero-ACRT finding. Numeric reproductions unverifiable. | Raises two P0s (global ACRT SE omits dose-share uncertainty; survey SEs double-weighted) in continuous_did.py, not the J==1 zero-ACRT estimand defect; treats the boundary convention as documented P3. Numeric reproductions and '19 focused tests' are unverifiable. | Findings are in continuous_did.py (global ACRT dose-share variance P0, survey double-weighting P0); nothing on the single-level all-zero derivative matrix. Reproduction numbers and test counts unverifiable." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Only the dose-frequency variance P0; cites unverifiable test run (127 passed) and 11% coverage simulation. | Only P0 is the dose-frequency variance omission; no single-level/zero-ACRT finding. Claims '127 passed, 8 skipped' and an '11% coverage' simulation; unverifiable here. | Only the dose-frequency variance P0 in continuous_did.py; cites an unverifiable 127-passed run and an 11% coverage simulation." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Touches `continuous_did_bspline.py:L324-L368` only as a P3 documented boundary convention and L243-L270 for non-finite doses; the J==1 all-zero derivative defect is not named. States pytest could not start; reproductions unverifiable. | Right symbol (saturated_derivative_design_matrix) but only as a P3 'documented convention'; the single-positive-level all-zero ACRT defect is not named. Three P1s on other defects. States pytest could not start; direct reproductions unverifiable. | Right file: cites continuous_did_bspline.py L324-L368 only as the documented P3 boundary convention and L243-L270 for a NaN/inf dose P1; does not identify the J==1 zero-ACRT estimand defect. Pytest reported unable to start; direct reproductions unverifiable." + }, + { + "case": "s2-acrt-single-dose", + "bug_id": "s2-acrt-single-dose:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "returns an all-zero operator when `J == 1` (`L358-L359`). Consequently, both `acrt_d` and `acrt_glob` are zero, and their analytical SEs are zero through the same operator", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Absolute-path citations under tools/reviewer-eval/runs/.worktrees// are the reviewer's own worktree (exempt per rubric). States pytest could not start. | States pytest could not start (no writable temp dir); relied on static inspection plus direct evaluation of the helper. Absolute-path citations are to its own worktree (exempt). | Cites files by absolute path under tools/reviewer-eval/runs/.worktrees// (exempt per rubric). Pytest reported unable to start; static inspection plus direct helper evaluation claimed." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Flags the same mis-framed screen at L1534-L1539/L1552-L1558 but argues the CiC side (not necessary) rather than the QDiD side (break is direct evidence). Cites unverifiable 93-test run and simulated p-values. | Same symbol (_handle_cic mean-trends screen) and same class, but a different defect: argues the screen is NOT necessary for CiC, rather than that the QDiD framing understates a mean-trend break as direct evidence against the additive model. Reproduction p-values and '93 tests passed' unverifiable. | Flags the same mean-PT screen text in _handle_cic (L1534-L1539) as mis-framed, but in the opposite direction: argues mean PT is not necessary for CiC and should be dropped, rather than that the QDiD additive model makes a mean-trend break direct evidence against the model. Numerical counterexample (p~2.2e-17) unverifiable. | verifier: The ground truth is that the shipped QDiD text understates the mean-trend screen's evidential value (under QDiD's additive model a mean-trend break IS direct evidence against the model) while claiming QDiD does not identify off mean parallel trends. This review attacks the same screen at the same location (L1534-L1539 within `_handle_cic`) but in the opposite direction: it argues mean PT is NOT necessary for CiC and should be demoted to descriptive. It never mentions QDiD's additive model, the 'does not identify off mean PT' claim, or that the text understates the check, so it is a different defect at the same location." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same location/screen; framing emphasizes CiC. Cites unverifiable 93-test run and numerical reproduction. | Same screen location, opposite-direction defect (CiC non-necessity) plus unrelated P1s (covariate conditioning, QDiD additive-separability omission); does not name the QDiD mis-framing of the mean-trend screen. Test-run and numerical claims unverifiable. | Same location, same class (mis-framed PT screen) but argues the CiC overstatement, not the QDiD understated-evidence/mis-framed identification defect. Also adds an additive-separability-omission P1. 93-test run unverifiable. | verifier: Same screen, same location, but again only the 'not necessary for CiC' direction; the review wants the necessity claim removed rather than strengthened for QDiD. Its separate P1 'Include QDiD\u2019s additive-separability assumption' (L1401-L1407, L1514-L1525) notes the assumption text omits `Y\u1d3a=h\u1d33(U,G)+h\u1d40(U,T)` but never links the additive model to the mean-trend screen or says the shipped QDiD text mis-states the evidential value of a mean-trend break. The ground-truth mis-framing of the identification argument is not named." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Also notes QDiD's additive model would justify the screen. Cites unverifiable 93-test run and `git diff --check`. | Same location/class; flags CiC non-necessity rather than the QDiD understatement. Mentions in passing that QDiD's additive model implies the screen but does not flag the shipped QDiD text. '93 passed' and `git diff --check` runs unverifiable. | Flags the mean-PT screen at L1528-L1558 as wrongly 'necessary' for CiC; mentions in the fix that QDiD's additive model implies the screen, but does not name the shipped QDiD framing as the defect. `git diff --check` code span contains no marker. Test count unverifiable. | verifier: Right location and right screen, but the finding is that the screen is wrongly applied to CiC. The only QDiD remark is a conditional aside in the fix ('If retained for QDiD, explain why QDiD\u2019s stronger additive model implies it'), which treats the QDiD screen as acceptable to retain; it does not say the shipped QDiD text claims QDiD does not identify off mean parallel trends, nor that the 'necessary-but-insufficient' framing understates a mean-trend break as direct evidence against the additive model. Per the verifier criterion, a 'would justify' remark is not the ground-truth defect." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same location/class; defect framed as CiC non-necessity, not the QDiD mis-framing. Also raises a P1 'snippet fails at advertised minimum' and a P1 on identification-range wording; unverifiable. | Same location/class; argues CiC does not imply mean PT and notes 'QDiD may retain a mean screen because its additive model implies common mean changes', but the named defect is the CiC overstatement, not the QDiD mis-framing. Review also contains '[model-redacted]' tokens. | verifier: Same pattern: the P1 is the CiC-direction objection at the right location, and the closing sentence 'QDiD may retain a mean screen because its additive model implies common mean changes' endorses rather than criticises the shipped QDiD screen. Its separate P1 'QDiD\u2019s identifying model is incomplete' concerns omitted additive-separability/monotonicity in the assumption text, not the evidential value of a mean-trend break, and the snippet P1 is about `pre_periods=`. No finding states that the shipped QDiD text mis-states or understates what a mean-trend break implies, which is the ground-truth defect." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance routine without a minimum-cluster guard. Its finite output passes through `safe_inference()`, producing misleading significance despite having no independent cluster replication.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites had.py:L3954-L3963 (forwarding site) rather than the delegating local-linear function; same defect. Numeric reproduction and suite passes unverifiable. | Cites unseen suites passing and a numeric single-cluster reproduction (se=0.287294, p=2.49e-6); unverifiable here. Also claims the survey workaround example raises NotImplementedError 'reproduced'. | Location cited is the forwarding site (had.py L3954-L3963) rather than bias_corrected_local_linear; same defect. Reproduction values and 'survey workaround raises NotImplementedError ... was reproduced' are unverifiable." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "The newly forwarded cluster array reaches a variance implementation without a minimum-cluster guard. [...] every cluster ID equal to zero, the fit returns `se=0.789637`, `p_value=0.864990`, and finite CI endpoints without any warning.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable 418-test run and reproduction. | Cites '418 tests passed' and a numeric reproduction; unverifiable here. | Cites unverifiable 418-test run and reproduction numbers." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "`_fit_continuous()` forwards clusters without validating the number represented in the active local-polynomial window. `lprobust_vce()` then forms an outer product even when `G=1`, yielding a finite value that passes `safe_inference()`.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States full pytest could not start; direct probe numbers unverifiable. | States full pytest could not start (no writable temp dir); direct probe numbers unverifiable here. | Pytest reported unable to start; direct probe numbers unverifiable." + }, + { + "case": "s2-had-cluster-guard", + "bug_id": "s2-had-cluster-guard:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "The newly threaded cluster array reaches `lprobust_vce()` without checking that at least two effective clusters remain. Unlike the mass-point path, which returns `se=NaN` for one cluster, the continuous path accepts the resulting positive numerical SE and passes it to `safe_inference()`.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "States pytest could not start; end-to-end probe numbers unverifiable. Extra P1 on cr1 label is outside ground truth. | States pytest could not start; end-to-end probe (se=0.0161, t=68.28) unverifiable. Extra P1 on `cr1` label vs nprobust NN-cluster meat is unverified. | Pytest reported unable to start; end-to-end probe numbers unverifiable. Additional P1 on cr1 label mismatch not verifiable from the bundle." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "The new guard can reject treatment contrasts, but `MultiPeriodDiD` substitutes residual DOF for per-period inference. [...] triggers the guard yet reports **p=0.00314**, CI **[1.4364, 3.0767]**. [...] The warning\u2019s promised inference suppression is therefore incomplete.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P1 (expected P0); located at linalg.py:L2138-L2142 with estimators.py consumers. Cites unverifiable 563-test run and reproduction. | Rates it P1 (expected P0) and tags it '[Newly identified]' although this is not a re-review. Cites '563 tests passed' and numeric reproductions; unverifiable here. | Severity stated P1 (expected P0); names the guarded/non-finite DOF yielding finite p/CI at linalg.py L2138-L2142 with downstream consumers. Reproduction values and 563-test run unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same CR2-BM guard in linalg.py but only the scale-invariance defect; explicitly asserts the inference path 'propagates unavailable DOFs consistently to NaN', the opposite of the ground truth. Test counts unverifiable. | Same guard in linalg.py but a different defect (scale invariance); explicitly asserts the opposite of the bug: 'The affected coefficient-inference path propagates unavailable DOFs consistently to NaN'. '563 tests passed' unverifiable. | Same guard in linalg.py (L2104-L2121) but a different defect (scale non-invariance); explicitly asserts the inference path 'propagates unavailable DOFs consistently to NaN', i.e. the opposite of the ground truth. Test counts unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same guard, scale-invariance defect only; states 'The warning makes this fail-closed rather than silent'. Reproduction unverifiable. | Same guard region in linalg.py, wrong defect (predictor-unit scale dependence); describes the guard as 'fail-closed rather than silent', missing the non-finite-DOF fail-open. Six-test run and 1e5 reproduction unverifiable. | Same guard, wrong defect (predictor-unit scale dependence); states 'The warning makes this fail-closed rather than silent'. Reproduction numbers unverifiable." + }, + { + "case": "s2-nan-dof-fail-closed", + "bug_id": "s2-nan-dof-fail-closed:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same guard, scale-invariance defect only; asserts NaN DOF makes t/p/CI NaN. States pytest could not start; reproduction unverifiable. | Same guard region, wrong defect (scale invariance); does not name the non-finite-DOF fail-open. States pytest could not start; direct reproduction numbers unverifiable. | Same guard, wrong defect (scale invariance); claims downstream fields become NaN. Pytest reported unable to start; reproduction unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "Balance is checked before filtering missing outcomes or covariates. A complete unit-period grid containing missing outcomes therefore bypasses RC routing.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test runs (133 passed) and golden-data reproduction numbers. | Cites unseen test counts ('133 passed, 8 skipped') and golden-data reproductions; unverifiable. Extra P1 on the stationarity warning wording unverified. | Golden-reindexing reproduction numbers and test counts unverifiable; contains '[model-redacted]' tokens." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "Balance is determined from identifier pairs before handling missing estimation values. Expanding the golden onto the complete unit-period grid, leaving absent outcomes as NaN and filling cohort labels, changes `used_rc_on_unbalanced_panel` from `True` to `False`.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Additional P0 on NaN first_treat row-order dependence is outside ground truth and unverifiable; cites 181-test run. | Also raises a P0 (NaN first_treat row-order dependence) with a claimed reproduction; '181 passed' and numbers unverifiable here. | Additional P0 (row-order-dependent cohort mass with NaN first_treat) rests on an unverifiable reproduction; 181-test run unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "`_is_unbalanced_panel` counts raw `(unit, time)` keys before outcome, covariate, cluster, or weight validity is resolved. A structurally complete panel with one missing outcome is classified as balanced, making `allow_unbalanced_panel=True` inert.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Graded P0 (expected P1). States pytest could not start; in-process reproduction unverifiable. | Rates it P0 (expected P1). States pytest could not start; in-process reproduction unverifiable. | Severity stated P0 (expected P1). Pytest reported unable to start; in-process reproduction unverifiable." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "bug_id": "s4-cs-unbalanced-nan-outcome:b1", + "arm": "M2", + "repeat": 1, + "verdict": "caught", + "evidence": "Python currently checks only raw `(unit, time)` row presence. An otherwise balanced panel with one `NaN` outcome therefore leaves `used_rc_on_unbalanced_panel=False` and uses within-cell panel differencing", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable test runs (527 passed / 458 passed). | Cites unseen test runs ('527 passed, 58 skipped'; '69 tests'; '458 tests'); unverifiable here. | Cites unverifiable 527-passed validation." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M1", + "repeat": 0, + "verdict": "caught", + "evidence": "`solve_ols()` marks dropped, collinear coefficients as NaN and already returns valid residuals for the retained model. Discarding those residuals and calculating `response - design @ coef` makes every residual NaN. PSU aggregation subsequently converts the NaN scores into zero totals, yielding **SE = 0**.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Symptom reported as SE=0 rather than NaN; same defect/location. Extra P0 saturated-sample finding and 117-test run unverifiable. | Also raises a P0 saturated-sample guard bypass with a claimed reproduction (SE 2.61e-16); '117 tests passed' and numbers unverifiable here. | Symptom described as SE=0 rather than NaN but same mechanism/location (L939-L957). Separate P0 (saturated survey sample yields 2.61e-16 SE) and the 0.044757 number rest on unverifiable reproductions." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M1", + "repeat": 1, + "verdict": "caught", + "evidence": "`solve_ols()` represents dropped coefficients as NaN. Recomputing residuals with `design @ coef` propagates those NaNs across every row. The PSU aggregation can then turn these into zero scores and return a zero variance.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable reproductions and 117-test run. | Also raises P0 saturated-sample and P1 zero-weight-horizon findings with claimed reproductions; test counts and numbers unverifiable here. | Separate P0 (saturated sample) and P1 (zero-weight horizon ValueError) rest on unverifiable reproductions; 117-test run unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M2", + "repeat": 0, + "verdict": "caught", + "evidence": "`solve_ols()` represents dropped columns with `NaN`, but `_estimate_survey_sample()` computes `design @ coef` without excluding them. The resulting all-NaN residual scores are reduced to a zero survey meat by the PSU `groupby().sum()`, yielding `se=0.0`.", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Cites unverifiable 105-test run. | Cites '105 tests pass'; constant-covariate reproduction unverifiable here. | 105-test run unverifiable." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "bug_id": "s4-lpdid-survey-unreduced-design:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same symbol (`_estimate_survey_sample`, L921-L962) but a different defect (dropped-PSU design reconstruction); rank-deficient unreduced-design sandwich not named. Reproduction SE numbers and 105-test run unverifiable. | Same symbol (_estimate_survey_sample L921-L962) but a different defect (dropped-PSU design reconstruction, rated P0); the unreduced-design/rank-deficient residual recomputation is not named. Pytest count and SE reproduction unverifiable. | Right symbol (_estimate_survey_sample L921-L962) but a different defect: dropped-PSU design reconstruction (P0, SE 0.054641 vs 0.055492, unverifiable), plus metadata/subpopulation findings. Does not name the discarded residuals / unreduced design rank-deficiency issue." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M1", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings are CI-inversion monotonicity and numerical tie misclassification in utils.py plus a positional-arg P2; no storage-vcov rank-deficient crash finding. Cites unverifiable 465-test run and golden-data reproductions. | Findings are CI-inversion non-monotonicity and projection tie misclassification (utils.py) plus a positional-arg P2; no finding on the storage vcov computed on the full rank-deficient design. Claims '465 passed' and numeric reproductions; unverifiable. | Findings are utils.py inversion non-monotonicity and projection tie-count P0s plus an estimators.py constructor-ordering P2; nothing on compute_robust_vcov over the unreduced design in _run_wild_bootstrap_inference. Reproductions and 465-passed run unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same two utils.py P0s; no estimators.py wrapper crash finding. Reproduction script and 59-test run unverifiable. | Two '[Newly identified]' P0s (CI inversion, projection tail counts) in utils.py; nothing on _run_wild_bootstrap_inference. Reproduction code/numbers and test counts unverifiable. | Only utils.py P0s (inversion, projection). Fenced python reproduction block contains no markers. Numerical claims unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M2", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Findings: CI inversion P0, absorbed-FE reabsorption P0, stale `_bootstrap_results` P1 (estimators.py L707-L737); none is the storage-vcov rank-deficient crash. States pytest could not start; counterexamples unverifiable. | P0s on CI inversion and FE non-reabsorption, P1 on stale `_bootstrap_results`; none concern the full-design storage vcov crash. States pytest could not start; counterexample numbers unverifiable. | Findings: utils.py inversion P0, twfe/estimators absorbed-FE reabsorption P0, stale _bootstrap_results P1 (estimators.py L707-L737); none about the rank-deficient storage-vcov crash. Pytest reported unable to start; counterexamples unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b1", + "arm": "M2", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Same CI-inversion and FE-projection P0s; no wrapper crash finding. Cites unverifiable 59-test run and six-cluster reproduction. | Same two P0 classes as repeat 0 (CI inversion, FE projection); nothing on _run_wild_bootstrap_inference. '59 passed' and six-cluster reproduction unverifiable. | Same two P0s as repeat 0 (inversion, absorbed-FE refit); nothing on _run_wild_bootstrap_inference's full-design vcov. Reproduction numbers unverifiable." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M1", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Mentions reviewing 'the shared HC1/CR1 saturated-design guard' but raises no guard-order finding in linalg.py. | Mentions 'the shared HC1/CR1 saturated-design guard' as in scope but raises no guard-order finding. | Mentions 'the shared HC1/CR1 saturated-design guard' as an affected area but raises no finding on guard ordering vs cluster-count validation." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M1", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No finding on the saturated-NaN-guard / cluster-count ordering in _compute_robust_vcov_numpy. | No finding in linalg.py _compute_robust_vcov_numpy." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M2", + "repeat": 0, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No finding on guard ordering in linalg.py. | No finding in linalg.py." + }, + { + "case": "s4-wcr-saturated-guard-order", + "bug_id": "s4-wcr-saturated-guard-order:b2", + "arm": "M2", + "repeat": 1, + "verdict": "missed", + "evidence": "", + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "No finding on guard ordering in linalg.py. | No finding in linalg.py." + } + ], + "negative_assessments": [ + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean (Looks good, no findings). Claims an unverifiable seed-23 geometry reconstruction. | Clean (Looks good, no findings). Claims an independent seed-23 geometry reconstruction; unverifiable here. | Looks good, no findings. Claims to have reconstructed seed-23 geometry (no units 80-200 km) - unverifiable." + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean. Claims an unverifiable independent geometry verification. | Clean. Claims independent geometry verification; unverifiable here. | Looks good, no findings. Geometry verification claim unverifiable." + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean; only two P3 informational entries. | Only two P3-informational entries (within allowed severities). | Looks good; two P3 informational entries only (within allowed severities)." + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean. Claims an unverifiable fixture geometry check. | Clean; all sections 'Severity: None'. Claims fixture verification of the 80-200 km band; unverifiable here. | Looks good, all sections 'Severity: None'. Fixture verification claim (zero units 80-200 km) unverifiable." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean; one P3 informational noting the documented Deviation from R. | Only a P3-informational note on the documented floor (allowed severity, known topic handled correctly). | Looks good; single P3 informational on the documented floor (allowed, known topic)." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean; one P3 informational on the documented floor. | Only a P3-informational note on the documented floor (allowed). | Looks good; single P3 informational on the documented floor (allowed, known topic)." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 0, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean; one P3 informational on the documented floor. | Only a P3-informational note on the documented floor (allowed). | Looks good; single P3 informational on the documented floor (allowed, known topic)." + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 1, + "findings": [], + "leak": "none", + "grader_flag": "none", + "leak_evidence": "", + "notes": "Clean; no findings. | Clean; no findings at any severity. | Looks good, no findings." + } + ] +} \ No newline at end of file diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-definitive.txt b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-definitive.txt new file mode 100644 index 00000000..ca1586fa --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-definitive.txt @@ -0,0 +1,3705 @@ +{ + "audit_summary": { + "cells": 44, + "cells_with_hits": 42, + "coverage": { + "covered": 4, + "no_commands": 0, + "total": 44, + "unaudited": 40 + }, + "orientation_only_cells": 5, + "unaudited_cells": 40 + }, + "audit_vacuous": false, + "campaign_reads": { + "cost_estimate_usd": { + "B": { + "input_price_per_1m": 4.0, + "model": "gpt-5.6-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 1.259472 + }, + "F": { + "input_price_per_1m": 2.0, + "model": "gpt-6.1-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 0.629736 + } + }, + "latency": { + "B": { + "max": 472.4614837919362, + "mean": 213.33513880114663, + "n": 22, + "p95": 413.60439145797864 + }, + "F": { + "max": 416.4505883329548, + "mean": 180.63859337303703, + "n": 22, + "p95": 276.9192527080886 + } + }, + "latency_flag_candidate_p95_over_2x_control": false, + "vendor_lifecycle": "operator_supplied" + }, + "candidate": "F", + "catch_status": { + "s1-coef-dict-collision|s1-coef-dict-collision:b1|B": "reliable", + "s1-coef-dict-collision|s1-coef-dict-collision:b1|F": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|B": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|F": "reliable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|B": "unstable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|F": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|B": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|F": "missed", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|B": "reliable", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|F": "reliable", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|B": "missed", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|F": "unstable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|B": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|F": "reliable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|B": "unstable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|F": "reliable", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|F": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|F": "missed" + }, + "contested": [], + "contested_gates": [], + "control": "B", + "definitive": true, + "downgrades": [], + "evidence_gaps": [], + "exposure": { + "first_subdir": "gpt61sol", + "frozen_head": "e1674c0e2a138f63300696f2b967268920c7e07b", + "head_changed": false, + "ledger_entries": 1, + "mechanical": true, + "reason": "first dispatch of the contrast (frozen revision)", + "subdirs": [ + "gpt61sol" + ], + "surface_diff_stat": "", + "this_subdir": "gpt61sol", + "timeout_only": false + }, + "fp_candidate": 0, + "fp_control": 0, + "fp_findings": { + "B": [], + "F": [] + }, + "gate_cells": { + "downgrade": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4581c043835af761f02f10ddacc67cd338cd7d9cc050ab41ad1d76eca63b8099", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "43875f510f7d4d2355ea7a1bbd06ce3a393d3cb5e31bb377b6bbad40b0d023c2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "6db06274942ca668070ccee3f005da1d2cc2f9cb908e4b2faeee02266b28aee1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc5e0932da413124af878ef15d712049afc6aedf5ca58d90111baabe01f28b9c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "de9ffa6055e0a2383967d3cbf194e3677e100d3b8cd2a94e1e19af29c83dafb9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4a23000128509bdf1a9dff7da14e195599c24a286e88f211089636ff81dcb463", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "c01dd2b41980c065e0a777d9d15a8e4e0e3ff0b3d64fcbcb1a9bc36cda03d565", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c00612992135e362b61ef9aa2c8840ca720851d362b5ca14ad59480746c78254", + "markers": [ + "../" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "be903554134a616a4f51158f58b4e3a6fa7c63ee7f792338eee51d14d562c019", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e7d59ebeae9764faf8602a13d1f56dcd3d82b82bd26e3e21c6aa7836f5a4e8ad", + "markers": [ + "HEAD~" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "bff3b5f05108e0e3bead6fdc57235ca733327489684a089019e7608712b2737b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e1973c0cba163fa32110a25fe567930fcdfe33c7032958599913efd0311d7898", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "642d39c1c8e8172f91e155f4ecaf711603bfadc4c61543df8cb6bfec3ac22c9d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "436ee100fd57903cf9e6eb66780804d032df8908bf0c33dbd78e5d69de1078b6", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "6bc9c7174b8708e37833c5fa3c06a6badf2620f825274205ddbbf27b6a29b4b9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "8c003bd8814eed70d32562a28c791afbcfafcd9c8c284e05718b2a3f04ea0356", + "markers": [ + "rev-parse", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "972096a15295e925f2e3cf87eee726b50fabd9ef05a627c24781c89d1daa86f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "679da91f949a511dbee6113010bc53b87533af6163b5fef3d71638f2803a5231", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1f0d006bf9e7caf5e7a47f03a5c6d89ee49e952bc50921c4f77293f757612b65", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "0fc03c2824d3dde62aefd151f0485e213a06e59d17821d84502a586763519ec5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "814f0f33cb379d7ac38340497f36e9195ce02d3f2de9d7ac57219278648acdea", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "031a2d49b3a82bc591181eee88488e1237d89f86b070e9914026376088a978cb", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b945e48385ff5ea9cf694e0f3f275a664d5d6ffeb4a7a83d6a0524836cd320ed", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "aee739303916771552df815506d15b8966cc3848ba41156b5a18e9fa1c8d3f11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7db0895e590ea7ecf2f5f4e4a960e883e47b14a532948b64fdc341d0aad21d62", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "9eb4e535ccab0a3db7afd4b81945d6618e017f48173ee821d15d2993d7cea36d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "79ef89a6f757830872163acb657afe333343b38d5f7bcb2a0f52ad5b28d6cd59", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "06c4414f2ba3ca58ffdd8b68e6594dc712a6de0c1a18d696fa36f05562a084fc", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "cc391d5c7315c5ee5895e7c4648ed7cab84686a0a577616f1a2896b1e043adf3", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60cce41f7825e3a471ff0276a7d6ce8c8d23d7452936dbda837dcc6fcddc638a", + "markers": [ + "-rn" + ], + "orientation_only": false + }, + { + "line_sha256": "27eeeb0397a46f1e0fc0f7cfd2e27fa5cb71b030d47d401629460ab83f82c659", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "dd849ca4a37bf4802b2e980a5f5a1ecec1bde2809fa5e2c9ae4462b5537eebaa", + "markers": [ + "git show HEAD", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2073d2e30017c6c0d3f45fac4c79b047ee85b22f738dadabe82b8da10e27f140", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "eefc1eb47dddd5e0c76f397a720c93e6cbabb0eb8cb191e267fffff59ca138a6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f8c2aaaaa9ad31c0c8496a39705bcec59fad5c1acdc87d8ca1280aec8d62d185", + "markers": [ + "HEAD~", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd9812cb38239143c14da974259380a76a5dcca7f30f7472aea3c5c728a76327", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3a48f5782a5b99f840d78e9108b5779b3f7e4da65719a8da0e0ee11b0b7e2c6e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3b298a3e9116ee16ca656adb3788f5b1c622968e55b0bbac384611c8780a06c2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "27b09e81480add37b82ec1e90fc091c3aa7d6531d14120dcb66356471ff905a0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3c365ebde7a7b9516774687d8c1f6db23e08fb4ef22403fd9b096d6e2e010e40", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "236a4cdb8f444afad1d920dc575956f38890768ce1dccfd95db2764b1b3c5f0f", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ffef8182975bcf1a8bf68c942c8ad74e6a0bf2d8955cbbc75ef83ee14766277a", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d165c839fb89e0cccb1f133424487a1765beca86386f68a46c6876408d0eef20", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60b823fc440ff4b309641567a2e51c65208cf2cbe0aca5ac87644632851e1926", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "932e76e69ddf9c66793543e58bf8f76ff4d571a48908b481b48a7bc2a146fed4", + "markers": [ + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "78b7d8375dacaa30cbd6915edd3f3d7ba7217c66d9c5969f31b557cc53dde711", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "831c08aa7dbe60925e452201bba20cc6d9ddcfe1baf738f2b7f8fe91cf183172", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "5d5ce7c4dd128ba5551eea81c15d63739a52d7b0c08a6b2f1d8df314a8f37667", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "f3d859fc65dc144719341882dd788a5a09f6372461936fa15a961f67ea360f41", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3330e7f41e67f7334081aabf48c0e2527f04cad55ba2348b5bd8edf5f33e270a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "406fc2aa4c793871d253d1f186bdb7949666edd60ddd2794bae18f4c50fea2b8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "20fc9ee9abd090f617c98093c06224ff938b2337f4bd9ee2b65732cf21bf7878", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "9356a1d8a8d8d5d98bbffc08bbd9261253ba9bb944871c4a590fdcf6673e8139", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1382c9aedc38269dd3adb840c0535836d775d252b85ca0e181e9f45bd6ae46fe", + "markers": [ + "rev-parse", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ad506db11fe1e5f12d1870c4e03120cf424bbb7dcdf7a59727f20616835b6ebe", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE", + "DOTDOT_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "47e685f03a4c622c078fb93098ef698a64424e0d653f537c30689f514889313b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc3c8e0736af923ddb6352a221a14b2aa2799e978a4023dd1e3677bc8479c7d5", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "dedf87ab4d5a344a6fa81f1744b6ed4020c4a357edb7c33f26260c69007346f6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "e32548de94b9b6a582b9f14a59eded3f913db7668802f7044f6b50d3e89ea872", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e3230c6c5073a314519f85a367485585743dc722a2fdc4838d48eb388d027ef0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ff969993a2523cfc41df3f56bc748392aff5654c077c7719d129654681b76ec8", + "markers": [ + "../", + "DOTDOT_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2c5e45dbe5528cfed07b9461e75b4d25f7f66a5b841b28072ab99c1e343ba8b3", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "decca74bfa41b6dbec7c7b4781d28d6f236093a4b8dc0635abcd619947717c40", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "f70a9944f4ebac984d6a738d3e3faacd74f8f32e5e3147c54bbb1a26f6e69f92", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "eee6b44b32b3942f6bf00eb8c4423d17fe2b257ebeaf810c4eece5bff8a1b1ee", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ab52c6965fadbafbcdb1fefaa11ccc759bc27c1985111b2c89dd90fef1512a0c", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1ef1848a5acc3f0a8f88dd9e5393583ad58bc56be70f617475404898547de547", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1cfbaf51506a24141cf75c07426866b69a1a1ccba434ff8df8d02201a3723ed5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "8296c2098c4f67cdca611e08631d9f617ec67f2012947fcae85284c48372ccf5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "630081281dfe68e8a7e95a0819d8bd67473640a54ab06f6eff2a6d453d915cbc", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a02a3ba8575d7c5708b82cc3ab08951b6d99f0383fa02aa37243e863624ae25f", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "88817ac4c3c77e9a120eac10af812d333f7360e92f7c36fce22a01b70b7ac27d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "745bca92a6317990f12ae0015cc860a2b2b9c87c75d3a40ffe2b1393db87b923", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d4d2bbcb3803d2a8c2a6276d64a163c606a68bc29ab06f0e819191353348c44d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb2317bf324a28dea4a564e79faaa41d77b2880c4988bcbb81cab3fc0ea1cd61", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "4004212ff6037092e741aefd6fff244d1c2f7a039094b40e107d1fb9dfac8243", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a7ae48ab626e6e00d4af7c6a0b93e29b15c275668ab485ba45c6122fd384c85d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3dbf3eb4cd1f6af99c22b6fa308fb5ae904f843a77aa11364439916f22c80f08", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fcfb17f721dcdd6c4b18ce747c256999d5eb0968fbb70f4e70b548a255fa9199", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "0f9e5909143e71137044eceb40e7357df8ee54c3a079fbc75e998b827838f601", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "50ca1d66942948ce263512134d0270fd7d9c3bc5fc7a96ffeb8d0bb9547586b5", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "828938b08e41b742df3b8e77ff9997545b92f9efac9f68e820aedd32fa4ed9a0", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "c0e20dbf5eaa1a2470f432f84ce58675f5c061550d1f0b971ea372b1360ba9e1", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fc39ef06b93c4b9b0ccf89af188624e1f6c91e0241d7963b23d39198694fdfdf", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "6875d112abc5b8f51f2f1a2d1a760eee4efd2b9e4f28ba5a5793666de5bcc48a", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d2d1b6837d19466e1b9d7d9c866a7c44f063cdb89937e528177494c5c96f6cc4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d3e46f8aa00286df8f09154b483fbe787d3843dda5747d2d5082fdf70dc2c95", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ec0989feac111d3cdf0c2ba9f71bb1b88da47c829393a22ad79fac517b0aeb5e", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "69e4fe69d2f7e3c4e19b23e1138010bc2139799c50685e2ebe93db174d9960e9", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "2c0e11e47247527c3ee4c480f29c859f20841d21c29206bc6036a14c30194423", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7f7681bc1e69ab9124a7cec506d4370888d7d18a8949f4ab61403030f6a17951", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "68d6b2ab40dfb15eb1af46862d8474310305c111f2d7af35053dae4ac8341738", + "markers": [ + " -r " + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5a19f132dcb84775c298072b9b10764020c4aef4c5bc58809b13bd528ec524fa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb36dd1d24ac5cd3b87e68a0e8fe23334027825e2f54fed3c04a7be3d902a59d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "952bd1f3229402a503da3603fb81f2ab7480ebebfc83d698f478961e83ddb08f", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1d5c7d2db199056204a4647e18f36a19ffc32d7807afab758b9eb8565e3715cc", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "24a684162da36bf1d24db913393f84e9cd06568fb760f550e8524bee3a59a40d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "475644924650e7798cd0cb94d6957bc66f5a649fe3995c9e4f639091412b3c18", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "eb6966faec145ae9e4ddf3a9bfc45ee10af3d729d7b6bb706479dab3f8069959", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ], + "fp": [ + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "11fb0d28296fdc35b9992eb7c7e9a64a7be354b37b48a97e2b4387b7a6f8d428", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "covered", + "hits": [ + { + "line_sha256": "9d9267900b68e4d3277b6fe63564bd20d8fd0b4054e91af0fce84553b8b8ab6b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "67acd7e054b4d1a326b500669d0dca3bbb33d566fb88422eb062bb2eb911283e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "7c15957270550fa7831ee2a625e1e165c4e3f37208647d5f63caf02af96ff544", + "markers": [ + "../" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [ + { + "line_sha256": "919e32e4f394efcfec8bb0fc00a5a96b41d8ae767ae8445762aeb7214b474251", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "b238f1c5111829ad160b0528335ac56f63694fc4600a10bf235d6169e6d705bf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "b80ef6d46b17aedd3f66b286ffcb7a7ff32eb8f85c8c28ba11cd2932813b76d7", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e8880b032890ee3cb48ab4fce5bcbea666d62cdb0ec285070bf9aa44001388be", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "d7ac19a1fdc5ba5440fe3d3dc19c1f8cb26a258287a27706a50c093a3cffb870", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "09c36ea49b6375dc55c36b4f9a1f5574d636e2c75d7340946439f65098b73268", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ], + "improvement": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4581c043835af761f02f10ddacc67cd338cd7d9cc050ab41ad1d76eca63b8099", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "43875f510f7d4d2355ea7a1bbd06ce3a393d3cb5e31bb377b6bbad40b0d023c2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "6db06274942ca668070ccee3f005da1d2cc2f9cb908e4b2faeee02266b28aee1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc5e0932da413124af878ef15d712049afc6aedf5ca58d90111baabe01f28b9c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "de9ffa6055e0a2383967d3cbf194e3677e100d3b8cd2a94e1e19af29c83dafb9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4a23000128509bdf1a9dff7da14e195599c24a286e88f211089636ff81dcb463", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "c01dd2b41980c065e0a777d9d15a8e4e0e3ff0b3d64fcbcb1a9bc36cda03d565", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c00612992135e362b61ef9aa2c8840ca720851d362b5ca14ad59480746c78254", + "markers": [ + "../" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "be903554134a616a4f51158f58b4e3a6fa7c63ee7f792338eee51d14d562c019", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e7d59ebeae9764faf8602a13d1f56dcd3d82b82bd26e3e21c6aa7836f5a4e8ad", + "markers": [ + "HEAD~" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "bff3b5f05108e0e3bead6fdc57235ca733327489684a089019e7608712b2737b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e1973c0cba163fa32110a25fe567930fcdfe33c7032958599913efd0311d7898", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "642d39c1c8e8172f91e155f4ecaf711603bfadc4c61543df8cb6bfec3ac22c9d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "436ee100fd57903cf9e6eb66780804d032df8908bf0c33dbd78e5d69de1078b6", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "6bc9c7174b8708e37833c5fa3c06a6badf2620f825274205ddbbf27b6a29b4b9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "8c003bd8814eed70d32562a28c791afbcfafcd9c8c284e05718b2a3f04ea0356", + "markers": [ + "rev-parse", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "972096a15295e925f2e3cf87eee726b50fabd9ef05a627c24781c89d1daa86f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "679da91f949a511dbee6113010bc53b87533af6163b5fef3d71638f2803a5231", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1f0d006bf9e7caf5e7a47f03a5c6d89ee49e952bc50921c4f77293f757612b65", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "0fc03c2824d3dde62aefd151f0485e213a06e59d17821d84502a586763519ec5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "814f0f33cb379d7ac38340497f36e9195ce02d3f2de9d7ac57219278648acdea", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "031a2d49b3a82bc591181eee88488e1237d89f86b070e9914026376088a978cb", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b945e48385ff5ea9cf694e0f3f275a664d5d6ffeb4a7a83d6a0524836cd320ed", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "aee739303916771552df815506d15b8966cc3848ba41156b5a18e9fa1c8d3f11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7db0895e590ea7ecf2f5f4e4a960e883e47b14a532948b64fdc341d0aad21d62", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "9eb4e535ccab0a3db7afd4b81945d6618e017f48173ee821d15d2993d7cea36d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "79ef89a6f757830872163acb657afe333343b38d5f7bcb2a0f52ad5b28d6cd59", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "06c4414f2ba3ca58ffdd8b68e6594dc712a6de0c1a18d696fa36f05562a084fc", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "cc391d5c7315c5ee5895e7c4648ed7cab84686a0a577616f1a2896b1e043adf3", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60cce41f7825e3a471ff0276a7d6ce8c8d23d7452936dbda837dcc6fcddc638a", + "markers": [ + "-rn" + ], + "orientation_only": false + }, + { + "line_sha256": "27eeeb0397a46f1e0fc0f7cfd2e27fa5cb71b030d47d401629460ab83f82c659", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "dd849ca4a37bf4802b2e980a5f5a1ecec1bde2809fa5e2c9ae4462b5537eebaa", + "markers": [ + "git show HEAD", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2073d2e30017c6c0d3f45fac4c79b047ee85b22f738dadabe82b8da10e27f140", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "eefc1eb47dddd5e0c76f397a720c93e6cbabb0eb8cb191e267fffff59ca138a6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f8c2aaaaa9ad31c0c8496a39705bcec59fad5c1acdc87d8ca1280aec8d62d185", + "markers": [ + "HEAD~", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd9812cb38239143c14da974259380a76a5dcca7f30f7472aea3c5c728a76327", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3a48f5782a5b99f840d78e9108b5779b3f7e4da65719a8da0e0ee11b0b7e2c6e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3b298a3e9116ee16ca656adb3788f5b1c622968e55b0bbac384611c8780a06c2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "27b09e81480add37b82ec1e90fc091c3aa7d6531d14120dcb66356471ff905a0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3c365ebde7a7b9516774687d8c1f6db23e08fb4ef22403fd9b096d6e2e010e40", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "236a4cdb8f444afad1d920dc575956f38890768ce1dccfd95db2764b1b3c5f0f", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ffef8182975bcf1a8bf68c942c8ad74e6a0bf2d8955cbbc75ef83ee14766277a", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d165c839fb89e0cccb1f133424487a1765beca86386f68a46c6876408d0eef20", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60b823fc440ff4b309641567a2e51c65208cf2cbe0aca5ac87644632851e1926", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "932e76e69ddf9c66793543e58bf8f76ff4d571a48908b481b48a7bc2a146fed4", + "markers": [ + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "78b7d8375dacaa30cbd6915edd3f3d7ba7217c66d9c5969f31b557cc53dde711", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "831c08aa7dbe60925e452201bba20cc6d9ddcfe1baf738f2b7f8fe91cf183172", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "5d5ce7c4dd128ba5551eea81c15d63739a52d7b0c08a6b2f1d8df314a8f37667", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "f3d859fc65dc144719341882dd788a5a09f6372461936fa15a961f67ea360f41", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3330e7f41e67f7334081aabf48c0e2527f04cad55ba2348b5bd8edf5f33e270a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "406fc2aa4c793871d253d1f186bdb7949666edd60ddd2794bae18f4c50fea2b8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "20fc9ee9abd090f617c98093c06224ff938b2337f4bd9ee2b65732cf21bf7878", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "9356a1d8a8d8d5d98bbffc08bbd9261253ba9bb944871c4a590fdcf6673e8139", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1382c9aedc38269dd3adb840c0535836d775d252b85ca0e181e9f45bd6ae46fe", + "markers": [ + "rev-parse", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ad506db11fe1e5f12d1870c4e03120cf424bbb7dcdf7a59727f20616835b6ebe", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE", + "DOTDOT_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "47e685f03a4c622c078fb93098ef698a64424e0d653f537c30689f514889313b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc3c8e0736af923ddb6352a221a14b2aa2799e978a4023dd1e3677bc8479c7d5", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "dedf87ab4d5a344a6fa81f1744b6ed4020c4a357edb7c33f26260c69007346f6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "e32548de94b9b6a582b9f14a59eded3f913db7668802f7044f6b50d3e89ea872", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e3230c6c5073a314519f85a367485585743dc722a2fdc4838d48eb388d027ef0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ff969993a2523cfc41df3f56bc748392aff5654c077c7719d129654681b76ec8", + "markers": [ + "../", + "DOTDOT_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2c5e45dbe5528cfed07b9461e75b4d25f7f66a5b841b28072ab99c1e343ba8b3", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "decca74bfa41b6dbec7c7b4781d28d6f236093a4b8dc0635abcd619947717c40", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "f70a9944f4ebac984d6a738d3e3faacd74f8f32e5e3147c54bbb1a26f6e69f92", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "eee6b44b32b3942f6bf00eb8c4423d17fe2b257ebeaf810c4eece5bff8a1b1ee", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ab52c6965fadbafbcdb1fefaa11ccc759bc27c1985111b2c89dd90fef1512a0c", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1ef1848a5acc3f0a8f88dd9e5393583ad58bc56be70f617475404898547de547", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1cfbaf51506a24141cf75c07426866b69a1a1ccba434ff8df8d02201a3723ed5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "8296c2098c4f67cdca611e08631d9f617ec67f2012947fcae85284c48372ccf5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "630081281dfe68e8a7e95a0819d8bd67473640a54ab06f6eff2a6d453d915cbc", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a02a3ba8575d7c5708b82cc3ab08951b6d99f0383fa02aa37243e863624ae25f", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "88817ac4c3c77e9a120eac10af812d333f7360e92f7c36fce22a01b70b7ac27d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "745bca92a6317990f12ae0015cc860a2b2b9c87c75d3a40ffe2b1393db87b923", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d4d2bbcb3803d2a8c2a6276d64a163c606a68bc29ab06f0e819191353348c44d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb2317bf324a28dea4a564e79faaa41d77b2880c4988bcbb81cab3fc0ea1cd61", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "4004212ff6037092e741aefd6fff244d1c2f7a039094b40e107d1fb9dfac8243", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a7ae48ab626e6e00d4af7c6a0b93e29b15c275668ab485ba45c6122fd384c85d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3dbf3eb4cd1f6af99c22b6fa308fb5ae904f843a77aa11364439916f22c80f08", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fcfb17f721dcdd6c4b18ce747c256999d5eb0968fbb70f4e70b548a255fa9199", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "0f9e5909143e71137044eceb40e7357df8ee54c3a079fbc75e998b827838f601", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "50ca1d66942948ce263512134d0270fd7d9c3bc5fc7a96ffeb8d0bb9547586b5", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "828938b08e41b742df3b8e77ff9997545b92f9efac9f68e820aedd32fa4ed9a0", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "c0e20dbf5eaa1a2470f432f84ce58675f5c061550d1f0b971ea372b1360ba9e1", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fc39ef06b93c4b9b0ccf89af188624e1f6c91e0241d7963b23d39198694fdfdf", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "6875d112abc5b8f51f2f1a2d1a760eee4efd2b9e4f28ba5a5793666de5bcc48a", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d2d1b6837d19466e1b9d7d9c866a7c44f063cdb89937e528177494c5c96f6cc4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d3e46f8aa00286df8f09154b483fbe787d3843dda5747d2d5082fdf70dc2c95", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ec0989feac111d3cdf0c2ba9f71bb1b88da47c829393a22ad79fac517b0aeb5e", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "69e4fe69d2f7e3c4e19b23e1138010bc2139799c50685e2ebe93db174d9960e9", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "2c0e11e47247527c3ee4c480f29c859f20841d21c29206bc6036a14c30194423", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7f7681bc1e69ab9124a7cec506d4370888d7d18a8949f4ab61403030f6a17951", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "68d6b2ab40dfb15eb1af46862d8474310305c111f2d7af35053dae4ac8341738", + "markers": [ + " -r " + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5a19f132dcb84775c298072b9b10764020c4aef4c5bc58809b13bd528ec524fa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb36dd1d24ac5cd3b87e68a0e8fe23334027825e2f54fed3c04a7be3d902a59d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "952bd1f3229402a503da3603fb81f2ab7480ebebfc83d698f478961e83ddb08f", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1d5c7d2db199056204a4647e18f36a19ffc32d7807afab758b9eb8565e3715cc", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "24a684162da36bf1d24db913393f84e9cd06568fb760f550e8524bee3a59a40d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "475644924650e7798cd0cb94d6957bc66f5a649fe3995c9e4f639091412b3c18", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "eb6966faec145ae9e4ddf3a9bfc45ee10af3d729d7b6bb706479dab3f8069959", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "11fb0d28296fdc35b9992eb7c7e9a64a7be354b37b48a97e2b4387b7a6f8d428", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-changelog-prose", + "coverage": "covered", + "hits": [ + { + "line_sha256": "9d9267900b68e4d3277b6fe63564bd20d8fd0b4054e91af0fce84553b8b8ab6b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-changelog-prose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "67acd7e054b4d1a326b500669d0dca3bbb33d566fb88422eb062bb2eb911283e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "7c15957270550fa7831ee2a625e1e165c4e3f37208647d5f63caf02af96ff544", + "markers": [ + "../" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "coverage": "covered", + "hits": [ + { + "line_sha256": "919e32e4f394efcfec8bb0fc00a5a96b41d8ae767ae8445762aeb7214b474251", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "b238f1c5111829ad160b0528335ac56f63694fc4600a10bf235d6169e6d705bf", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "b80ef6d46b17aedd3f66b286ffcb7a7ff32eb8f85c8c28ba11cd2932813b76d7", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s3-wcr-pfloor-comment", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e8880b032890ee3cb48ab4fce5bcbea666d62cdb0ec285070bf9aa44001388be", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "d7ac19a1fdc5ba5440fe3d3dc19c1f8cb26a258287a27706a50c093a3cffb870", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "09c36ea49b6375dc55c36b4f9a1f5574d636e2c75d7340946439f65098b73268", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ], + "regression": [ + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4581c043835af761f02f10ddacc67cd338cd7d9cc050ab41ad1d76eca63b8099", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "43875f510f7d4d2355ea7a1bbd06ce3a393d3cb5e31bb377b6bbad40b0d023c2", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "6db06274942ca668070ccee3f005da1d2cc2f9cb908e4b2faeee02266b28aee1", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc5e0932da413124af878ef15d712049afc6aedf5ca58d90111baabe01f28b9c", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "de9ffa6055e0a2383967d3cbf194e3677e100d3b8cd2a94e1e19af29c83dafb9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "4a23000128509bdf1a9dff7da14e195599c24a286e88f211089636ff81dcb463", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-coef-dict-collision", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "c01dd2b41980c065e0a777d9d15a8e4e0e3ff0b3d64fcbcb1a9bc36cda03d565", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "c00612992135e362b61ef9aa2c8840ca720851d362b5ca14ad59480746c78254", + "markers": [ + "../" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "be903554134a616a4f51158f58b4e3a6fa7c63ee7f792338eee51d14d562c019", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e7d59ebeae9764faf8602a13d1f56dcd3d82b82bd26e3e21c6aa7836f5a4e8ad", + "markers": [ + "HEAD~" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "bff3b5f05108e0e3bead6fdc57235ca733327489684a089019e7608712b2737b", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s1-cs-dr-plugin-se", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e1973c0cba163fa32110a25fe567930fcdfe33c7032958599913efd0311d7898", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "642d39c1c8e8172f91e155f4ecaf711603bfadc4c61543df8cb6bfec3ac22c9d", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "436ee100fd57903cf9e6eb66780804d032df8908bf0c33dbd78e5d69de1078b6", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "6bc9c7174b8708e37833c5fa3c06a6badf2620f825274205ddbbf27b6a29b4b9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "8c003bd8814eed70d32562a28c791afbcfafcd9c8c284e05718b2a3f04ea0356", + "markers": [ + "rev-parse", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "972096a15295e925f2e3cf87eee726b50fabd9ef05a627c24781c89d1daa86f9", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "679da91f949a511dbee6113010bc53b87533af6163b5fef3d71638f2803a5231", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1f0d006bf9e7caf5e7a47f03a5c6d89ee49e952bc50921c4f77293f757612b65", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "0fc03c2824d3dde62aefd151f0485e213a06e59d17821d84502a586763519ec5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "814f0f33cb379d7ac38340497f36e9195ce02d3f2de9d7ac57219278648acdea", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "031a2d49b3a82bc591181eee88488e1237d89f86b070e9914026376088a978cb", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-acrt-single-dose", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "b945e48385ff5ea9cf694e0f3f275a664d5d6ffeb4a7a83d6a0524836cd320ed", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "aee739303916771552df815506d15b8966cc3848ba41156b5a18e9fa1c8d3f11", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "7db0895e590ea7ecf2f5f4e4a960e883e47b14a532948b64fdc341d0aad21d62", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "9eb4e535ccab0a3db7afd4b81945d6618e017f48173ee821d15d2993d7cea36d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "79ef89a6f757830872163acb657afe333343b38d5f7bcb2a0f52ad5b28d6cd59", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "06c4414f2ba3ca58ffdd8b68e6594dc712a6de0c1a18d696fa36f05562a084fc", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-cic-practitioner-screen", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "cc391d5c7315c5ee5895e7c4648ed7cab84686a0a577616f1a2896b1e043adf3", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60cce41f7825e3a471ff0276a7d6ce8c8d23d7452936dbda837dcc6fcddc638a", + "markers": [ + "-rn" + ], + "orientation_only": false + }, + { + "line_sha256": "27eeeb0397a46f1e0fc0f7cfd2e27fa5cb71b030d47d401629460ab83f82c659", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "dd849ca4a37bf4802b2e980a5f5a1ecec1bde2809fa5e2c9ae4462b5537eebaa", + "markers": [ + "git show HEAD", + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2073d2e30017c6c0d3f45fac4c79b047ee85b22f738dadabe82b8da10e27f140", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "eefc1eb47dddd5e0c76f397a720c93e6cbabb0eb8cb191e267fffff59ca138a6", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "f8c2aaaaa9ad31c0c8496a39705bcec59fad5c1acdc87d8ca1280aec8d62d185", + "markers": [ + "HEAD~", + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": true, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-had-cluster-guard", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fd9812cb38239143c14da974259380a76a5dcca7f30f7472aea3c5c728a76327", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3a48f5782a5b99f840d78e9108b5779b3f7e4da65719a8da0e0ee11b0b7e2c6e", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3b298a3e9116ee16ca656adb3788f5b1c622968e55b0bbac384611c8780a06c2", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "27b09e81480add37b82ec1e90fc091c3aa7d6531d14120dcb66356471ff905a0", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "3c365ebde7a7b9516774687d8c1f6db23e08fb4ef22403fd9b096d6e2e010e40", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "236a4cdb8f444afad1d920dc575956f38890768ce1dccfd95db2764b1b3c5f0f", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ffef8182975bcf1a8bf68c942c8ad74e6a0bf2d8955cbbc75ef83ee14766277a", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d165c839fb89e0cccb1f133424487a1765beca86386f68a46c6876408d0eef20", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "60b823fc440ff4b309641567a2e51c65208cf2cbe0aca5ac87644632851e1926", + "markers": [ + "GIT_DIR_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s2-nan-dof-fail-closed", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "932e76e69ddf9c66793543e58bf8f76ff4d571a48908b481b48a7bc2a146fed4", + "markers": [ + "../", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "489459885bf6c841d581dbfda5e833865b8b402a358a4f68438b494087efba4d", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "78b7d8375dacaa30cbd6915edd3f3d7ba7217c66d9c5969f31b557cc53dde711", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "831c08aa7dbe60925e452201bba20cc6d9ddcfe1baf738f2b7f8fe91cf183172", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "5d5ce7c4dd128ba5551eea81c15d63739a52d7b0c08a6b2f1d8df314a8f37667", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "f3d859fc65dc144719341882dd788a5a09f6372461936fa15a961f67ea360f41", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3330e7f41e67f7334081aabf48c0e2527f04cad55ba2348b5bd8edf5f33e270a", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "406fc2aa4c793871d253d1f186bdb7949666edd60ddd2794bae18f4c50fea2b8", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "20fc9ee9abd090f617c98093c06224ff938b2337f4bd9ee2b65732cf21bf7878", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "9356a1d8a8d8d5d98bbffc08bbd9261253ba9bb944871c4a590fdcf6673e8139", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1382c9aedc38269dd3adb840c0535836d775d252b85ca0e181e9f45bd6ae46fe", + "markers": [ + "rev-parse", + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ad506db11fe1e5f12d1870c4e03120cf424bbb7dcdf7a59727f20616835b6ebe", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE", + "DOTDOT_RE", + "GIT_DIR_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "47e685f03a4c622c078fb93098ef698a64424e0d653f537c30689f514889313b", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "bc3c8e0736af923ddb6352a221a14b2aa2799e978a4023dd1e3677bc8479c7d5", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "dedf87ab4d5a344a6fa81f1744b6ed4020c4a357edb7c33f26260c69007346f6", + "markers": [ + "--format", + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "e32548de94b9b6a582b9f14a59eded3f913db7668802f7044f6b50d3e89ea872", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "e3230c6c5073a314519f85a367485585743dc722a2fdc4838d48eb388d027ef0", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ff969993a2523cfc41df3f56bc748392aff5654c077c7719d129654681b76ec8", + "markers": [ + "../", + "DOTDOT_RE", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-cs-unbalanced-nan-outcome", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "2c5e45dbe5528cfed07b9461e75b4d25f7f66a5b841b28072ab99c1e343ba8b3", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "decca74bfa41b6dbec7c7b4781d28d6f236093a4b8dc0635abcd619947717c40", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "f70a9944f4ebac984d6a738d3e3faacd74f8f32e5e3147c54bbb1a26f6e69f92", + "markers": [ + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "eee6b44b32b3942f6bf00eb8c4423d17fe2b257ebeaf810c4eece5bff8a1b1ee", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "ab52c6965fadbafbcdb1fefaa11ccc759bc27c1985111b2c89dd90fef1512a0c", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "1ef1848a5acc3f0a8f88dd9e5393583ad58bc56be70f617475404898547de547", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "1cfbaf51506a24141cf75c07426866b69a1a1ccba434ff8df8d02201a3723ed5", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "939df9e90b1fd0dd614277f53754beb6d922ab69239419ae287dae2016de94e5", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "8296c2098c4f67cdca611e08631d9f617ec67f2012947fcae85284c48372ccf5", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "630081281dfe68e8a7e95a0819d8bd67473640a54ab06f6eff2a6d453d915cbc", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a02a3ba8575d7c5708b82cc3ab08951b6d99f0383fa02aa37243e863624ae25f", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "88817ac4c3c77e9a120eac10af812d333f7360e92f7c36fce22a01b70b7ac27d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "745bca92a6317990f12ae0015cc860a2b2b9c87c75d3a40ffe2b1393db87b923", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "d4d2bbcb3803d2a8c2a6276d64a163c606a68bc29ab06f0e819191353348c44d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb2317bf324a28dea4a564e79faaa41d77b2880c4988bcbb81cab3fc0ea1cd61", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "4004212ff6037092e741aefd6fff244d1c2f7a039094b40e107d1fb9dfac8243", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "a7ae48ab626e6e00d4af7c6a0b93e29b15c275668ab485ba45c6122fd384c85d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "3dbf3eb4cd1f6af99c22b6fa308fb5ae904f843a77aa11364439916f22c80f08", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-lpdid-survey-unreduced-design", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "fcfb17f721dcdd6c4b18ce747c256999d5eb0968fbb70f4e70b548a255fa9199", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "0f9e5909143e71137044eceb40e7357df8ee54c3a079fbc75e998b827838f601", + "markers": [ + "worktrees/", + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "50ca1d66942948ce263512134d0270fd7d9c3bc5fc7a96ffeb8d0bb9547586b5", + "markers": [ + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "828938b08e41b742df3b8e77ff9997545b92f9efac9f68e820aedd32fa4ed9a0", + "markers": [ + "GIT_LOG_RE", + "REF_LIST_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "c0e20dbf5eaa1a2470f432f84ce58675f5c061550d1f0b971ea372b1360ba9e1", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "fc39ef06b93c4b9b0ccf89af188624e1f6c91e0241d7963b23d39198694fdfdf", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "6875d112abc5b8f51f2f1a2d1a760eee4efd2b9e4f28ba5a5793666de5bcc48a", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "d2d1b6837d19466e1b9d7d9c866a7c44f063cdb89937e528177494c5c96f6cc4", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "B", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "8d3e46f8aa00286df8f09154b483fbe787d3843dda5747d2d5082fdf70dc2c95", + "markers": [ + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "ec0989feac111d3cdf0c2ba9f71bb1b88da47c829393a22ad79fac517b0aeb5e", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "69e4fe69d2f7e3c4e19b23e1138010bc2139799c50685e2ebe93db174d9960e9", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "2c0e11e47247527c3ee4c480f29c859f20841d21c29206bc6036a14c30194423", + "markers": [ + "-maxdepth" + ], + "orientation_only": false + }, + { + "line_sha256": "7f7681bc1e69ab9124a7cec506d4370888d7d18a8949f4ab61403030f6a17951", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "68d6b2ab40dfb15eb1af46862d8474310305c111f2d7af35053dae4ac8341738", + "markers": [ + " -r " + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 1 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "5a19f132dcb84775c298072b9b10764020c4aef4c5bc58809b13bd528ec524fa", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "cb36dd1d24ac5cd3b87e68a0e8fe23334027825e2f54fed3c04a7be3d902a59d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "952bd1f3229402a503da3603fb81f2ab7480ebebfc83d698f478961e83ddb08f", + "markers": [ + "tools/reviewer-eval/", + "repo_root" + ], + "orientation_only": false + } + ], + "orientation_only": false, + "repeat": 0 + }, + { + "arm": "F", + "case": "s4-wcr-saturated-guard-order", + "coverage": "unaudited", + "hits": [ + { + "line_sha256": "1d5c7d2db199056204a4647e18f36a19ffc32d7807afab758b9eb8565e3715cc", + "markers": [ + "../", + "GIT_LOG_RE", + "DOTDOT_RE" + ], + "orientation_only": false + }, + { + "line_sha256": "24a684162da36bf1d24db913393f84e9cd06568fb760f550e8524bee3a59a40d", + "markers": [ + "repo_root" + ], + "orientation_only": false + }, + { + "line_sha256": "475644924650e7798cd0cb94d6957bc66f5a649fe3995c9e4f639091412b3c18", + "markers": [ + "--format", + "GIT_LOG_RE" + ], + "orientation_only": true + }, + { + "line_sha256": "eb6966faec145ae9e4ddf3a9bfc45ee10af3d729d7b6bb706479dab3f8069959", + "markers": [ + "BARE_SHOW_RE" + ], + "orientation_only": true + } + ], + "orientation_only": false, + "repeat": 1 + } + ] + }, + "improvements": [], + "informational": { + "B": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + }, + "F": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + } + }, + "judgment_flags": [ + { + "bug": "s2-acrt-single-dose|s2-acrt-single-dose:b1", + "kind": "b_unstable_e_missed" + } + ], + "label": "PARITY (DEFINITIVE)", + "regressions": [], + "verdict": "PARITY", + "voided_cells": [] +} + +VERDICT: PARITY (DEFINITIVE) diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-provisional.txt b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-provisional.txt new file mode 100644 index 00000000..18019204 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verdict-provisional.txt @@ -0,0 +1,112 @@ +{ + "audit_vacuous": null, + "campaign_reads": { + "cost_estimate_usd": { + "B": { + "input_price_per_1m": 4.0, + "model": "gpt-5.6-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 1.259472 + }, + "F": { + "input_price_per_1m": 2.0, + "model": "gpt-6.1-sol", + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + "prompt_tokens_est": 314868, + "usd": 0.629736 + } + }, + "latency": { + "B": { + "max": 472.4614837919362, + "mean": 213.33513880114663, + "n": 22, + "p95": 413.60439145797864 + }, + "F": { + "max": 416.4505883329548, + "mean": 180.63859337303703, + "n": 22, + "p95": 276.9192527080886 + } + }, + "latency_flag_candidate_p95_over_2x_control": false, + "vendor_lifecycle": "operator_supplied" + }, + "candidate": "F", + "catch_status": { + "s1-coef-dict-collision|s1-coef-dict-collision:b1|B": "reliable", + "s1-coef-dict-collision|s1-coef-dict-collision:b1|F": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|B": "reliable", + "s1-cs-dr-plugin-se|s1-cs-dr-plugin-se:b1|F": "reliable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|B": "unstable", + "s2-acrt-single-dose|s2-acrt-single-dose:b1|F": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|B": "missed", + "s2-cic-practitioner-screen|s2-cic-practitioner-screen:b1|F": "missed", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|B": "reliable", + "s2-had-cluster-guard|s2-had-cluster-guard:b1|F": "reliable", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|B": "missed", + "s2-nan-dof-fail-closed|s2-nan-dof-fail-closed:b1|F": "unstable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|B": "reliable", + "s4-cs-unbalanced-nan-outcome|s4-cs-unbalanced-nan-outcome:b1|F": "reliable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|B": "unstable", + "s4-lpdid-survey-unreduced-design|s4-lpdid-survey-unreduced-design:b1|F": "reliable", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b1|F": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|B": "missed", + "s4-wcr-saturated-guard-order|s4-wcr-saturated-guard-order:b2|F": "missed" + }, + "contested": [], + "contested_gates": [], + "control": "B", + "definitive": false, + "downgrades": [], + "evidence_gaps": [], + "exposure": { + "first_subdir": "gpt61sol", + "frozen_head": "e1674c0e2a138f63300696f2b967268920c7e07b", + "head_changed": false, + "ledger_entries": 1, + "mechanical": true, + "reason": "first dispatch of the contrast (frozen revision)", + "subdirs": [ + "gpt61sol" + ], + "surface_diff_stat": "", + "this_subdir": "gpt61sol", + "timeout_only": false + }, + "fp_candidate": 0, + "fp_control": 0, + "fp_findings": { + "B": [], + "F": [] + }, + "gate_cells": null, + "improvements": [], + "informational": { + "B": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + }, + "F": { + "hallucinations": 22, + "leaks": 0, + "unconfirmed_flags": 0 + } + }, + "judgment_flags": [ + { + "bug": "s2-acrt-single-dose|s2-acrt-single-dose:b1", + "kind": "b_unstable_e_missed" + } + ], + "label": "PARITY (PROVISIONAL, no session-log audit)", + "regressions": [], + "verdict": "PARITY", + "voided_cells": [] +} + +VERDICT: PARITY (PROVISIONAL, no session-log audit) diff --git a/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verifier.json b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verifier.json new file mode 100644 index 00000000..380291b4 --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-10-gpt61sol-grading/verifier.json @@ -0,0 +1,410 @@ +{ + "pass1": { + "pass": 1, + "rulings": [ + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 0, + "verdict": "partial", + "evidence": "### P1 \u2014 Mean parallel trends is not necessary for CiC ... **Impact:** The guidance says a mean-trend break undermines CiC\u2019s time-invariance assumption. The Registry\u2019s model allows different group distributions of U and arbitrary strictly increasing time transformations; these can produce different mean trends while satisfying identification. ... **Concrete fix:** Remove the necessity claim and \u201cundermines\u201d interpretation for CiC. Present mean comparisons as descriptive and use the distributional placebo for model-relevant guidance.", + "leak": "none", + "leak_evidence": "", + "reason": "The ground truth is that the shipped QDiD text understates the mean-trend screen's evidential value (under QDiD's additive model a mean-trend break IS direct evidence against the model) while claiming QDiD does not identify off mean parallel trends. This review attacks the same screen at the same location (L1534-L1539 within `_handle_cic`) but in the opposite direction: it argues mean PT is NOT necessary for CiC and should be demoted to descriptive. It never mentions QDiD's additive model, the 'does not identify off mean PT' claim, or that the text understates the check, so it is a different defect at the same location." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M1", + "repeat": 1, + "verdict": "partial", + "evidence": "**P1 \u2014 Remove the necessary mean-parallel-trends requirement for CiC.** Location: `diff_diff/practitioner.py:L1534-L1539`, `L1552-L1558`. **Impact:** CiC permits different untreated mean trends across groups. ... **Concrete fix:** Remove the necessary-condition claim and shared snippet comment for CiC. Recommend the distributional placebo already supplied, or describe mean trends as descriptive only.", + "leak": "none", + "leak_evidence": "", + "reason": "Same screen, same location, but again only the 'not necessary for CiC' direction; the review wants the necessity claim removed rather than strengthened for QDiD. Its separate P1 'Include QDiD\u2019s additive-separability assumption' (L1401-L1407, L1514-L1525) notes the assumption text omits `Y\u1d3a=h\u1d33(U,G)+h\u1d40(U,T)` but never links the additive model to the mean-trend screen or says the shipped QDiD text mis-states the evidential value of a mean-trend break. The ground-truth mis-framing of the identification argument is not named." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 0, + "verdict": "partial", + "evidence": "### P1 \u2014 Mean parallel trends is not a necessary CiC screen **Location:** `diff_diff/practitioner.py:L1528-L1558` **Impact:** The new handler says a pre-period mean-trend break undermines CiC time invariance and labels `check_parallel_trends()` a \u201cnecessary\u201d screen. ... **Concrete fix:** Remove the mean-parallel-trends recommendation from the CiC branch and rely on the distributional two-pre-period placebo. If retained for QDiD, explain why QDiD\u2019s stronger additive model implies it and pass an explicit pre-period list.", + "leak": "none", + "leak_evidence": "", + "reason": "Right location and right screen, but the finding is that the screen is wrongly applied to CiC. The only QDiD remark is a conditional aside in the fix ('If retained for QDiD, explain why QDiD\u2019s stronger additive model implies it'), which treats the QDiD screen as acceptable to retain; it does not say the shipped QDiD text claims QDiD does not identify off mean parallel trends, nor that the 'necessary-but-insufficient' framing understates a mean-trend break as direct evidence against the additive model. Per the verifier criterion, a 'would justify' remark is not the ground-truth defect." + }, + { + "case": "s2-cic-practitioner-screen", + "bug_id": "s2-cic-practitioner-screen:b1", + "arm": "M2", + "repeat": 1, + "verdict": "partial", + "evidence": "### P1 \u2014 Mean parallel trends is not a necessary CiC screen - **Location:** `diff_diff/practitioner.py:L1528-L1558` - **Impact:** The CiC branch calls a mean-trend comparison \u201cnecessary\u201d and says a mean-trend break undermines time invariance. ... - **Concrete fix:** Remove the generic mean-parallel-trends recommendation from the CiC branch, or label it purely descriptive and explicitly state that failure does not violate CiC. Use the distributional two-pre-period placebo already provided as the relevant screen. QDiD may retain a mean screen because its additive model implies common mean changes.", + "leak": "none", + "leak_evidence": "", + "reason": "Same pattern: the P1 is the CiC-direction objection at the right location, and the closing sentence 'QDiD may retain a mean screen because its additive model implies common mean changes' endorses rather than criticises the shipped QDiD screen. Its separate P1 'QDiD\u2019s identifying model is incomplete' concerns omitted additive-separability/monotonicity in the assumption text, not the evidential value of a mean-trend break, and the snippet P1 is about `pre_periods=`. No finding states that the shipped QDiD text mis-states or understates what a mean-trend break implies, which is the ground-truth defect." + } + ] + }, + "pass2": { + "pass": 2, + "leak_rulings": [ + { + "case": "s1-coef-dict-collision", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-coef-dict-collision", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-coef-dict-collision", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Cites CHANGELOG.md:L11 and REGISTRY.md:L2551-L2554 as repository docs describing the skip; in-repo files, not refs or the answer key." + }, + { + "case": "s1-coef-dict-collision", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Only a pytest command (`pytest -q -s -p no:cacheprovider tests/...`); no git command, no commit-message citation." + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "REGISTRY.md:L545-L580 cited as recording that the ddof=1 plug-in was removed; in-repo documentation, not a leak source." + }, + { + "case": "s1-cs-dr-plugin-se", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-acrt-single-dose", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Absolute paths under tools/reviewer-eval/runs/.worktrees// are the reviewer's own worktree (exempt); the 'D in {0,1}, ACRT = ATT' statement is sourced from docs/methodology/continuous-did.md:L186-L205, an in-repo file." + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Prose mentions `git diff --check` passing; a working-tree whitespace check naming no ref, range or main/master - not a confirmation." + }, + { + "case": "s2-cic-practitioner-screen", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-had-cluster-guard", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s2-nan-dof-fail-closed", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Mentions 'The previous attribution concern is resolved' - a re-review framing, not a source citation; no git command or commit-message reference." + }, + { + "case": "s3-changelog-prose", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Same 'prior attribution concern is resolved' framing; no confirming evidence." + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-changelog-prose", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s3-wcr-pfloor-comment", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Heading 'Balance detection counts missing outcomes as observed' paraphrases the rationale but is well under 12 words and not verbatim." + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-cs-unbalanced-nan-outcome", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "Quotes the code fragment `response - design @ coef`, which the rationale also quotes; code fragments from the diff never count, and the surrounding prose is not a verbatim rationale sentence." + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-lpdid-survey-unreduced-design", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Only a pytest command (`pytest -s -p no:cacheprovider tests/test_lpdid.py -q`); no git command." + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M1", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M1", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "Fenced python reproduction block contains only numpy/diff_diff calls; no marker-list entries." + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M2", + "repeat": 0, + "leak": "none", + "leak_evidence": "", + "note": "" + }, + { + "case": "s4-wcr-saturated-guard-order", + "arm": "M2", + "repeat": 1, + "leak": "none", + "leak_evidence": "", + "note": "" + } + ], + "grade_corrections": [], + "s3_corrections": [] + } +} \ No newline at end of file diff --git a/tools/reviewer-eval/config/configs.json b/tools/reviewer-eval/config/configs.json index e07d97ed..ac6807c9 100644 --- a/tools/reviewer-eval/config/configs.json +++ b/tools/reviewer-eval/config/configs.json @@ -1,5 +1,5 @@ { - "_comment": "Reviewer arms. The 2026-07 gpt-5.5 -> gpt-5.6 evaluation ran with A as control; after the GO verdict (66-run blinded 4-arm campaign, campaigns/2026-07-gpt56-VERDICT.md) gpt-5.6-sol @ xhigh became production, so B carries role=control and A is retained as the previous-production arm for future back-comparisons. The 2026-10 campaign (campaigns/2026-10-gpt6-VERDICT.md, DECISION_RULE.md) compares B (control) against E (gpt-6-astra @ xhigh, the candidate); E's FIRST corpus run is the campaign itself (pre-registration by construction). treatment_fields declares which Config fields are the experimental treatment; every other confound (sandbox / action_version / cli_version, plus effort when not declared) is pinned and asserted identical across arms, and every selected multi-arm subset must decompose into single-field contrasts (see eval_core.runner). cli_version re-pinned to codex-cli 0.162.1 on 2026-10-16 (one CLI serves all arms and the runner asserts the live `codex --version` matches): B @ xhigh verified by harness smoke on 0.162.1; E = gpt-6-astra @ xhigh accepted by a `codex exec` probe on 0.162.1 (the exact argv `_build_codex_cmd` emits, under an isolated CODEX_HOME); A, C and D are historical 2026-07 arms NOT re-verified on 0.162.1 (D's `max` included; an unsupported effort fails closed as INFRA_ERROR). Astra off-corpus latency probe (2026-10-16, codex-cli 0.162.1, isolated CODEX_HOME, production openai_review.py on merged PR #841, 554 changed lines, effort xhigh): real = 324.57 s wall-clock, 14 shell commands, 26 web-search events; below the 1800 s threshold, so CALL_TIMEOUT_S stays 3600. Contrast check: B -> E differ in `model` only.", + "_comment": "Reviewer arms. The 2026-07 gpt-5.5 -> gpt-5.6 evaluation ran with A as control; after the GO verdict (66-run blinded 4-arm campaign, campaigns/2026-07-gpt56-VERDICT.md) gpt-5.6-sol @ xhigh became production, so B carries role=control and A is retained as the previous-production arm for future back-comparisons. The 2026-10 campaign (campaigns/2026-10-gpt6-VERDICT.md, DECISION_RULE.md) compares B (control) against E (gpt-6-astra @ xhigh, the candidate); E's FIRST corpus run is the campaign itself (pre-registration by construction). treatment_fields declares which Config fields are the experimental treatment; every other confound (sandbox / action_version / cli_version, plus effort when not declared) is pinned and asserted identical across arms, and every selected multi-arm subset must decompose into single-field contrasts (see eval_core.runner). cli_version re-pinned to codex-cli 0.162.1 on 2026-10-16 (one CLI serves all arms and the runner asserts the live `codex --version` matches): B @ xhigh verified by harness smoke on 0.162.1; E = gpt-6-astra @ xhigh accepted by a `codex exec` probe on 0.162.1 (the exact argv `_build_codex_cmd` emits, under an isolated CODEX_HOME); A, C and D are historical 2026-07 arms NOT re-verified on 0.162.1 (D's `max` included; an unsupported effort fails closed as INFRA_ERROR). Astra off-corpus latency probe (2026-10-16, codex-cli 0.162.1, isolated CODEX_HOME, production openai_review.py on merged PR #841, 554 changed lines, effort xhigh): real = 324.57 s wall-clock, 14 shell commands, 26 web-search events; below the 1800 s threshold, so CALL_TIMEOUT_S stays 3600. Contrast check: B -> E differ in `model` only. Campaign 2 (2026-10-17, DECISION_RULE.md; campaign 1's rule archived as campaigns/2026-10-gpt6-DECISION_RULE.md, mechanical NO-GO on one regression) compares B (control, re-sampled) against F (gpt-6.1-sol @ xhigh): F accepted by the same `codex exec` probe shape on 0.162.1 under the ChatGPT-account login (gpt-6-sol is accepted too on 0.162.1; the 0.153.4 rejection no longer holds), F's FIRST corpus run is campaign 2. Contrast check: B -> F differ in `model` only. gpt-6.1-sol off-corpus latency probe (2026-10-17, codex-cli 0.162.1, isolated CODEX_HOME, production openai_review.py on merged PR #841, same diff as the astra probe, effort xhigh): real = 260.77 s wall-clock, 15 shell commands (15/15 parsed), 28 web-search events; below the 1800 s threshold, so CALL_TIMEOUT_S stays 3600. OUTCOMES (2026-10-17): campaign 1 (B vs E, runs/gpt6, frozen 225a4afb, tag campaign-gpt6) = mechanical DEFINITIVE NO-GO on one regression (s4-cs-unbalanced-nan-outcome), user accepted; campaign 2 (B vs F, runs/gpt61sol, frozen e1674c0e, tag campaign-gpt61sol) = mechanical DEFINITIVE PARITY, user accepted as GO; gpt-6.1-sol @ xhigh became production (CI + local + plan-review), so F carries role=control and B is retained as the previous-production arm. See campaigns/2026-10-gpt6-VERDICT.md and campaigns/2026-10-gpt61sol-VERDICT.md. The candidate-exposure gate exempts the role=control arm (production is measured by definition), so bare `smoke` keeps working after a swap.", "treatment_fields": [ "model", "effort" @@ -16,13 +16,12 @@ }, { "id": "B", - "role": "control", "model": "gpt-5.6-sol", "effort": "xhigh", "sandbox": "read-only", "action_version": "v1", "cli_version": "codex-cli 0.162.1", - "label": "control / current production" + "label": "previous production (pre-2026-10 swap); control of the 2026-10 campaigns" }, { "id": "C", @@ -49,7 +48,17 @@ "sandbox": "read-only", "action_version": "v1", "cli_version": "codex-cli 0.162.1", - "label": "candidate: GPT-6 frontier (2026-10 campaign)" + "label": "candidate of 2026-10 campaign 1 (gpt-6-astra): mechanical NO-GO, accepted" + }, + { + "id": "F", + "model": "gpt-6.1-sol", + "effort": "xhigh", + "sandbox": "read-only", + "action_version": "v1", + "cli_version": "codex-cli 0.162.1", + "label": "control / current production (2026-10 campaign 2: PARITY, accepted as GO)", + "role": "control" } ] } diff --git a/tools/reviewer-eval/reviewer_verdict.py b/tools/reviewer-eval/reviewer_verdict.py index 698cb9ae..8097daa7 100644 --- a/tools/reviewer-eval/reviewer_verdict.py +++ b/tools/reviewer-eval/reviewer_verdict.py @@ -41,7 +41,7 @@ # The ONE pre-registered gating contrast (control, candidate). A PR-2 swap does # not change it; a future campaign registers its own pair in its own PR-1. -REGISTERED_CONTRASTS = (("B", "E"),) +REGISTERED_CONTRASTS = (("B", "E"), ("B", "F")) _VERDICT_VOCAB = ("caught", "partial", "missed") _SEVERITY_VOCAB = ("P0", "P1", "P2", "P3") diff --git a/tools/reviewer-eval/run_eval.py b/tools/reviewer-eval/run_eval.py index bcd7067d..44adc9ab 100644 --- a/tools/reviewer-eval/run_eval.py +++ b/tools/reviewer-eval/run_eval.py @@ -25,9 +25,9 @@ Usage: python tools/reviewer-eval/run_eval.py verify-corpus python tools/reviewer-eval/run_eval.py smoke # smokes the role=control arm - python tools/reviewer-eval/run_eval.py run --subdir gpt6 --configs B,E --k 2 - python tools/reviewer-eval/run_eval.py compare --subdir gpt6 --blinded - python tools/reviewer-eval/run_eval.py audit --subdir gpt6 -o /hits.json + python tools/reviewer-eval/run_eval.py run --subdir gpt61sol --configs B,F --k 2 + python tools/reviewer-eval/run_eval.py compare --subdir gpt61sol --blinded + python tools/reviewer-eval/run_eval.py audit --subdir gpt61sol -o /hits.json python tools/reviewer-eval/run_eval.py verdict --subdir gpt6 --grades \\ --control B --candidate E [--audit-hits /hits.json] [--expect-head ] """ @@ -340,7 +340,11 @@ def _candidate_exposure_violations( import reviewer_verdict selected = {c.id for c in configs} - cands = selected & _registered_candidates() + # The role=control arm is production: it is measured by definition, so a + # former candidate promoted by a swap (2026-10: F) stays smokeable and may + # run outside its registered contrast; every OTHER registered candidate is + # confined to its campaign shape. + cands = (selected & _registered_candidates()) - {_control_id()} if not cands: return [] if command == "smoke": @@ -1745,7 +1749,7 @@ def _build_parser() -> argparse.ArgumentParser: ps.set_defaults(func=cmd_smoke) pr = sub.add_parser("run", help="full comparison matrix; saves raw reviews") - pr.add_argument("--configs", default="B,E") + pr.add_argument("--configs", default="B,F") pr.add_argument("--strata", nargs="*", default=None) pr.add_argument("--subdir", default="full") pr.add_argument(