From d8ff09bca97a29627367bbd6c1ab958f01435be4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 5 Sep 2026 12:57:35 -0400 Subject: [PATCH 1/2] Bind judge provenance to the verdict it describes Peer review of #164 (gate f4abb02c) found that re-judging a case through the Codex runner could keep the Claude runner's provenance sidecar: prepare_audit removed verdict.json but left verdict.meta.json, the Codex runner never touched the sidecar, and the freezer preferred it over codex.log. A later freeze would have miscounted judges. - prepare_audit unlinks verdict.meta.json together with a stale verdict. - Both runners write verdict.meta.json with the verdict's sha256 and the runner name (the Codex runner now writes one too, with the model from its log header), and remove any leftover sidecar before judging. - The freezer counts a sidecar only when its hash matches the case's current verdict.json; otherwise it reads codex.log, else "unknown". - scripts/backfill_verdict_provenance.py binds existing sidecars (only when the recorded judging time sits within tolerance of the verdict file's mtime) and writes Codex sidecars from codex.log. Run on the audit tree: 350 bound, 318 written; the tally reproduces the frozen manifest (350 Claude Opus 5, 318 GPT-5.6 Sol). - Tests: sidecar binding rules; prepare_audit cleanup; a cross-runner re-judge through both runner scripts with fake CLIs. Co-Authored-By: Claude Fable 5.1 --- policybench/audit.py | 3 + scripts/backfill_verdict_provenance.py | 98 ++++++++++++ scripts/freeze_snapshot.py | 53 ++++-- scripts/run_audit_claude.sh | 12 +- scripts/run_audit_codex.sh | 44 ++++- tests/test_audit.py | 3 + tests/test_judge_provenance.py | 213 +++++++++++++++++++++++++ 7 files changed, 406 insertions(+), 20 deletions(-) create mode 100644 scripts/backfill_verdict_provenance.py create mode 100644 tests/test_judge_provenance.py diff --git a/policybench/audit.py b/policybench/audit.py index b53e111..0c947b1 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -416,6 +416,9 @@ def prepare_audit( and prompt_path.read_text() != new_prompt ): verdict_path.unlink() + # The provenance sidecar describes that verdict; a re-judge by + # the other runner must not inherit it. + (case_dir / "verdict.meta.json").unlink(missing_ok=True) prompt_path.write_text(new_prompt) return cases diff --git a/scripts/backfill_verdict_provenance.py b/scripts/backfill_verdict_provenance.py new file mode 100644 index 0000000..de59fa5 --- /dev/null +++ b/scripts/backfill_verdict_provenance.py @@ -0,0 +1,98 @@ +"""Bind existing judge-provenance sidecars to the verdicts they describe. + +Runners now write ``verdict.meta.json`` with the verdict's sha256 so the +freezer can tell a current sidecar from one left behind by a re-judged case. +Sidecars written before that field existed are bound here, and Codex-judged +cases that predate Codex sidecars get one from the ``codex.log`` header. A +sidecar is only bound when its recorded judging time sits within +``--tolerance`` seconds of the verdict file's modification time, so a sidecar +that cannot be shown to belong to the current verdict is left alone (the +freezer then falls back to ``codex.log`` or counts the case as unknown). + + uv run python scripts/backfill_verdict_provenance.py [--dry-run] [cases_dir] +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from freeze_snapshot import AUDIT_CASES_DIR # noqa: E402 + + +def _utc(ts: float) -> str: + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).isoformat() + + +def backfill(cases_dir: Path, *, tolerance: float, dry_run: bool) -> dict[str, int]: + counts = {"bound": 0, "codex_sidecar_written": 0, "left_alone": 0, "already": 0} + for case_dir in sorted(cases_dir.iterdir()): + verdict_path = case_dir / "verdict.json" + if not verdict_path.is_file(): + continue + digest = hashlib.sha256(verdict_path.read_bytes()).hexdigest() + verdict_mtime = verdict_path.stat().st_mtime + meta_path = case_dir / "verdict.meta.json" + if meta_path.is_file(): + meta = json.loads(meta_path.read_text()) + if meta.get("verdict_sha256"): + counts["already"] += 1 + continue + judged_at = datetime.datetime.fromisoformat( + meta["judged_at_utc"] + ).timestamp() + if abs(judged_at - verdict_mtime) > tolerance: + counts["left_alone"] += 1 + continue + meta["verdict_sha256"] = digest + meta.setdefault("judge_runner", "scripts/run_audit_claude.sh") + if not dry_run: + meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True)) + counts["bound"] += 1 + continue + codex_log = case_dir / "codex.log" + if codex_log.is_file(): + match = re.search( + r"^model: (\S+)$", + codex_log.read_text(encoding="utf-8", errors="replace"), + re.M, + ) + if not match: + counts["left_alone"] += 1 + continue + meta = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": digest, + "judge_model_requested": "default", + "judge_model_reported": [match.group(1)], + "judged_at_utc": _utc(verdict_mtime), + "backfilled_from": "codex.log", + } + if not dry_run: + meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True)) + counts["codex_sidecar_written"] += 1 + continue + counts["left_alone"] += 1 + return counts + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("cases_dir", nargs="?", default=str(AUDIT_CASES_DIR)) + parser.add_argument("--tolerance", type=float, default=600.0) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + counts = backfill( + Path(args.cases_dir), tolerance=args.tolerance, dry_run=args.dry_run + ) + print(json.dumps(counts)) + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index f89479a..aab18a9 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -162,14 +162,34 @@ } +def verdict_provenance(case_dir: Path) -> dict | None: + """The ``verdict.meta.json`` sidecar for a case, if it describes the + case's current ``verdict.json``. + + Both runners write the sidecar with the verdict's sha256. A sidecar whose + hash does not match (or that predates the hash field) belongs to an + earlier verdict, most likely one the other runner has since replaced, and + is ignored so a re-judged case cannot keep stale provenance. + """ + meta_path = case_dir / "verdict.meta.json" + verdict_path = case_dir / "verdict.json" + if not meta_path.is_file() or not verdict_path.is_file(): + return None + meta = json.loads(meta_path.read_text()) + if meta.get("verdict_sha256") != sha256_file(verdict_path): + return None + return meta + + def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: """Tally which judge model produced each case verdict in the audit tree. - A case with a ``verdict.meta.json`` sidecar was judged (or re-judged) by - the Claude Code runner, which records the judge model it requested; the - Codex runner records its model in the ``model:`` line of ``codex.log``. - A case with neither is counted under ``unknown`` so the manifest cannot - silently claim provenance it does not have. + Each runner writes a ``verdict.meta.json`` sidecar bound to its verdict by + sha256 (the model requested and the model the CLI reported); only a + sidecar that matches the case's current verdict counts. Cases judged by + the Codex runner before it wrote sidecars are read from the ``model:`` + line of ``codex.log``. A case with neither is counted under ``unknown`` + so the manifest cannot silently claim provenance it does not have. """ if not cases_dir.is_dir(): raise SystemExit(f"Audit case tree not found: {cases_dir}") @@ -179,15 +199,21 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: if not (case_dir / "verdict.json").is_file(): continue judged += 1 - meta_path = case_dir / "verdict.meta.json" codex_log = case_dir / "codex.log" - if meta_path.is_file(): - meta = json.loads(meta_path.read_text()) + meta = verdict_provenance(case_dir) + if meta is not None: judge = meta["judge_model_requested"] reported = meta.get("judge_model_reported") or [] if judge in {"opus", "claude-opus-5"} and "claude-opus-5" in reported: judge = "claude-opus-5" - runner = JUDGE_RUNNERS["claude"] + if judge == "default" and len(reported) == 1: + judge = reported[0] + runner_key = ( + "codex" + if "run_audit_codex" in str(meta.get("judge_runner", "")) + else "claude" + ) + runner = JUDGE_RUNNERS[runner_key] day = str(meta.get("judged_at_utc", ""))[:10] elif codex_log.is_file(): match = re.search(r"^model: (\S+)$", codex_log.read_text(), re.M) @@ -208,10 +234,11 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: "cases_judged": judged, "by_judge": dict(sorted(by_judge.items())), "note": ( - "Judge model per case: a verdict.meta.json sidecar (Claude Code " - "runner) or the codex.log model header (Codex runner) in the audit " - "tree. Verdicts classify misses after scoring and change no score. " - "Both judge models are also board rows." + "Judge model per case: the verdict.meta.json sidecar bound to the " + "case's verdict by sha256 (either runner), else the codex.log model " + "header (Codex runner before it wrote sidecars). Verdicts classify " + "misses after scoring and change no score. Both judge models are " + "also board rows." ), } diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 98f45b4..218c624 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -69,7 +69,7 @@ verdict_ok() { extract_verdict() { envelope="$1"; out_tmp="$2"; meta_tmp="$3"; requested_model="$4"; cli_version="$5" "$PYTHON" - "$envelope" "$out_tmp" "$meta_tmp" "$requested_model" "$cli_version" <<'PY' -import datetime, json, sys +import datetime, hashlib, json, sys envelope_path, out_path, meta_path, requested_model, cli_version = sys.argv[1:6] try: envelope = json.load(open(envelope_path)) @@ -89,9 +89,13 @@ if verdict is None: sys.exit(1) if not isinstance(verdict, dict): sys.exit(1) -json.dump(verdict, open(out_path, "w"), indent=2, sort_keys=True) +verdict_bytes = json.dumps(verdict, indent=2, sort_keys=True).encode("utf-8") +open(out_path, "wb").write(verdict_bytes) meta = { "judge_runner": "scripts/run_audit_claude.sh", + # Binds the sidecar to this verdict: a sidecar whose hash does not match + # the case's verdict.json is stale and carries no provenance. + "verdict_sha256": hashlib.sha256(verdict_bytes).hexdigest(), "judge_model_requested": requested_model, "judge_model_reported": sorted((envelope.get("modelUsage") or {}).keys()), "judge_cli_version": cli_version, @@ -113,7 +117,9 @@ classify_one() { envelope="$case_dir/claude.json" [ -f "$prompt" ] || return 0 verdict_ok "$out" && return 0 - rm -f "$tmp" "$meta_tmp" "$envelope" + # No valid verdict: any sidecar left behind describes a verdict that no + # longer exists (re-prepared case) and must not outlive it. + rm -f "$tmp" "$meta_tmp" "$envelope" "$case_dir/verdict.meta.json" # Self-contained prompt, no tools, no project instructions; the schema # enforces the JSON shape. Publish atomically only once it validates. CLAUDE_CODE_SAFE_MODE=1 CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 \ diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index 7a410f4..dfb7d62 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -9,6 +9,12 @@ # Resumable: a case is skipped once it has a verdict.json carrying the required # keys. Re-run freely after interruptions or to fill in failures. Concurrency, # model, and reasoning effort are tunable via env. Portable to bash 3.2 (macOS). +# +# Judge provenance: beside each verdict.json the runner writes verdict.meta.json +# (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex +# reports in its log header, the model requested, the UTC timestamp, and the +# verdict's sha256, so a re-judged case can never keep the other runner's +# provenance. set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -65,8 +71,9 @@ classify_one() { # Self-contained prompt; read-only sandbox; enforce the JSON shape. Write to a # temp file and publish atomically only once it validates, so an interrupted # run never leaves a half-written verdict that looks done. Default reasoning - # effort (xhigh) is wasteful for classification, so it is lowered. - rm -f "$tmp" + # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar + # left from a previous verdict describes a verdict that no longer exists. + rm -f "$tmp" "$case_dir/verdict.meta.json" codex exec \ --sandbox read-only \ --skip-git-repo-check \ @@ -77,15 +84,44 @@ classify_one() { --output-schema "$SCHEMA" \ -o "$tmp" \ - < "$prompt" > "$case_dir/codex.log" 2>&1 - if verdict_ok "$tmp"; then + if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ + "$case_dir/verdict.meta.json.tmp"; then mv -f "$tmp" "$out" + mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" echo "[ok] $(basename "$case_dir")" else - rm -f "$tmp" + rm -f "$tmp" "$case_dir/verdict.meta.json.tmp" echo "[FAIL] $(basename "$case_dir") (see codex.log)" fi } +# Provenance sidecar for a validated verdict: the model Codex reports in its +# log header (`model: ...`), the model requested (or "default"), the UTC time, +# and the verdict's sha256 so the sidecar cannot outlive the verdict it describes. +write_provenance() { + verdict_path="$1"; log_path="$2"; meta_out="$3" + "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" <<'PY' +import datetime, hashlib, json, re, sys +verdict_path, log_path, meta_out, requested = sys.argv[1:5] +verdict = open(verdict_path, "rb").read() +reported = [] +try: + match = re.search(r"^model: (\S+)$", open(log_path, encoding="utf-8", errors="replace").read(), re.M) + if match: + reported = [match.group(1)] +except OSError: + pass +meta = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": hashlib.sha256(verdict).hexdigest(), + "judge_model_requested": requested, + "judge_model_reported": reported, + "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), +} +json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True) +PY +} + total=$(find "$CASES_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') echo "audit: $total cases | parallel=$PARALLEL effort=$EFFORT model=${AUDIT_MODEL:-default}" diff --git a/tests/test_audit.py b/tests/test_audit.py index fc45b8a..3d4a3d7 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -498,6 +498,8 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): s0 = cases[0] verdict_path = audit_dir / "cases" / s0.case_id / "verdict.json" verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') + meta_path = verdict_path.with_name("verdict.meta.json") + meta_path.write_text('{"judge_model_requested": "opus"}') # Re-run m1 with a different wrong answer -> the case prompt changes. pd.DataFrame( @@ -514,6 +516,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): ).to_csv(d / "predictions.csv", index=False) prepare_audit(d, audit_dir) assert not verdict_path.exists() # stale verdict dropped + assert not meta_path.exists() # and its provenance sidecar with it # An unchanged case keeps its verdict. verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py new file mode 100644 index 0000000..7200b7f --- /dev/null +++ b/tests/test_judge_provenance.py @@ -0,0 +1,213 @@ +"""Judge provenance follows the verdict it describes, across both runners.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from freeze_snapshot import audit_judge_provenance, verdict_provenance # noqa: E402 + +from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 + +VERDICT = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": "Both models used last year's threshold.", + "models": [ + { + "model": "m1", + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": "Applied the 2025 threshold instead of the 2026 one.", + } + ], +} + + +def _case(cases: Path, name: str, verdict: dict) -> Path: + case_dir = cases / name + case_dir.mkdir(parents=True) + (case_dir / "verdict.json").write_text( + json.dumps(verdict, indent=2, sort_keys=True) + ) + return case_dir + + +def _sidecar(case_dir: Path, *, model: str, runner: str, bound: bool) -> None: + meta = { + "judge_runner": runner, + "judge_model_requested": model, + "judge_model_reported": ["claude-opus-5"] if model == "opus" else [model], + "judged_at_utc": "2026-09-05T02:40:00+00:00", + } + if bound: + meta["verdict_sha256"] = hashlib.sha256( + (case_dir / "verdict.json").read_bytes() + ).hexdigest() + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + + +def _codex_log(case_dir: Path, model: str) -> None: + (case_dir / "codex.log").write_text( + f"OpenAI Codex v0.144.0\n--------\nmodel: {model}\n" + ) + + +def test_sidecar_counts_only_when_bound_to_the_current_verdict(tmp_path: Path): + cases = tmp_path / "cases" + fresh = _case(cases, "fresh_claude", VERDICT) + _sidecar(fresh, model="opus", runner="scripts/run_audit_claude.sh", bound=True) + _codex_log(fresh, "gpt-5.6-sol") # an older Codex attempt; the sidecar wins + + stale = _case( + cases, "stale_claude_then_codex", {**VERDICT, "case_rationale": "new"} + ) + _sidecar(stale, model="opus", runner="scripts/run_audit_claude.sh", bound=False) + (stale / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_claude.sh", + "judge_model_requested": "opus", + "judge_model_reported": ["claude-opus-5"], + "judged_at_utc": "2026-09-05T02:40:00+00:00", + "verdict_sha256": "0" * 64, + } + ) + ) + _codex_log(stale, "gpt-5.6-sol") + + legacy = _case(cases, "legacy_sidecar_no_hash", VERDICT) + _sidecar(legacy, model="opus", runner="scripts/run_audit_claude.sh", bound=False) + + codex = _case(cases, "codex_sidecar", VERDICT) + _sidecar(codex, model="default", runner="scripts/run_audit_codex.sh", bound=True) + (codex / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_codex.sh", + "judge_model_requested": "default", + "judge_model_reported": ["gpt-5.6-sol"], + "judged_at_utc": "2026-09-05T03:00:00+00:00", + "verdict_sha256": hashlib.sha256( + (codex / "verdict.json").read_bytes() + ).hexdigest(), + } + ) + ) + + assert verdict_provenance(fresh) is not None + assert verdict_provenance(stale) is None + assert verdict_provenance(legacy) is None + + tally = audit_judge_provenance(cases) + assert tally["cases_judged"] == 4 + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + # fresh -> Opus via bound sidecar; stale -> Sol via codex.log (sidecar + # ignored); legacy -> unknown (no hash, no codex.log); codex -> Sol. + assert by_judge == {"claude-opus-5": 1, "gpt-5.6-sol": 2, "unknown": 1} + + +def _fake_cli(path: Path, body: str) -> None: + path.write_text("#!/bin/sh\n" + body) + path.chmod(0o755) + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): + """Claude judges a case; the case is re-prepared (verdict gone, stale + sidecar left behind as before the fix); Codex re-judges it. The published + provenance must be Codex's, both in the sidecar and in the tally.""" + audit_dir = tmp_path / "audit" + cases = audit_dir / "cases" + case_dir = cases / "us__scenario_001__snap" + case_dir.mkdir(parents=True) + (audit_dir / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA)) + (case_dir / "prompt.md").write_text("Classify this miss.\n") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + # Fake CLIs read their canned output from files, so the verdict text (which + # contains an apostrophe) never passes through shell quoting. + envelope_path = tmp_path / "envelope.json" + envelope_path.write_text( + json.dumps( + { + "structured_output": VERDICT, + "modelUsage": {"claude-opus-5": {}}, + "session_id": "s", + } + ) + ) + verdict_path = tmp_path / "canned_verdict.json" + verdict_path.write_text(json.dumps(VERDICT)) + # Fake claude: prints the CLI JSON envelope with the structured verdict. + _fake_cli( + bin_dir / "claude", + 'if [ "$1" = --version ]; then echo "9.9.9 (fake)"; exit 0; fi\n' + f'cat >/dev/null; cat "{envelope_path}"\n', + ) + # Fake codex: writes the -o file and logs its model header to stdout. + _fake_cli( + bin_dir / "codex", + 'out=""; while [ $# -gt 0 ]; do' + ' if [ "$1" = -o ]; then out="$2"; shift; fi; shift; done\n' + "cat >/dev/null\n" + 'echo "OpenAI Codex v0.144.0"; echo "--------"; echo "model: gpt-5.6-sol"\n' + f'cat "{verdict_path}" > "$out"\n', + ) + env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "AUDIT_PYTHON": sys.executable, + "AUDIT_PARALLEL": "1", + } + + claude = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_claude.sh"), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert claude.returncode == 0, claude.stderr + claude.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == "scripts/run_audit_claude.sh" + assert ( + meta["verdict_sha256"] + == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() + ) + assert { + j: e["cases"] for j, e in audit_judge_provenance(cases)["by_judge"].items() + } == {"claude-opus-5": 1} + + # Re-prepared case: the verdict is gone but (as before the fix) the sidecar + # was left behind. Codex re-judges. + (case_dir / "verdict.json").unlink() + codex = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert codex.returncode == 0, codex.stderr + codex.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == "scripts/run_audit_codex.sh" + assert meta["judge_model_reported"] == ["gpt-5.6-sol"] + assert ( + meta["verdict_sha256"] + == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() + ) + assert { + j: e["cases"] for j, e in audit_judge_provenance(cases)["by_judge"].items() + } == {"gpt-5.6-sol": 1} From c31484f95b04b3a382e380546c949203b028a87c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 5 Sep 2026 13:12:50 -0400 Subject: [PATCH 2/2] Require ownership evidence before trusting legacy provenance Gate review of #166 (round 1): the backfill's timestamp tolerance could bind a legacy Claude sidecar to a Codex verdict written minutes later, and the freezer's codex.log fallback could attribute a current hash-less Claude verdict to an older Codex attempt. - Backfill binds a legacy sidecar only when the verdict file dates from its judging time AND no codex.log was written after that time; a verdict with no sidecar at all can only have come from the Codex runner, which is the evidence for writing its Codex sidecar. - The freezer reads codex.log only when the case has no sidecar and the log was written alongside the verdict (600 s); a stale or hash-less sidecar beside an unrelated log counts as unknown. - Tests for both rules; the real audit tree's tally still reproduces the frozen manifest (350 / 318) and the backfill is idempotent (668 already). Co-Authored-By: Claude Fable 5.1 --- scripts/backfill_verdict_provenance.py | 23 ++++-- scripts/freeze_snapshot.py | 49 +++++++++--- tests/test_judge_provenance.py | 102 ++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 17 deletions(-) diff --git a/scripts/backfill_verdict_provenance.py b/scripts/backfill_verdict_provenance.py index de59fa5..0808803 100644 --- a/scripts/backfill_verdict_provenance.py +++ b/scripts/backfill_verdict_provenance.py @@ -4,10 +4,13 @@ freezer can tell a current sidecar from one left behind by a re-judged case. Sidecars written before that field existed are bound here, and Codex-judged cases that predate Codex sidecars get one from the ``codex.log`` header. A -sidecar is only bound when its recorded judging time sits within -``--tolerance`` seconds of the verdict file's modification time, so a sidecar -that cannot be shown to belong to the current verdict is left alone (the -freezer then falls back to ``codex.log`` or counts the case as unknown). +legacy Claude sidecar is bound only when its recorded judging time sits +within ``--tolerance`` seconds of the verdict file's modification time and no +``codex.log`` was written after that time (a later Codex attempt may own the +current verdict). A sidecar that cannot be shown to belong to the current +verdict is left alone, and the freezer then counts the case as unknown. A +verdict with no sidecar at all can only have come from the Codex runner (the +Claude runner always wrote one), so it gets a Codex sidecar from its log. uv run python scripts/backfill_verdict_provenance.py [--dry-run] [cases_dir] """ @@ -39,6 +42,7 @@ def backfill(cases_dir: Path, *, tolerance: float, dry_run: bool) -> dict[str, i digest = hashlib.sha256(verdict_path.read_bytes()).hexdigest() verdict_mtime = verdict_path.stat().st_mtime meta_path = case_dir / "verdict.meta.json" + codex_log = case_dir / "codex.log" if meta_path.is_file(): meta = json.loads(meta_path.read_text()) if meta.get("verdict_sha256"): @@ -50,13 +54,22 @@ def backfill(cases_dir: Path, *, tolerance: float, dry_run: bool) -> dict[str, i if abs(judged_at - verdict_mtime) > tolerance: counts["left_alone"] += 1 continue + # A Codex attempt after the sidecar\'s judging time may own the + # current verdict; timestamp proximity alone is not ownership. + if codex_log.is_file() and codex_log.stat().st_mtime > judged_at + 1.0: + counts["left_alone"] += 1 + continue meta["verdict_sha256"] = digest meta.setdefault("judge_runner", "scripts/run_audit_claude.sh") if not dry_run: meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True)) counts["bound"] += 1 continue - codex_log = case_dir / "codex.log" + # No sidecar at all: the Claude runner always wrote one, so this + # verdict came from the Codex runner before it wrote sidecars; its + # codex.log names the model. (The log may predate the verdict file's + # mtime when the verdict was rewritten later; the absence of any + # sidecar, not the timestamps, is the ownership evidence here.) if codex_log.is_file(): match = re.search( r"^model: (\S+)$", diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index aab18a9..bf72971 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -181,14 +181,44 @@ def verdict_provenance(case_dir: Path) -> dict | None: return meta -def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: +# A codex.log may stand in for a missing sidecar only when it was written +# alongside the current verdict: the Codex runner writes the log and, on +# success, the verdict within the same call. +LOG_FALLBACK_TOLERANCE_SECONDS = 600.0 + + +def codex_log_describes_verdict( + case_dir: Path, tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS +) -> bool: + """Whether ``codex.log`` is evidence about the case's current verdict. + + Only when no sidecar exists at all (the Claude runner always wrote one, so + a sidecar-less verdict can only come from the Codex runner) and the log's + modification time sits within ``tolerance`` seconds of the verdict's. A + log left beside a later verdict from the other runner, or one hours older + than the verdict, describes an earlier attempt and proves nothing. + """ + codex_log = case_dir / "codex.log" + verdict_path = case_dir / "verdict.json" + if not codex_log.is_file() or not verdict_path.is_file(): + return False + if (case_dir / "verdict.meta.json").is_file(): + return False + return abs(codex_log.stat().st_mtime - verdict_path.stat().st_mtime) <= tolerance + + +def audit_judge_provenance( + cases_dir: Path = AUDIT_CASES_DIR, + log_tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS, +) -> dict: """Tally which judge model produced each case verdict in the audit tree. Each runner writes a ``verdict.meta.json`` sidecar bound to its verdict by sha256 (the model requested and the model the CLI reported); only a - sidecar that matches the case's current verdict counts. Cases judged by - the Codex runner before it wrote sidecars are read from the ``model:`` - line of ``codex.log``. A case with neither is counted under ``unknown`` + sidecar that matches the case's current verdict counts. A case with no + sidecar at all whose ``codex.log`` was written alongside the verdict is + read from the log's ``model:`` line. Anything else, including a stale or + hash-less sidecar beside an unrelated log, is counted under ``unknown`` so the manifest cannot silently claim provenance it does not have. """ if not cases_dir.is_dir(): @@ -215,7 +245,7 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: ) runner = JUDGE_RUNNERS[runner_key] day = str(meta.get("judged_at_utc", ""))[:10] - elif codex_log.is_file(): + elif codex_log_describes_verdict(case_dir, log_tolerance): match = re.search(r"^model: (\S+)$", codex_log.read_text(), re.M) judge = match.group(1) if match else "unknown" runner = JUDGE_RUNNERS["codex"] @@ -235,10 +265,11 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: "by_judge": dict(sorted(by_judge.items())), "note": ( "Judge model per case: the verdict.meta.json sidecar bound to the " - "case's verdict by sha256 (either runner), else the codex.log model " - "header (Codex runner before it wrote sidecars). Verdicts classify " - "misses after scoring and change no score. Both judge models are " - "also board rows." + "case's verdict by sha256 (either runner), else, for a case with no " + "sidecar, the codex.log model header when the log was written " + "alongside the verdict (Codex runner before it wrote sidecars). " + "Verdicts classify misses after scoring and change no score. Both " + "judge models are also board rows." ), } diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 7200b7f..dbdad0a 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -113,9 +113,10 @@ def test_sidecar_counts_only_when_bound_to_the_current_verdict(tmp_path: Path): tally = audit_judge_provenance(cases) assert tally["cases_judged"] == 4 by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} - # fresh -> Opus via bound sidecar; stale -> Sol via codex.log (sidecar - # ignored); legacy -> unknown (no hash, no codex.log); codex -> Sol. - assert by_judge == {"claude-opus-5": 1, "gpt-5.6-sol": 2, "unknown": 1} + # fresh -> Opus via bound sidecar; stale -> unknown (a sidecar that does + # not match the verdict is not provenance, and its presence rules out the + # codex.log fallback); legacy -> unknown (no hash); codex -> Sol. + assert by_judge == {"claude-opus-5": 1, "gpt-5.6-sol": 1, "unknown": 2} def _fake_cli(path: Path, body: str) -> None: @@ -211,3 +212,98 @@ def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): assert { j: e["cases"] for j, e in audit_judge_provenance(cases)["by_judge"].items() } == {"gpt-5.6-sol": 1} + + +def _touch(path: Path, when: float) -> None: + os.utime(path, (when, when)) + + +def test_log_fallback_needs_a_contemporaneous_log_and_no_sidecar(tmp_path: Path): + """A hash-less sidecar beside an older codex.log (a legacy Codex-to-Claude + re-judge) is unknown, not Codex; a sidecar-less verdict is Codex only when + its log was written alongside it.""" + cases = tmp_path / "cases" + now = 1_800_000_000.0 + + claude_legacy = _case(cases, "legacy_claude_after_codex", VERDICT) + _sidecar( + claude_legacy, model="opus", runner="scripts/run_audit_claude.sh", bound=False + ) + _codex_log(claude_legacy, "gpt-5.6-sol") + _touch(claude_legacy / "codex.log", now - 86_400) + _touch(claude_legacy / "verdict.json", now) + + codex_fresh = _case(cases, "codex_no_sidecar_fresh_log", VERDICT) + _codex_log(codex_fresh, "gpt-5.6-sol") + _touch(codex_fresh / "codex.log", now - 30) + _touch(codex_fresh / "verdict.json", now) + + codex_old = _case(cases, "codex_no_sidecar_stale_log", VERDICT) + _codex_log(codex_old, "gpt-5.6-sol") + _touch(codex_old / "codex.log", now - 86_400) + _touch(codex_old / "verdict.json", now) + + tally = audit_judge_provenance(cases) + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + assert by_judge == {"gpt-5.6-sol": 1, "unknown": 2} + + +def test_backfill_binds_only_with_ownership_evidence(tmp_path: Path): + from backfill_verdict_provenance import backfill + + cases = tmp_path / "cases" + now = 1_800_000_000.0 + judged_at = "2027-01-15T00:00:00+00:00" + import datetime + + judged_ts = datetime.datetime.fromisoformat(judged_at).timestamp() + + def legacy_claude(name: str, *, log_offset: float | None) -> Path: + case_dir = _case(cases, name, VERDICT) + (case_dir / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_claude.sh", + "judge_model_requested": "opus", + "judge_model_reported": ["claude-opus-5"], + "judged_at_utc": judged_at, + } + ) + ) + _touch(case_dir / "verdict.json", judged_ts + 5) + if log_offset is not None: + _codex_log(case_dir, "gpt-5.6-sol") + _touch(case_dir / "codex.log", judged_ts + log_offset) + return case_dir + + bound = legacy_claude("claude_then_nothing", log_offset=None) + bound_older_log = legacy_claude("codex_then_claude", log_offset=-3_600) + # Codex re-judged two minutes after the Claude sidecar was written: the + # sidecar does not own the current verdict, however close the times are. + contested = legacy_claude("claude_then_codex", log_offset=120) + _touch(contested / "verdict.json", judged_ts + 125) + + codex_only = _case(cases, "codex_only", VERDICT) + _codex_log(codex_only, "gpt-5.6-sol") + _touch(codex_only / "codex.log", now - 86_400) + _touch(codex_only / "verdict.json", now) + + counts = backfill(cases, tolerance=600.0, dry_run=False) + assert counts == { + "bound": 2, + "codex_sidecar_written": 1, + "left_alone": 1, + "already": 0, + } + for case_dir in (bound, bound_older_log): + assert verdict_provenance(case_dir)["judge_model_requested"] == "opus" + assert "verdict_sha256" not in json.loads( + (contested / "verdict.meta.json").read_text() + ) + assert verdict_provenance(codex_only)["judge_model_reported"] == ["gpt-5.6-sol"] + + tally = audit_judge_provenance(cases) + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + assert by_judge == {"claude-opus-5": 2, "gpt-5.6-sol": 1, "unknown": 1} + # Idempotent: a second pass changes nothing. + assert backfill(cases, tolerance=600.0, dry_run=False)["already"] == 3