diff --git a/policybench/audit.py b/policybench/audit.py index b53e111..0c947b1 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -416,6 +416,9 @@ def prepare_audit( and prompt_path.read_text() != new_prompt ): verdict_path.unlink() + # The provenance sidecar describes that verdict; a re-judge by + # the other runner must not inherit it. + (case_dir / "verdict.meta.json").unlink(missing_ok=True) prompt_path.write_text(new_prompt) return cases diff --git a/scripts/backfill_verdict_provenance.py b/scripts/backfill_verdict_provenance.py new file mode 100644 index 0000000..0808803 --- /dev/null +++ b/scripts/backfill_verdict_provenance.py @@ -0,0 +1,111 @@ +"""Bind existing judge-provenance sidecars to the verdicts they describe. + +Runners now write ``verdict.meta.json`` with the verdict's sha256 so the +freezer can tell a current sidecar from one left behind by a re-judged case. +Sidecars written before that field existed are bound here, and Codex-judged +cases that predate Codex sidecars get one from the ``codex.log`` header. A +legacy Claude sidecar is bound only when its recorded judging time sits +within ``--tolerance`` seconds of the verdict file's modification time and no +``codex.log`` was written after that time (a later Codex attempt may own the +current verdict). A sidecar that cannot be shown to belong to the current +verdict is left alone, and the freezer then counts the case as unknown. A +verdict with no sidecar at all can only have come from the Codex runner (the +Claude runner always wrote one), so it gets a Codex sidecar from its log. + + uv run python scripts/backfill_verdict_provenance.py [--dry-run] [cases_dir] +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from freeze_snapshot import AUDIT_CASES_DIR # noqa: E402 + + +def _utc(ts: float) -> str: + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).isoformat() + + +def backfill(cases_dir: Path, *, tolerance: float, dry_run: bool) -> dict[str, int]: + counts = {"bound": 0, "codex_sidecar_written": 0, "left_alone": 0, "already": 0} + for case_dir in sorted(cases_dir.iterdir()): + verdict_path = case_dir / "verdict.json" + if not verdict_path.is_file(): + continue + digest = hashlib.sha256(verdict_path.read_bytes()).hexdigest() + verdict_mtime = verdict_path.stat().st_mtime + meta_path = case_dir / "verdict.meta.json" + codex_log = case_dir / "codex.log" + if meta_path.is_file(): + meta = json.loads(meta_path.read_text()) + if meta.get("verdict_sha256"): + counts["already"] += 1 + continue + judged_at = datetime.datetime.fromisoformat( + meta["judged_at_utc"] + ).timestamp() + if abs(judged_at - verdict_mtime) > tolerance: + counts["left_alone"] += 1 + continue + # A Codex attempt after the sidecar\'s judging time may own the + # current verdict; timestamp proximity alone is not ownership. + if codex_log.is_file() and codex_log.stat().st_mtime > judged_at + 1.0: + counts["left_alone"] += 1 + continue + meta["verdict_sha256"] = digest + meta.setdefault("judge_runner", "scripts/run_audit_claude.sh") + if not dry_run: + meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True)) + counts["bound"] += 1 + continue + # No sidecar at all: the Claude runner always wrote one, so this + # verdict came from the Codex runner before it wrote sidecars; its + # codex.log names the model. (The log may predate the verdict file's + # mtime when the verdict was rewritten later; the absence of any + # sidecar, not the timestamps, is the ownership evidence here.) + if codex_log.is_file(): + match = re.search( + r"^model: (\S+)$", + codex_log.read_text(encoding="utf-8", errors="replace"), + re.M, + ) + if not match: + counts["left_alone"] += 1 + continue + meta = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": digest, + "judge_model_requested": "default", + "judge_model_reported": [match.group(1)], + "judged_at_utc": _utc(verdict_mtime), + "backfilled_from": "codex.log", + } + if not dry_run: + meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True)) + counts["codex_sidecar_written"] += 1 + continue + counts["left_alone"] += 1 + return counts + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("cases_dir", nargs="?", default=str(AUDIT_CASES_DIR)) + parser.add_argument("--tolerance", type=float, default=600.0) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + counts = backfill( + Path(args.cases_dir), tolerance=args.tolerance, dry_run=args.dry_run + ) + print(json.dumps(counts)) + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index f89479a..bf72971 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -162,14 +162,64 @@ } -def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: +def verdict_provenance(case_dir: Path) -> dict | None: + """The ``verdict.meta.json`` sidecar for a case, if it describes the + case's current ``verdict.json``. + + Both runners write the sidecar with the verdict's sha256. A sidecar whose + hash does not match (or that predates the hash field) belongs to an + earlier verdict, most likely one the other runner has since replaced, and + is ignored so a re-judged case cannot keep stale provenance. + """ + meta_path = case_dir / "verdict.meta.json" + verdict_path = case_dir / "verdict.json" + if not meta_path.is_file() or not verdict_path.is_file(): + return None + meta = json.loads(meta_path.read_text()) + if meta.get("verdict_sha256") != sha256_file(verdict_path): + return None + return meta + + +# A codex.log may stand in for a missing sidecar only when it was written +# alongside the current verdict: the Codex runner writes the log and, on +# success, the verdict within the same call. +LOG_FALLBACK_TOLERANCE_SECONDS = 600.0 + + +def codex_log_describes_verdict( + case_dir: Path, tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS +) -> bool: + """Whether ``codex.log`` is evidence about the case's current verdict. + + Only when no sidecar exists at all (the Claude runner always wrote one, so + a sidecar-less verdict can only come from the Codex runner) and the log's + modification time sits within ``tolerance`` seconds of the verdict's. A + log left beside a later verdict from the other runner, or one hours older + than the verdict, describes an earlier attempt and proves nothing. + """ + codex_log = case_dir / "codex.log" + verdict_path = case_dir / "verdict.json" + if not codex_log.is_file() or not verdict_path.is_file(): + return False + if (case_dir / "verdict.meta.json").is_file(): + return False + return abs(codex_log.stat().st_mtime - verdict_path.stat().st_mtime) <= tolerance + + +def audit_judge_provenance( + cases_dir: Path = AUDIT_CASES_DIR, + log_tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS, +) -> dict: """Tally which judge model produced each case verdict in the audit tree. - A case with a ``verdict.meta.json`` sidecar was judged (or re-judged) by - the Claude Code runner, which records the judge model it requested; the - Codex runner records its model in the ``model:`` line of ``codex.log``. - A case with neither is counted under ``unknown`` so the manifest cannot - silently claim provenance it does not have. + Each runner writes a ``verdict.meta.json`` sidecar bound to its verdict by + sha256 (the model requested and the model the CLI reported); only a + sidecar that matches the case's current verdict counts. A case with no + sidecar at all whose ``codex.log`` was written alongside the verdict is + read from the log's ``model:`` line. Anything else, including a stale or + hash-less sidecar beside an unrelated log, is counted under ``unknown`` + so the manifest cannot silently claim provenance it does not have. """ if not cases_dir.is_dir(): raise SystemExit(f"Audit case tree not found: {cases_dir}") @@ -179,17 +229,23 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: if not (case_dir / "verdict.json").is_file(): continue judged += 1 - meta_path = case_dir / "verdict.meta.json" codex_log = case_dir / "codex.log" - if meta_path.is_file(): - meta = json.loads(meta_path.read_text()) + meta = verdict_provenance(case_dir) + if meta is not None: judge = meta["judge_model_requested"] reported = meta.get("judge_model_reported") or [] if judge in {"opus", "claude-opus-5"} and "claude-opus-5" in reported: judge = "claude-opus-5" - runner = JUDGE_RUNNERS["claude"] + if judge == "default" and len(reported) == 1: + judge = reported[0] + runner_key = ( + "codex" + if "run_audit_codex" in str(meta.get("judge_runner", "")) + else "claude" + ) + runner = JUDGE_RUNNERS[runner_key] day = str(meta.get("judged_at_utc", ""))[:10] - elif codex_log.is_file(): + elif codex_log_describes_verdict(case_dir, log_tolerance): match = re.search(r"^model: (\S+)$", codex_log.read_text(), re.M) judge = match.group(1) if match else "unknown" runner = JUDGE_RUNNERS["codex"] @@ -208,10 +264,12 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict: "cases_judged": judged, "by_judge": dict(sorted(by_judge.items())), "note": ( - "Judge model per case: a verdict.meta.json sidecar (Claude Code " - "runner) or the codex.log model header (Codex runner) in the audit " - "tree. Verdicts classify misses after scoring and change no score. " - "Both judge models are also board rows." + "Judge model per case: the verdict.meta.json sidecar bound to the " + "case's verdict by sha256 (either runner), else, for a case with no " + "sidecar, the codex.log model header when the log was written " + "alongside the verdict (Codex runner before it wrote sidecars). " + "Verdicts classify misses after scoring and change no score. Both " + "judge models are also board rows." ), } diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 98f45b4..218c624 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -69,7 +69,7 @@ verdict_ok() { extract_verdict() { envelope="$1"; out_tmp="$2"; meta_tmp="$3"; requested_model="$4"; cli_version="$5" "$PYTHON" - "$envelope" "$out_tmp" "$meta_tmp" "$requested_model" "$cli_version" <<'PY' -import datetime, json, sys +import datetime, hashlib, json, sys envelope_path, out_path, meta_path, requested_model, cli_version = sys.argv[1:6] try: envelope = json.load(open(envelope_path)) @@ -89,9 +89,13 @@ if verdict is None: sys.exit(1) if not isinstance(verdict, dict): sys.exit(1) -json.dump(verdict, open(out_path, "w"), indent=2, sort_keys=True) +verdict_bytes = json.dumps(verdict, indent=2, sort_keys=True).encode("utf-8") +open(out_path, "wb").write(verdict_bytes) meta = { "judge_runner": "scripts/run_audit_claude.sh", + # Binds the sidecar to this verdict: a sidecar whose hash does not match + # the case's verdict.json is stale and carries no provenance. + "verdict_sha256": hashlib.sha256(verdict_bytes).hexdigest(), "judge_model_requested": requested_model, "judge_model_reported": sorted((envelope.get("modelUsage") or {}).keys()), "judge_cli_version": cli_version, @@ -113,7 +117,9 @@ classify_one() { envelope="$case_dir/claude.json" [ -f "$prompt" ] || return 0 verdict_ok "$out" && return 0 - rm -f "$tmp" "$meta_tmp" "$envelope" + # No valid verdict: any sidecar left behind describes a verdict that no + # longer exists (re-prepared case) and must not outlive it. + rm -f "$tmp" "$meta_tmp" "$envelope" "$case_dir/verdict.meta.json" # Self-contained prompt, no tools, no project instructions; the schema # enforces the JSON shape. Publish atomically only once it validates. CLAUDE_CODE_SAFE_MODE=1 CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 \ diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index 7a410f4..dfb7d62 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -9,6 +9,12 @@ # Resumable: a case is skipped once it has a verdict.json carrying the required # keys. Re-run freely after interruptions or to fill in failures. Concurrency, # model, and reasoning effort are tunable via env. Portable to bash 3.2 (macOS). +# +# Judge provenance: beside each verdict.json the runner writes verdict.meta.json +# (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex +# reports in its log header, the model requested, the UTC timestamp, and the +# verdict's sha256, so a re-judged case can never keep the other runner's +# provenance. set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -65,8 +71,9 @@ classify_one() { # Self-contained prompt; read-only sandbox; enforce the JSON shape. Write to a # temp file and publish atomically only once it validates, so an interrupted # run never leaves a half-written verdict that looks done. Default reasoning - # effort (xhigh) is wasteful for classification, so it is lowered. - rm -f "$tmp" + # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar + # left from a previous verdict describes a verdict that no longer exists. + rm -f "$tmp" "$case_dir/verdict.meta.json" codex exec \ --sandbox read-only \ --skip-git-repo-check \ @@ -77,15 +84,44 @@ classify_one() { --output-schema "$SCHEMA" \ -o "$tmp" \ - < "$prompt" > "$case_dir/codex.log" 2>&1 - if verdict_ok "$tmp"; then + if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ + "$case_dir/verdict.meta.json.tmp"; then mv -f "$tmp" "$out" + mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" echo "[ok] $(basename "$case_dir")" else - rm -f "$tmp" + rm -f "$tmp" "$case_dir/verdict.meta.json.tmp" echo "[FAIL] $(basename "$case_dir") (see codex.log)" fi } +# Provenance sidecar for a validated verdict: the model Codex reports in its +# log header (`model: ...`), the model requested (or "default"), the UTC time, +# and the verdict's sha256 so the sidecar cannot outlive the verdict it describes. +write_provenance() { + verdict_path="$1"; log_path="$2"; meta_out="$3" + "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" <<'PY' +import datetime, hashlib, json, re, sys +verdict_path, log_path, meta_out, requested = sys.argv[1:5] +verdict = open(verdict_path, "rb").read() +reported = [] +try: + match = re.search(r"^model: (\S+)$", open(log_path, encoding="utf-8", errors="replace").read(), re.M) + if match: + reported = [match.group(1)] +except OSError: + pass +meta = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": hashlib.sha256(verdict).hexdigest(), + "judge_model_requested": requested, + "judge_model_reported": reported, + "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), +} +json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True) +PY +} + total=$(find "$CASES_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') echo "audit: $total cases | parallel=$PARALLEL effort=$EFFORT model=${AUDIT_MODEL:-default}" diff --git a/tests/test_audit.py b/tests/test_audit.py index fc45b8a..3d4a3d7 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -498,6 +498,8 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): s0 = cases[0] verdict_path = audit_dir / "cases" / s0.case_id / "verdict.json" verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') + meta_path = verdict_path.with_name("verdict.meta.json") + meta_path.write_text('{"judge_model_requested": "opus"}') # Re-run m1 with a different wrong answer -> the case prompt changes. pd.DataFrame( @@ -514,6 +516,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): ).to_csv(d / "predictions.csv", index=False) prepare_audit(d, audit_dir) assert not verdict_path.exists() # stale verdict dropped + assert not meta_path.exists() # and its provenance sidecar with it # An unchanged case keeps its verdict. verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py new file mode 100644 index 0000000..dbdad0a --- /dev/null +++ b/tests/test_judge_provenance.py @@ -0,0 +1,309 @@ +"""Judge provenance follows the verdict it describes, across both runners.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from freeze_snapshot import audit_judge_provenance, verdict_provenance # noqa: E402 + +from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 + +VERDICT = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": "Both models used last year's threshold.", + "models": [ + { + "model": "m1", + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": "Applied the 2025 threshold instead of the 2026 one.", + } + ], +} + + +def _case(cases: Path, name: str, verdict: dict) -> Path: + case_dir = cases / name + case_dir.mkdir(parents=True) + (case_dir / "verdict.json").write_text( + json.dumps(verdict, indent=2, sort_keys=True) + ) + return case_dir + + +def _sidecar(case_dir: Path, *, model: str, runner: str, bound: bool) -> None: + meta = { + "judge_runner": runner, + "judge_model_requested": model, + "judge_model_reported": ["claude-opus-5"] if model == "opus" else [model], + "judged_at_utc": "2026-09-05T02:40:00+00:00", + } + if bound: + meta["verdict_sha256"] = hashlib.sha256( + (case_dir / "verdict.json").read_bytes() + ).hexdigest() + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + + +def _codex_log(case_dir: Path, model: str) -> None: + (case_dir / "codex.log").write_text( + f"OpenAI Codex v0.144.0\n--------\nmodel: {model}\n" + ) + + +def test_sidecar_counts_only_when_bound_to_the_current_verdict(tmp_path: Path): + cases = tmp_path / "cases" + fresh = _case(cases, "fresh_claude", VERDICT) + _sidecar(fresh, model="opus", runner="scripts/run_audit_claude.sh", bound=True) + _codex_log(fresh, "gpt-5.6-sol") # an older Codex attempt; the sidecar wins + + stale = _case( + cases, "stale_claude_then_codex", {**VERDICT, "case_rationale": "new"} + ) + _sidecar(stale, model="opus", runner="scripts/run_audit_claude.sh", bound=False) + (stale / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_claude.sh", + "judge_model_requested": "opus", + "judge_model_reported": ["claude-opus-5"], + "judged_at_utc": "2026-09-05T02:40:00+00:00", + "verdict_sha256": "0" * 64, + } + ) + ) + _codex_log(stale, "gpt-5.6-sol") + + legacy = _case(cases, "legacy_sidecar_no_hash", VERDICT) + _sidecar(legacy, model="opus", runner="scripts/run_audit_claude.sh", bound=False) + + codex = _case(cases, "codex_sidecar", VERDICT) + _sidecar(codex, model="default", runner="scripts/run_audit_codex.sh", bound=True) + (codex / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_codex.sh", + "judge_model_requested": "default", + "judge_model_reported": ["gpt-5.6-sol"], + "judged_at_utc": "2026-09-05T03:00:00+00:00", + "verdict_sha256": hashlib.sha256( + (codex / "verdict.json").read_bytes() + ).hexdigest(), + } + ) + ) + + assert verdict_provenance(fresh) is not None + assert verdict_provenance(stale) is None + assert verdict_provenance(legacy) is None + + tally = audit_judge_provenance(cases) + assert tally["cases_judged"] == 4 + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + # fresh -> Opus via bound sidecar; stale -> unknown (a sidecar that does + # not match the verdict is not provenance, and its presence rules out the + # codex.log fallback); legacy -> unknown (no hash); codex -> Sol. + assert by_judge == {"claude-opus-5": 1, "gpt-5.6-sol": 1, "unknown": 2} + + +def _fake_cli(path: Path, body: str) -> None: + path.write_text("#!/bin/sh\n" + body) + path.chmod(0o755) + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): + """Claude judges a case; the case is re-prepared (verdict gone, stale + sidecar left behind as before the fix); Codex re-judges it. The published + provenance must be Codex's, both in the sidecar and in the tally.""" + audit_dir = tmp_path / "audit" + cases = audit_dir / "cases" + case_dir = cases / "us__scenario_001__snap" + case_dir.mkdir(parents=True) + (audit_dir / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA)) + (case_dir / "prompt.md").write_text("Classify this miss.\n") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + # Fake CLIs read their canned output from files, so the verdict text (which + # contains an apostrophe) never passes through shell quoting. + envelope_path = tmp_path / "envelope.json" + envelope_path.write_text( + json.dumps( + { + "structured_output": VERDICT, + "modelUsage": {"claude-opus-5": {}}, + "session_id": "s", + } + ) + ) + verdict_path = tmp_path / "canned_verdict.json" + verdict_path.write_text(json.dumps(VERDICT)) + # Fake claude: prints the CLI JSON envelope with the structured verdict. + _fake_cli( + bin_dir / "claude", + 'if [ "$1" = --version ]; then echo "9.9.9 (fake)"; exit 0; fi\n' + f'cat >/dev/null; cat "{envelope_path}"\n', + ) + # Fake codex: writes the -o file and logs its model header to stdout. + _fake_cli( + bin_dir / "codex", + 'out=""; while [ $# -gt 0 ]; do' + ' if [ "$1" = -o ]; then out="$2"; shift; fi; shift; done\n' + "cat >/dev/null\n" + 'echo "OpenAI Codex v0.144.0"; echo "--------"; echo "model: gpt-5.6-sol"\n' + f'cat "{verdict_path}" > "$out"\n', + ) + env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "AUDIT_PYTHON": sys.executable, + "AUDIT_PARALLEL": "1", + } + + claude = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_claude.sh"), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert claude.returncode == 0, claude.stderr + claude.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == "scripts/run_audit_claude.sh" + assert ( + meta["verdict_sha256"] + == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() + ) + assert { + j: e["cases"] for j, e in audit_judge_provenance(cases)["by_judge"].items() + } == {"claude-opus-5": 1} + + # Re-prepared case: the verdict is gone but (as before the fix) the sidecar + # was left behind. Codex re-judges. + (case_dir / "verdict.json").unlink() + codex = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert codex.returncode == 0, codex.stderr + codex.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == "scripts/run_audit_codex.sh" + assert meta["judge_model_reported"] == ["gpt-5.6-sol"] + assert ( + meta["verdict_sha256"] + == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() + ) + assert { + j: e["cases"] for j, e in audit_judge_provenance(cases)["by_judge"].items() + } == {"gpt-5.6-sol": 1} + + +def _touch(path: Path, when: float) -> None: + os.utime(path, (when, when)) + + +def test_log_fallback_needs_a_contemporaneous_log_and_no_sidecar(tmp_path: Path): + """A hash-less sidecar beside an older codex.log (a legacy Codex-to-Claude + re-judge) is unknown, not Codex; a sidecar-less verdict is Codex only when + its log was written alongside it.""" + cases = tmp_path / "cases" + now = 1_800_000_000.0 + + claude_legacy = _case(cases, "legacy_claude_after_codex", VERDICT) + _sidecar( + claude_legacy, model="opus", runner="scripts/run_audit_claude.sh", bound=False + ) + _codex_log(claude_legacy, "gpt-5.6-sol") + _touch(claude_legacy / "codex.log", now - 86_400) + _touch(claude_legacy / "verdict.json", now) + + codex_fresh = _case(cases, "codex_no_sidecar_fresh_log", VERDICT) + _codex_log(codex_fresh, "gpt-5.6-sol") + _touch(codex_fresh / "codex.log", now - 30) + _touch(codex_fresh / "verdict.json", now) + + codex_old = _case(cases, "codex_no_sidecar_stale_log", VERDICT) + _codex_log(codex_old, "gpt-5.6-sol") + _touch(codex_old / "codex.log", now - 86_400) + _touch(codex_old / "verdict.json", now) + + tally = audit_judge_provenance(cases) + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + assert by_judge == {"gpt-5.6-sol": 1, "unknown": 2} + + +def test_backfill_binds_only_with_ownership_evidence(tmp_path: Path): + from backfill_verdict_provenance import backfill + + cases = tmp_path / "cases" + now = 1_800_000_000.0 + judged_at = "2027-01-15T00:00:00+00:00" + import datetime + + judged_ts = datetime.datetime.fromisoformat(judged_at).timestamp() + + def legacy_claude(name: str, *, log_offset: float | None) -> Path: + case_dir = _case(cases, name, VERDICT) + (case_dir / "verdict.meta.json").write_text( + json.dumps( + { + "judge_runner": "scripts/run_audit_claude.sh", + "judge_model_requested": "opus", + "judge_model_reported": ["claude-opus-5"], + "judged_at_utc": judged_at, + } + ) + ) + _touch(case_dir / "verdict.json", judged_ts + 5) + if log_offset is not None: + _codex_log(case_dir, "gpt-5.6-sol") + _touch(case_dir / "codex.log", judged_ts + log_offset) + return case_dir + + bound = legacy_claude("claude_then_nothing", log_offset=None) + bound_older_log = legacy_claude("codex_then_claude", log_offset=-3_600) + # Codex re-judged two minutes after the Claude sidecar was written: the + # sidecar does not own the current verdict, however close the times are. + contested = legacy_claude("claude_then_codex", log_offset=120) + _touch(contested / "verdict.json", judged_ts + 125) + + codex_only = _case(cases, "codex_only", VERDICT) + _codex_log(codex_only, "gpt-5.6-sol") + _touch(codex_only / "codex.log", now - 86_400) + _touch(codex_only / "verdict.json", now) + + counts = backfill(cases, tolerance=600.0, dry_run=False) + assert counts == { + "bound": 2, + "codex_sidecar_written": 1, + "left_alone": 1, + "already": 0, + } + for case_dir in (bound, bound_older_log): + assert verdict_provenance(case_dir)["judge_model_requested"] == "opus" + assert "verdict_sha256" not in json.loads( + (contested / "verdict.meta.json").read_text() + ) + assert verdict_provenance(codex_only)["judge_model_reported"] == ["gpt-5.6-sol"] + + tally = audit_judge_provenance(cases) + by_judge = {judge: entry["cases"] for judge, entry in tally["by_judge"].items()} + assert by_judge == {"claude-opus-5": 2, "gpt-5.6-sol": 1, "unknown": 1} + # Idempotent: a second pass changes nothing. + assert backfill(cases, tolerance=600.0, dry_run=False)["already"] == 3