Bind judge provenance to the verdict it describes

policybench/audit.py

@@ -416,6 +416,9 @@ def prepare_audit(
                 and prompt_path.read_text() != new_prompt
             ):
                 verdict_path.unlink()
+                # The provenance sidecar describes that verdict; a re-judge by
+                # the other runner must not inherit it.
+                (case_dir / "verdict.meta.json").unlink(missing_ok=True)
             prompt_path.write_text(new_prompt)
     return cases
 

scripts/backfill_verdict_provenance.py

@@ -0,0 +1,111 @@
+"""Bind existing judge-provenance sidecars to the verdicts they describe.
+
+Runners now write ``verdict.meta.json`` with the verdict's sha256 so the
+freezer can tell a current sidecar from one left behind by a re-judged case.
+Sidecars written before that field existed are bound here, and Codex-judged
+cases that predate Codex sidecars get one from the ``codex.log`` header. A
+legacy Claude sidecar is bound only when its recorded judging time sits
+within ``--tolerance`` seconds of the verdict file's modification time and no
+``codex.log`` was written after that time (a later Codex attempt may own the
+current verdict). A sidecar that cannot be shown to belong to the current
+verdict is left alone, and the freezer then counts the case as unknown. A
+verdict with no sidecar at all can only have come from the Codex runner (the
+Claude runner always wrote one), so it gets a Codex sidecar from its log.
+
+    uv run python scripts/backfill_verdict_provenance.py [--dry-run] [cases_dir]
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from freeze_snapshot import AUDIT_CASES_DIR  # noqa: E402
+
+
+def _utc(ts: float) -> str:
+    return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).isoformat()
+
+
+def backfill(cases_dir: Path, *, tolerance: float, dry_run: bool) -> dict[str, int]:
+    counts = {"bound": 0, "codex_sidecar_written": 0, "left_alone": 0, "already": 0}
+    for case_dir in sorted(cases_dir.iterdir()):
+        verdict_path = case_dir / "verdict.json"
+        if not verdict_path.is_file():
+            continue
+        digest = hashlib.sha256(verdict_path.read_bytes()).hexdigest()
+        verdict_mtime = verdict_path.stat().st_mtime
+        meta_path = case_dir / "verdict.meta.json"
+        codex_log = case_dir / "codex.log"
+        if meta_path.is_file():
+            meta = json.loads(meta_path.read_text())
+            if meta.get("verdict_sha256"):
+                counts["already"] += 1
+                continue
+            judged_at = datetime.datetime.fromisoformat(
+                meta["judged_at_utc"]
+            ).timestamp()
+            if abs(judged_at - verdict_mtime) > tolerance:
+                counts["left_alone"] += 1
+                continue
+            # A Codex attempt after the sidecar\'s judging time may own the
+            # current verdict; timestamp proximity alone is not ownership.
+            if codex_log.is_file() and codex_log.stat().st_mtime > judged_at + 1.0:
+                counts["left_alone"] += 1
+                continue
+            meta["verdict_sha256"] = digest
+            meta.setdefault("judge_runner", "scripts/run_audit_claude.sh")
+            if not dry_run:
+                meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True))
+            counts["bound"] += 1
+            continue
+        # No sidecar at all: the Claude runner always wrote one, so this
+        # verdict came from the Codex runner before it wrote sidecars; its
+        # codex.log names the model. (The log may predate the verdict file's
+        # mtime when the verdict was rewritten later; the absence of any
+        # sidecar, not the timestamps, is the ownership evidence here.)
+        if codex_log.is_file():
+            match = re.search(
+                r"^model: (\S+)$",
+                codex_log.read_text(encoding="utf-8", errors="replace"),
+                re.M,
+            )
+            if not match:
+                counts["left_alone"] += 1
+                continue
+            meta = {
+                "judge_runner": "scripts/run_audit_codex.sh",
+                "verdict_sha256": digest,
+                "judge_model_requested": "default",
+                "judge_model_reported": [match.group(1)],
+                "judged_at_utc": _utc(verdict_mtime),
+                "backfilled_from": "codex.log",
+            }
+            if not dry_run:
+                meta_path.write_text(json.dumps(meta, indent=2, sort_keys=True))
+            counts["codex_sidecar_written"] += 1
+            continue
+        counts["left_alone"] += 1
+    return counts
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("cases_dir", nargs="?", default=str(AUDIT_CASES_DIR))
+    parser.add_argument("--tolerance", type=float, default=600.0)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    counts = backfill(
+        Path(args.cases_dir), tolerance=args.tolerance, dry_run=args.dry_run
+    )
+    print(json.dumps(counts))
+
+
+if __name__ == "__main__":
+    main()

scripts/freeze_snapshot.py

@@ -162,14 +162,64 @@
 }
 
 
-def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict:
+def verdict_provenance(case_dir: Path) -> dict | None:
+    """The ``verdict.meta.json`` sidecar for a case, if it describes the
+    case's current ``verdict.json``.
+
+    Both runners write the sidecar with the verdict's sha256. A sidecar whose
+    hash does not match (or that predates the hash field) belongs to an
+    earlier verdict, most likely one the other runner has since replaced, and
+    is ignored so a re-judged case cannot keep stale provenance.
+    """
+    meta_path = case_dir / "verdict.meta.json"
+    verdict_path = case_dir / "verdict.json"
+    if not meta_path.is_file() or not verdict_path.is_file():
+        return None
+    meta = json.loads(meta_path.read_text())
+    if meta.get("verdict_sha256") != sha256_file(verdict_path):
+        return None
+    return meta
+
+
+# A codex.log may stand in for a missing sidecar only when it was written
+# alongside the current verdict: the Codex runner writes the log and, on
+# success, the verdict within the same call.
+LOG_FALLBACK_TOLERANCE_SECONDS = 600.0
+
+
+def codex_log_describes_verdict(
+    case_dir: Path, tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS
+) -> bool:
+    """Whether ``codex.log`` is evidence about the case's current verdict.
+
+    Only when no sidecar exists at all (the Claude runner always wrote one, so
+    a sidecar-less verdict can only come from the Codex runner) and the log's
+    modification time sits within ``tolerance`` seconds of the verdict's. A
+    log left beside a later verdict from the other runner, or one hours older
+    than the verdict, describes an earlier attempt and proves nothing.
+    """
+    codex_log = case_dir / "codex.log"
+    verdict_path = case_dir / "verdict.json"
+    if not codex_log.is_file() or not verdict_path.is_file():
+        return False
+    if (case_dir / "verdict.meta.json").is_file():
+        return False
+    return abs(codex_log.stat().st_mtime - verdict_path.stat().st_mtime) <= tolerance
+
+
+def audit_judge_provenance(
+    cases_dir: Path = AUDIT_CASES_DIR,
+    log_tolerance: float = LOG_FALLBACK_TOLERANCE_SECONDS,
+) -> dict:
     """Tally which judge model produced each case verdict in the audit tree.
 
-    A case with a ``verdict.meta.json`` sidecar was judged (or re-judged) by
-    the Claude Code runner, which records the judge model it requested; the
-    Codex runner records its model in the ``model:`` line of ``codex.log``.
-    A case with neither is counted under ``unknown`` so the manifest cannot
-    silently claim provenance it does not have.
+    Each runner writes a ``verdict.meta.json`` sidecar bound to its verdict by
+    sha256 (the model requested and the model the CLI reported); only a
+    sidecar that matches the case's current verdict counts. A case with no
+    sidecar at all whose ``codex.log`` was written alongside the verdict is
+    read from the log's ``model:`` line. Anything else, including a stale or
+    hash-less sidecar beside an unrelated log, is counted under ``unknown``
+    so the manifest cannot silently claim provenance it does not have.
     """
     if not cases_dir.is_dir():
         raise SystemExit(f"Audit case tree not found: {cases_dir}")
@@ -179,17 +229,23 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict:
         if not (case_dir / "verdict.json").is_file():
             continue
         judged += 1
-        meta_path = case_dir / "verdict.meta.json"
         codex_log = case_dir / "codex.log"
-        if meta_path.is_file():
-            meta = json.loads(meta_path.read_text())
+        meta = verdict_provenance(case_dir)
+        if meta is not None:
             judge = meta["judge_model_requested"]
             reported = meta.get("judge_model_reported") or []
             if judge in {"opus", "claude-opus-5"} and "claude-opus-5" in reported:
                 judge = "claude-opus-5"
-            runner = JUDGE_RUNNERS["claude"]
+            if judge == "default" and len(reported) == 1:
+                judge = reported[0]
+            runner_key = (
+                "codex"
+                if "run_audit_codex" in str(meta.get("judge_runner", ""))
+                else "claude"
+            )
+            runner = JUDGE_RUNNERS[runner_key]
             day = str(meta.get("judged_at_utc", ""))[:10]
-        elif codex_log.is_file():
+        elif codex_log_describes_verdict(case_dir, log_tolerance):
             match = re.search(r"^model: (\S+)$", codex_log.read_text(), re.M)
             judge = match.group(1) if match else "unknown"
             runner = JUDGE_RUNNERS["codex"]
@@ -208,10 +264,12 @@ def audit_judge_provenance(cases_dir: Path = AUDIT_CASES_DIR) -> dict:
         "cases_judged": judged,
         "by_judge": dict(sorted(by_judge.items())),
         "note": (
-            "Judge model per case: a verdict.meta.json sidecar (Claude Code "
-            "runner) or the codex.log model header (Codex runner) in the audit "
-            "tree. Verdicts classify misses after scoring and change no score. "
-            "Both judge models are also board rows."
+            "Judge model per case: the verdict.meta.json sidecar bound to the "
+            "case's verdict by sha256 (either runner), else, for a case with no "
+            "sidecar, the codex.log model header when the log was written "
+            "alongside the verdict (Codex runner before it wrote sidecars). "
+            "Verdicts classify misses after scoring and change no score. Both "
+            "judge models are also board rows."
         ),
     }
 

scripts/run_audit_claude.sh

@@ -69,7 +69,7 @@ verdict_ok() {
 extract_verdict() {
   envelope="$1"; out_tmp="$2"; meta_tmp="$3"; requested_model="$4"; cli_version="$5"
   "$PYTHON" - "$envelope" "$out_tmp" "$meta_tmp" "$requested_model" "$cli_version" <<'PY'
-import datetime, json, sys
+import datetime, hashlib, json, sys
 envelope_path, out_path, meta_path, requested_model, cli_version = sys.argv[1:6]
 try:
     envelope = json.load(open(envelope_path))
@@ -89,9 +89,13 @@ if verdict is None:
         sys.exit(1)
 if not isinstance(verdict, dict):
     sys.exit(1)
-json.dump(verdict, open(out_path, "w"), indent=2, sort_keys=True)
+verdict_bytes = json.dumps(verdict, indent=2, sort_keys=True).encode("utf-8")
+open(out_path, "wb").write(verdict_bytes)
 meta = {
     "judge_runner": "scripts/run_audit_claude.sh",
+    # Binds the sidecar to this verdict: a sidecar whose hash does not match
+    # the case's verdict.json is stale and carries no provenance.
+    "verdict_sha256": hashlib.sha256(verdict_bytes).hexdigest(),
     "judge_model_requested": requested_model,
     "judge_model_reported": sorted((envelope.get("modelUsage") or {}).keys()),
     "judge_cli_version": cli_version,
@@ -113,7 +117,9 @@ classify_one() {
   envelope="$case_dir/claude.json"
   [ -f "$prompt" ] || return 0
   verdict_ok "$out" && return 0
-  rm -f "$tmp" "$meta_tmp" "$envelope"
+  # No valid verdict: any sidecar left behind describes a verdict that no
+  # longer exists (re-prepared case) and must not outlive it.
+  rm -f "$tmp" "$meta_tmp" "$envelope" "$case_dir/verdict.meta.json"
   # Self-contained prompt, no tools, no project instructions; the schema
   # enforces the JSON shape. Publish atomically only once it validates.
   CLAUDE_CODE_SAFE_MODE=1 CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 \

scripts/run_audit_codex.sh

@@ -9,6 +9,12 @@
 # Resumable: a case is skipped once it has a verdict.json carrying the required
 # keys. Re-run freely after interruptions or to fill in failures. Concurrency,
 # model, and reasoning effort are tunable via env. Portable to bash 3.2 (macOS).
+#
+# Judge provenance: beside each verdict.json the runner writes verdict.meta.json
+# (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex
+# reports in its log header, the model requested, the UTC timestamp, and the
+# verdict's sha256, so a re-judged case can never keep the other runner's
+# provenance.
 set -u
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -65,8 +71,9 @@ classify_one() {
   # Self-contained prompt; read-only sandbox; enforce the JSON shape. Write to a
   # temp file and publish atomically only once it validates, so an interrupted
   # run never leaves a half-written verdict that looks done. Default reasoning
-  # effort (xhigh) is wasteful for classification, so it is lowered.
-  rm -f "$tmp"
+  # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar
+  # left from a previous verdict describes a verdict that no longer exists.
+  rm -f "$tmp" "$case_dir/verdict.meta.json"
   codex exec \
     --sandbox read-only \
     --skip-git-repo-check \
@@ -77,15 +84,44 @@ classify_one() {
     --output-schema "$SCHEMA" \
     -o "$tmp" \
     - < "$prompt" > "$case_dir/codex.log" 2>&1
-  if verdict_ok "$tmp"; then
+  if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \
+      "$case_dir/verdict.meta.json.tmp"; then
     mv -f "$tmp" "$out"
+    mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json"
     echo "[ok] $(basename "$case_dir")"
   else
-    rm -f "$tmp"
+    rm -f "$tmp" "$case_dir/verdict.meta.json.tmp"
     echo "[FAIL] $(basename "$case_dir") (see codex.log)"
   fi
 }
 
+# Provenance sidecar for a validated verdict: the model Codex reports in its
+# log header (`model: ...`), the model requested (or "default"), the UTC time,
+# and the verdict's sha256 so the sidecar cannot outlive the verdict it describes.
+write_provenance() {
+  verdict_path="$1"; log_path="$2"; meta_out="$3"
+  "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" <<'PY'
+import datetime, hashlib, json, re, sys
+verdict_path, log_path, meta_out, requested = sys.argv[1:5]
+verdict = open(verdict_path, "rb").read()
+reported = []
+try:
+    match = re.search(r"^model: (\S+)$", open(log_path, encoding="utf-8", errors="replace").read(), re.M)
+    if match:
+        reported = [match.group(1)]
+except OSError:
+    pass
+meta = {
+    "judge_runner": "scripts/run_audit_codex.sh",
+    "verdict_sha256": hashlib.sha256(verdict).hexdigest(),
+    "judge_model_requested": requested,
+    "judge_model_reported": reported,
+    "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+}
+json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True)
+PY
+}
+
 total=$(find "$CASES_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
 echo "audit: $total cases | parallel=$PARALLEL effort=$EFFORT model=${AUDIT_MODEL:-default}"
 

tests/test_audit.py

@@ -498,6 +498,8 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path):
     s0 = cases[0]
     verdict_path = audit_dir / "cases" / s0.case_id / "verdict.json"
     verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}')
+    meta_path = verdict_path.with_name("verdict.meta.json")
+    meta_path.write_text('{"judge_model_requested": "opus"}')
 
     # Re-run m1 with a different wrong answer -> the case prompt changes.
     pd.DataFrame(
@@ -514,6 +516,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path):
     ).to_csv(d / "predictions.csv", index=False)
     prepare_audit(d, audit_dir)
     assert not verdict_path.exists()  # stale verdict dropped
+    assert not meta_path.exists()  # and its provenance sidecar with it
 
     # An unchanged case keeps its verdict.
     verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}')