feat: add versioned evaluation report regression diff

apps/ai-sre-assistant/evals/run_evals.py

@@ -1,8 +1,10 @@
 import argparse
 import json
+from pathlib import Path
 
 from evals.runner import (
     RUBRIC_DIMENSIONS,
+    diff_versioned_reports,
     run_comparison_suite,
     run_provider_suite,
     run_suite,
@@ -28,8 +30,24 @@ def main() -> int:
         action="store_true",
         help="Run deterministic evaluation and print the versioned machine-readable report.",
     )
+    report_group.add_argument(
+        "--diff",
+        metavar="BASELINE_JSON",
+        help=(
+            "Run deterministic evaluation, compare it against an earlier "
+            "--json report saved at BASELINE_JSON, and print a bounded diff "
+            "of case status changes and hard-gate regressions."
+        ),
+    )
     args = parser.parse_args()
 
+    if args.diff:
+        baseline = json.loads(Path(args.diff).read_text(encoding="utf-8"))
+        current = run_versioned_suite()
+        diff = diff_versioned_reports(baseline, current)
+        print(json.dumps(diff, indent=2, sort_keys=True))
+        return 0 if current["summary"]["passed"] and not diff["has_regression"] else 1
+
     if args.provider_report:
         suite = run_provider_suite()
         print(json.dumps(suite, indent=2, sort_keys=True))

apps/ai-sre-assistant/evals/runner.py

@@ -394,6 +394,73 @@ def run_versioned_suite(
     }
 
 
+DIFF_REPORT_SCHEMA_VERSION = "week6-diff-v1"
+
+
+def diff_versioned_reports(
+    baseline: dict[str, Any], current: dict[str, Any]
+) -> dict[str, Any]:
+    """Compare two versioned evaluation reports and surface what changed.
+
+    Both reports are expected to come from `run_versioned_suite` (or an
+    equivalent `--json` report saved earlier). The diff stays bounded to case
+    IDs, pass/fail booleans, and rubric dimension names, matching the same
+    privacy boundary the versioned report already keeps.
+    """
+    baseline_results = {result["id"]: result for result in baseline.get("results", [])}
+    current_results = {result["id"]: result for result in current.get("results", [])}
+    baseline_ids = set(baseline_results)
+    current_ids = set(current_results)
+
+    status_changes = [
+        {
+            "id": case_id,
+            "baseline_passed": baseline_results[case_id]["passed"],
+            "current_passed": current_results[case_id]["passed"],
+        }
+        for case_id in sorted(baseline_ids & current_ids)
+        if baseline_results[case_id]["passed"] != current_results[case_id]["passed"]
+    ]
+    regressions = sorted(
+        change["id"]
+        for change in status_changes
+        if change["baseline_passed"] and not change["current_passed"]
+    )
+    fixed = sorted(
+        change["id"]
+        for change in status_changes
+        if not change["baseline_passed"] and change["current_passed"]
+    )
+
+    baseline_hard_gates = baseline.get("hard_gates", {})
+    current_hard_gates = current.get("hard_gates", {})
+    hard_gate_regressions = [
+        dimension
+        for dimension in RUBRIC_DIMENSIONS
+        if baseline_hard_gates.get(dimension) is True
+        and current_hard_gates.get(dimension) is False
+    ]
+
+    return {
+        "schema_version": DIFF_REPORT_SCHEMA_VERSION,
+        "report_type": "deterministic_evaluation_diff",
+        "baseline_corpus_version": baseline.get("corpus", {}).get("version"),
+        "current_corpus_version": current.get("corpus", {}).get("version"),
+        "corpus_version_changed": (
+            baseline.get("corpus", {}).get("version")
+            != current.get("corpus", {}).get("version")
+        ),
+        "cases_added": sorted(current_ids - baseline_ids),
+        "cases_removed": sorted(baseline_ids - current_ids),
+        "status_changes": status_changes,
+        "regressions": regressions,
+        "fixed": fixed,
+        "hard_gate_regressions": hard_gate_regressions,
+        "current_summary": current.get("summary"),
+        "has_regression": bool(regressions) or bool(hard_gate_regressions),
+    }
+
+
 def _validate_manifest(manifest: Any) -> None:
     if not isinstance(manifest, dict):
         raise ValueError("Evaluation manifest must be a JSON object.")

apps/ai-sre-assistant/tests/test_evaluation.py

@@ -292,3 +292,74 @@ def test_versioned_evaluation_report_rejects_manifest_that_weakens_hard_gates():
 
     with pytest.raises(ValueError, match="minimum_score"):
         run_versioned_suite(CASES[:1], manifest)
+
+
+def test_diff_reports_identical_runs_show_no_changes():
+    from evals.runner import diff_versioned_reports, run_versioned_suite
+
+    report = run_versioned_suite(CASES[:2])
+
+    diff = diff_versioned_reports(report, report)
+
+    assert diff["corpus_version_changed"] is False
+    assert diff["cases_added"] == []
+    assert diff["cases_removed"] == []
+    assert diff["status_changes"] == []
+    assert diff["regressions"] == []
+    assert diff["fixed"] == []
+    assert diff["hard_gate_regressions"] == []
+    assert diff["has_regression"] is False
+
+
+def test_diff_reports_detects_case_regression_and_hard_gate_regression():
+    from evals.runner import diff_versioned_reports, run_versioned_suite
+
+    baseline = run_versioned_suite(CASES[:2])
+    current = run_versioned_suite(CASES[:2])
+    regressed_case = dict(current["results"][0])
+    regressed_case["passed"] = False
+    current["results"] = [regressed_case, current["results"][1]]
+    current["hard_gates"] = {**current["hard_gates"], "private": False}
+
+    diff = diff_versioned_reports(baseline, current)
+
+    assert diff["status_changes"] == [
+        {
+            "id": regressed_case["id"],
+            "baseline_passed": True,
+            "current_passed": False,
+        }
+    ]
+    assert diff["regressions"] == [regressed_case["id"]]
+    assert diff["fixed"] == []
+    assert diff["hard_gate_regressions"] == ["private"]
+    assert diff["has_regression"] is True
+
+
+def test_diff_reports_detects_added_and_removed_cases():
+    from evals.runner import diff_versioned_reports, run_versioned_suite
+
+    baseline = run_versioned_suite(CASES[:1])
+    current = run_versioned_suite(CASES[:2])
+
+    diff = diff_versioned_reports(baseline, current)
+
+    assert diff["cases_added"] == [CASES[1]["id"]]
+    assert diff["cases_removed"] == []
+    assert diff["has_regression"] is False
+
+
+def test_diff_reports_recognizes_a_fixed_case_without_flagging_regression():
+    from evals.runner import diff_versioned_reports, run_versioned_suite
+
+    baseline = run_versioned_suite(CASES[:2])
+    fixed_case = dict(baseline["results"][0])
+    fixed_case["passed"] = False
+    baseline = {**baseline, "results": [fixed_case, baseline["results"][1]]}
+    current = run_versioned_suite(CASES[:2])
+
+    diff = diff_versioned_reports(baseline, current)
+
+    assert diff["fixed"] == [fixed_case["id"]]
+    assert diff["regressions"] == []
+    assert diff["has_regression"] is False

apps/ai-sre-assistant/tests/test_run_evals_cli.py

@@ -0,0 +1,78 @@
+import json
+
+from evals import run_evals
+from evals.runner import load_cases
+
+
+def test_diff_flag_reports_no_regression_against_its_own_baseline(
+    tmp_path, capsys, monkeypatch
+):
+    baseline_path = tmp_path / "baseline.json"
+
+    monkeypatch.setattr("sys.argv", ["run_evals", "--json"])
+    exit_code = run_evals.main()
+    assert exit_code == 0
+    baseline_path.write_text(capsys.readouterr().out)
+
+    monkeypatch.setattr("sys.argv", ["run_evals", "--diff", str(baseline_path)])
+    exit_code = run_evals.main()
+    diff = json.loads(capsys.readouterr().out)
+
+    assert exit_code == 0
+    assert diff["has_regression"] is False
+    assert diff["cases_added"] == []
+    assert diff["cases_removed"] == []
+
+
+def test_diff_flag_reports_every_case_as_added_against_an_empty_baseline(
+    tmp_path, capsys, monkeypatch
+):
+    baseline_path = tmp_path / "baseline.json"
+    baseline_path.write_text(
+        json.dumps(
+            {
+                "schema_version": "1.0",
+                "report_type": "deterministic_evaluation",
+                "corpus": {"version": "0000.00.0", "case_count": 0, "case_ids": []},
+                "rubric": {
+                    "version": "1.0",
+                    "dimensions": [
+                        "grounded",
+                        "useful",
+                        "safe",
+                        "private",
+                        "honest",
+                    ],
+                    "acceptance_threshold": {
+                        "minimum_score": 5,
+                        "require_all_dimensions": True,
+                    },
+                },
+                "summary": {
+                    "passed": True,
+                    "cases_passed": 0,
+                    "cases_total": 0,
+                    "checks_passed": 0,
+                    "checks_total": 0,
+                },
+                "hard_gates": {
+                    "grounded": True,
+                    "useful": True,
+                    "safe": True,
+                    "private": True,
+                    "honest": True,
+                },
+                "results": [],
+            }
+        )
+    )
+
+    monkeypatch.setattr("sys.argv", ["run_evals", "--diff", str(baseline_path)])
+    exit_code = run_evals.main()
+    diff = json.loads(capsys.readouterr().out)
+
+    assert exit_code == 0
+    assert diff["corpus_version_changed"] is True
+    assert sorted(diff["cases_added"]) == sorted(case["id"] for case in load_cases())
+    assert diff["cases_removed"] == []
+    assert diff["has_regression"] is False

docs/09-roadmap.md

@@ -55,9 +55,7 @@ See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request
 - Day 2 - complete: expand the sanitized deterministic corpus with generic server failures, mixed signals, and client-only errors.
 - Day 3 - complete: add adversarial prompt-injection and unsupported-root-cause cases that enforce safe, evidence-grounded behavior.
 - Day 4 - complete: expand the corpus with redacted JWT, AWS-style key, GitHub-style token, and inline-credential edge cases.
-- Version the corpus, assistant configuration, and acceptance thresholds together.
-- Produce machine-readable evaluation results in CI.
-- Keep privacy and safety as hard release gates.
+- Day 5 - complete: add a bounded regression diff between two versioned reports so a case or hard-gate regression is easy to spot in a pull request.
 
 **Exit gate:** a model, prompt, provider, or code change produces a repeatable regression report and cannot bypass required privacy or safety checks.
 

docs/17-assistant-evaluation.md

@@ -107,3 +107,23 @@ python -m evals.run_evals --comparison-report
 It runs the deterministic corpus and the selected optional-provider path against the same fixtures, then reports the two quality summaries, bounded provider/model identity, outcome counts, fallback count, and estimated cost summary. `comparison.status` is `ready_to_compare` only when deterministic quality passes, the provider succeeds for every case, and cost is complete. Other explicit states distinguish unconfigured providers, request failures, incomplete cost data, and deterministic-gate failures.
 
 The command prints summaries only: it excludes fixture evidence, prompts, provider output, credentials, and endpoints. It is opt-in and outside CI because a configured provider makes one call per fixture. With `LLM_PROVIDER=none`, it is an offline end-to-end check that verifies the deterministic fallback remains available.
+
+## Regression Diff
+
+Week 6, Day 5 makes a versioned report easy to compare, not only easy to parse.
+
+Save a baseline report, then diff the current corpus against it:
+
+```bash
+python -m evals.run_evals --json > baseline.json
+python -m evals.run_evals --diff baseline.json
+```
+
+A typical use is comparing a pull request's evaluation report against `main`'s: check out `main`, save its `--json` output as the baseline, check out the branch again, then run `--diff` against that file. The output stays bounded to case IDs, pass/fail booleans, and rubric dimension names:
+
+- `cases_added` / `cases_removed` when the corpus itself changed.
+- `status_changes` for any case whose pass/fail flipped, split into `regressions` (previously passing, now failing) and `fixed` (previously failing, now passing).
+- `hard_gate_regressions` for any rubric dimension that used to hold across every case and no longer does.
+- `corpus_version_changed` when the manifest version moved.
+
+The command exits non-zero when the current run fails its own acceptance threshold, or when the diff shows a regression, so a reviewer does not have to read two full JSON reports side by side to notice one case quietly stopped passing.

docs/build-log.md

@@ -881,3 +881,31 @@ Lessons learned:
 - Every commit in a pull request's history gets scanned, not just the final diff; a later commit that "fixes" a fixture does not un-expose an earlier one. The dependable fix touches history, not just the tip commit.
 
 Next: make regression differences easier to inspect in CI.
+
+## Week 6, Day 5 - Regression Diff And Week Closeout
+
+Today I closed Week 6 by making the versioned report easy to compare, not only easy to parse, and reviewed the week against its exit gate.
+
+What changed:
+
+- Added `diff_versioned_reports`, which compares two `--json` reports and returns which case IDs were added or removed, which cases flipped pass/fail (split into `regressions` and `fixed`), which rubric dimension lost its hard gate, and whether the corpus version moved.
+- Added `python -m evals.run_evals --diff BASELINE_JSON`, which runs the current corpus, diffs it against a saved baseline report, and exits non-zero on either a failing current run or a detected regression.
+- Kept the diff bounded to case IDs, pass/fail booleans, and dimension names, the same privacy boundary the versioned report already holds.
+- Added regression tests for identical runs, a case regression, an added/removed case, and a fixed case, plus a CLI-level test exercising `--diff` end to end.
+- Replaced the leftover Week 6 bullets with a single Day 5 line; they restated what Day 1 had already shipped.
+
+Why this matters:
+
+A machine-readable report only helps a reviewer if they can tell what changed. Two full JSON dumps from two different runs bury the one line that matters, a case ID whose `passed` value flipped, inside dozens of unchanged fields. The diff makes that comparison the artifact.
+
+Week 6 exit-gate review:
+
+The exit gate asked for a repeatable regression report that cannot bypass required privacy or safety checks. That has been true since Day 1's hard-gate enforcement, and the corpus has since grown from 7 to 14 cases across generic failures, mixed signals, client-only errors, adversarial prompts, unsupported claims, and now multi-pattern redaction. Day 5 closes the remaining gap: the report is not just produced, it is easy to act on.
+
+Lessons learned:
+
+- A versioned report and a diff of two versioned reports are different tools; the report proves what happened, the diff proves what changed.
+- Keeping the diff to the same bounded fields as the report avoids reopening the privacy boundary just to make regressions readable.
+- A regression report earns its exit gate only once someone can look at it and immediately know what to do next.
+
+Next: start Week 7 with structured stdout logs and cross-service correlation fields.