diff --git a/apps/ai-sre-assistant/evals/run_evals.py b/apps/ai-sre-assistant/evals/run_evals.py index 8807a8c..22f16e9 100644 --- a/apps/ai-sre-assistant/evals/run_evals.py +++ b/apps/ai-sre-assistant/evals/run_evals.py @@ -1,8 +1,10 @@ import argparse import json +from pathlib import Path from evals.runner import ( RUBRIC_DIMENSIONS, + diff_versioned_reports, run_comparison_suite, run_provider_suite, run_suite, @@ -28,8 +30,24 @@ def main() -> int: action="store_true", help="Run deterministic evaluation and print the versioned machine-readable report.", ) + report_group.add_argument( + "--diff", + metavar="BASELINE_JSON", + help=( + "Run deterministic evaluation, compare it against an earlier " + "--json report saved at BASELINE_JSON, and print a bounded diff " + "of case status changes and hard-gate regressions." + ), + ) args = parser.parse_args() + if args.diff: + baseline = json.loads(Path(args.diff).read_text(encoding="utf-8")) + current = run_versioned_suite() + diff = diff_versioned_reports(baseline, current) + print(json.dumps(diff, indent=2, sort_keys=True)) + return 0 if current["summary"]["passed"] and not diff["has_regression"] else 1 + if args.provider_report: suite = run_provider_suite() print(json.dumps(suite, indent=2, sort_keys=True)) diff --git a/apps/ai-sre-assistant/evals/runner.py b/apps/ai-sre-assistant/evals/runner.py index 9f5dc8e..aad8dec 100644 --- a/apps/ai-sre-assistant/evals/runner.py +++ b/apps/ai-sre-assistant/evals/runner.py @@ -394,6 +394,73 @@ def run_versioned_suite( } +DIFF_REPORT_SCHEMA_VERSION = "week6-diff-v1" + + +def diff_versioned_reports( + baseline: dict[str, Any], current: dict[str, Any] +) -> dict[str, Any]: + """Compare two versioned evaluation reports and surface what changed. + + Both reports are expected to come from `run_versioned_suite` (or an + equivalent `--json` report saved earlier). The diff stays bounded to case + IDs, pass/fail booleans, and rubric dimension names, matching the same + privacy boundary the versioned report already keeps. + """ + baseline_results = {result["id"]: result for result in baseline.get("results", [])} + current_results = {result["id"]: result for result in current.get("results", [])} + baseline_ids = set(baseline_results) + current_ids = set(current_results) + + status_changes = [ + { + "id": case_id, + "baseline_passed": baseline_results[case_id]["passed"], + "current_passed": current_results[case_id]["passed"], + } + for case_id in sorted(baseline_ids & current_ids) + if baseline_results[case_id]["passed"] != current_results[case_id]["passed"] + ] + regressions = sorted( + change["id"] + for change in status_changes + if change["baseline_passed"] and not change["current_passed"] + ) + fixed = sorted( + change["id"] + for change in status_changes + if not change["baseline_passed"] and change["current_passed"] + ) + + baseline_hard_gates = baseline.get("hard_gates", {}) + current_hard_gates = current.get("hard_gates", {}) + hard_gate_regressions = [ + dimension + for dimension in RUBRIC_DIMENSIONS + if baseline_hard_gates.get(dimension) is True + and current_hard_gates.get(dimension) is False + ] + + return { + "schema_version": DIFF_REPORT_SCHEMA_VERSION, + "report_type": "deterministic_evaluation_diff", + "baseline_corpus_version": baseline.get("corpus", {}).get("version"), + "current_corpus_version": current.get("corpus", {}).get("version"), + "corpus_version_changed": ( + baseline.get("corpus", {}).get("version") + != current.get("corpus", {}).get("version") + ), + "cases_added": sorted(current_ids - baseline_ids), + "cases_removed": sorted(baseline_ids - current_ids), + "status_changes": status_changes, + "regressions": regressions, + "fixed": fixed, + "hard_gate_regressions": hard_gate_regressions, + "current_summary": current.get("summary"), + "has_regression": bool(regressions) or bool(hard_gate_regressions), + } + + def _validate_manifest(manifest: Any) -> None: if not isinstance(manifest, dict): raise ValueError("Evaluation manifest must be a JSON object.") diff --git a/apps/ai-sre-assistant/tests/test_evaluation.py b/apps/ai-sre-assistant/tests/test_evaluation.py index 6a209a6..24f436c 100644 --- a/apps/ai-sre-assistant/tests/test_evaluation.py +++ b/apps/ai-sre-assistant/tests/test_evaluation.py @@ -292,3 +292,74 @@ def test_versioned_evaluation_report_rejects_manifest_that_weakens_hard_gates(): with pytest.raises(ValueError, match="minimum_score"): run_versioned_suite(CASES[:1], manifest) + + +def test_diff_reports_identical_runs_show_no_changes(): + from evals.runner import diff_versioned_reports, run_versioned_suite + + report = run_versioned_suite(CASES[:2]) + + diff = diff_versioned_reports(report, report) + + assert diff["corpus_version_changed"] is False + assert diff["cases_added"] == [] + assert diff["cases_removed"] == [] + assert diff["status_changes"] == [] + assert diff["regressions"] == [] + assert diff["fixed"] == [] + assert diff["hard_gate_regressions"] == [] + assert diff["has_regression"] is False + + +def test_diff_reports_detects_case_regression_and_hard_gate_regression(): + from evals.runner import diff_versioned_reports, run_versioned_suite + + baseline = run_versioned_suite(CASES[:2]) + current = run_versioned_suite(CASES[:2]) + regressed_case = dict(current["results"][0]) + regressed_case["passed"] = False + current["results"] = [regressed_case, current["results"][1]] + current["hard_gates"] = {**current["hard_gates"], "private": False} + + diff = diff_versioned_reports(baseline, current) + + assert diff["status_changes"] == [ + { + "id": regressed_case["id"], + "baseline_passed": True, + "current_passed": False, + } + ] + assert diff["regressions"] == [regressed_case["id"]] + assert diff["fixed"] == [] + assert diff["hard_gate_regressions"] == ["private"] + assert diff["has_regression"] is True + + +def test_diff_reports_detects_added_and_removed_cases(): + from evals.runner import diff_versioned_reports, run_versioned_suite + + baseline = run_versioned_suite(CASES[:1]) + current = run_versioned_suite(CASES[:2]) + + diff = diff_versioned_reports(baseline, current) + + assert diff["cases_added"] == [CASES[1]["id"]] + assert diff["cases_removed"] == [] + assert diff["has_regression"] is False + + +def test_diff_reports_recognizes_a_fixed_case_without_flagging_regression(): + from evals.runner import diff_versioned_reports, run_versioned_suite + + baseline = run_versioned_suite(CASES[:2]) + fixed_case = dict(baseline["results"][0]) + fixed_case["passed"] = False + baseline = {**baseline, "results": [fixed_case, baseline["results"][1]]} + current = run_versioned_suite(CASES[:2]) + + diff = diff_versioned_reports(baseline, current) + + assert diff["fixed"] == [fixed_case["id"]] + assert diff["regressions"] == [] + assert diff["has_regression"] is False diff --git a/apps/ai-sre-assistant/tests/test_run_evals_cli.py b/apps/ai-sre-assistant/tests/test_run_evals_cli.py new file mode 100644 index 0000000..8b3cf50 --- /dev/null +++ b/apps/ai-sre-assistant/tests/test_run_evals_cli.py @@ -0,0 +1,78 @@ +import json + +from evals import run_evals +from evals.runner import load_cases + + +def test_diff_flag_reports_no_regression_against_its_own_baseline( + tmp_path, capsys, monkeypatch +): + baseline_path = tmp_path / "baseline.json" + + monkeypatch.setattr("sys.argv", ["run_evals", "--json"]) + exit_code = run_evals.main() + assert exit_code == 0 + baseline_path.write_text(capsys.readouterr().out) + + monkeypatch.setattr("sys.argv", ["run_evals", "--diff", str(baseline_path)]) + exit_code = run_evals.main() + diff = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert diff["has_regression"] is False + assert diff["cases_added"] == [] + assert diff["cases_removed"] == [] + + +def test_diff_flag_reports_every_case_as_added_against_an_empty_baseline( + tmp_path, capsys, monkeypatch +): + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "report_type": "deterministic_evaluation", + "corpus": {"version": "0000.00.0", "case_count": 0, "case_ids": []}, + "rubric": { + "version": "1.0", + "dimensions": [ + "grounded", + "useful", + "safe", + "private", + "honest", + ], + "acceptance_threshold": { + "minimum_score": 5, + "require_all_dimensions": True, + }, + }, + "summary": { + "passed": True, + "cases_passed": 0, + "cases_total": 0, + "checks_passed": 0, + "checks_total": 0, + }, + "hard_gates": { + "grounded": True, + "useful": True, + "safe": True, + "private": True, + "honest": True, + }, + "results": [], + } + ) + ) + + monkeypatch.setattr("sys.argv", ["run_evals", "--diff", str(baseline_path)]) + exit_code = run_evals.main() + diff = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert diff["corpus_version_changed"] is True + assert sorted(diff["cases_added"]) == sorted(case["id"] for case in load_cases()) + assert diff["cases_removed"] == [] + assert diff["has_regression"] is False diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index dbce6f4..df40252 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -55,9 +55,7 @@ See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request - Day 2 - complete: expand the sanitized deterministic corpus with generic server failures, mixed signals, and client-only errors. - Day 3 - complete: add adversarial prompt-injection and unsupported-root-cause cases that enforce safe, evidence-grounded behavior. - Day 4 - complete: expand the corpus with redacted JWT, AWS-style key, GitHub-style token, and inline-credential edge cases. -- Version the corpus, assistant configuration, and acceptance thresholds together. -- Produce machine-readable evaluation results in CI. -- Keep privacy and safety as hard release gates. +- Day 5 - complete: add a bounded regression diff between two versioned reports so a case or hard-gate regression is easy to spot in a pull request. **Exit gate:** a model, prompt, provider, or code change produces a repeatable regression report and cannot bypass required privacy or safety checks. diff --git a/docs/17-assistant-evaluation.md b/docs/17-assistant-evaluation.md index 61e72f2..9e5342c 100644 --- a/docs/17-assistant-evaluation.md +++ b/docs/17-assistant-evaluation.md @@ -107,3 +107,23 @@ python -m evals.run_evals --comparison-report It runs the deterministic corpus and the selected optional-provider path against the same fixtures, then reports the two quality summaries, bounded provider/model identity, outcome counts, fallback count, and estimated cost summary. `comparison.status` is `ready_to_compare` only when deterministic quality passes, the provider succeeds for every case, and cost is complete. Other explicit states distinguish unconfigured providers, request failures, incomplete cost data, and deterministic-gate failures. The command prints summaries only: it excludes fixture evidence, prompts, provider output, credentials, and endpoints. It is opt-in and outside CI because a configured provider makes one call per fixture. With `LLM_PROVIDER=none`, it is an offline end-to-end check that verifies the deterministic fallback remains available. + +## Regression Diff + +Week 6, Day 5 makes a versioned report easy to compare, not only easy to parse. + +Save a baseline report, then diff the current corpus against it: + +```bash +python -m evals.run_evals --json > baseline.json +python -m evals.run_evals --diff baseline.json +``` + +A typical use is comparing a pull request's evaluation report against `main`'s: check out `main`, save its `--json` output as the baseline, check out the branch again, then run `--diff` against that file. The output stays bounded to case IDs, pass/fail booleans, and rubric dimension names: + +- `cases_added` / `cases_removed` when the corpus itself changed. +- `status_changes` for any case whose pass/fail flipped, split into `regressions` (previously passing, now failing) and `fixed` (previously failing, now passing). +- `hard_gate_regressions` for any rubric dimension that used to hold across every case and no longer does. +- `corpus_version_changed` when the manifest version moved. + +The command exits non-zero when the current run fails its own acceptance threshold, or when the diff shows a regression, so a reviewer does not have to read two full JSON reports side by side to notice one case quietly stopped passing. diff --git a/docs/build-log.md b/docs/build-log.md index 4eb968c..5268b9e 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -881,3 +881,31 @@ Lessons learned: - Every commit in a pull request's history gets scanned, not just the final diff; a later commit that "fixes" a fixture does not un-expose an earlier one. The dependable fix touches history, not just the tip commit. Next: make regression differences easier to inspect in CI. + +## Week 6, Day 5 - Regression Diff And Week Closeout + +Today I closed Week 6 by making the versioned report easy to compare, not only easy to parse, and reviewed the week against its exit gate. + +What changed: + +- Added `diff_versioned_reports`, which compares two `--json` reports and returns which case IDs were added or removed, which cases flipped pass/fail (split into `regressions` and `fixed`), which rubric dimension lost its hard gate, and whether the corpus version moved. +- Added `python -m evals.run_evals --diff BASELINE_JSON`, which runs the current corpus, diffs it against a saved baseline report, and exits non-zero on either a failing current run or a detected regression. +- Kept the diff bounded to case IDs, pass/fail booleans, and dimension names, the same privacy boundary the versioned report already holds. +- Added regression tests for identical runs, a case regression, an added/removed case, and a fixed case, plus a CLI-level test exercising `--diff` end to end. +- Replaced the leftover Week 6 bullets with a single Day 5 line; they restated what Day 1 had already shipped. + +Why this matters: + +A machine-readable report only helps a reviewer if they can tell what changed. Two full JSON dumps from two different runs bury the one line that matters, a case ID whose `passed` value flipped, inside dozens of unchanged fields. The diff makes that comparison the artifact. + +Week 6 exit-gate review: + +The exit gate asked for a repeatable regression report that cannot bypass required privacy or safety checks. That has been true since Day 1's hard-gate enforcement, and the corpus has since grown from 7 to 14 cases across generic failures, mixed signals, client-only errors, adversarial prompts, unsupported claims, and now multi-pattern redaction. Day 5 closes the remaining gap: the report is not just produced, it is easy to act on. + +Lessons learned: + +- A versioned report and a diff of two versioned reports are different tools; the report proves what happened, the diff proves what changed. +- Keeping the diff to the same bounded fields as the report avoids reopening the privacy boundary just to make regressions readable. +- A regression report earns its exit gate only once someone can look at it and immediately know what to do next. + +Next: start Week 7 with structured stdout logs and cross-service correlation fields.