diff --git a/METRICS.md b/METRICS.md index 26d2b9b..5e85b33 100644 --- a/METRICS.md +++ b/METRICS.md @@ -188,3 +188,42 @@ the signal does not catch confidently wrong rankings. - Sandbox repo; each case's diffs are small and fit the 3000-char truncation. - The baseline is already 90% top-1, leaving little room for runbook context to help; retrieval only delivers the right runbook in about half the cases. + +## Model comparison and re-run (2026-10-07) + +Same eval set v2, no runbooks, 24 cases × 3 trials per model. Raw results: +`eval/results/baseline_v2_sonnet5_rerun_2026-10-07.json` and +`eval/results/baseline_v2_haiku45_2026-10-07.json`; reproduce with +`python -m eval.report `. + +| Metric | Claude Sonnet 5 (re-run) | Claude Haiku 4.5 | +|---|---|---| +| Top-1 | **91.7% (66/72)** | 80.6% (58/72) | +| Top-3 | 100% | 100% | +| MRR | 0.958 | 0.903 | +| Null-runbook cases top-1 | 20/21 | 18/21 | +| Ranking latency, median | 6.5s | 4.5s | +| Tokens per call, mean (in / out) | 3066 / 665 | 2698 / 536 | +| Cost per ranking call, list price | ~$0.013 | ~$0.005 | + +- **Stability:** the Sonnet 5 re-run (91.7%) matches the Phase 9 baseline + (90.3%) within one trial, so the 90% figure is not a lucky run. +- **Haiku 4.5 is ~2.4x cheaper and ~2s faster per ranking but 11 points less + accurate** (paired: 2 case wins, 5 losses, 17 ties). Its losses include + `config_reformat_hidden_string` and `worker_queue_renamed` (0/3 each), the + cases where the decoy is most surface-plausible. At ~$0.013 per incident + the accuracy is worth more than the saving, so **Sonnet 5 stays the + default**. `SENTINEL_MODEL` makes the choice configurable. +- Haiku 4.5 runs without extended thinking by default, while Sonnet 5 thinks + adaptively; this compares the models as shipped by default, not at matched + reasoning effort. +- Costs use list prices ($2/$10 per MTok for Sonnet 5, $1/$5 for Haiku 4.5) + times the metered tokens, for the ranking call only (the postmortem is a + separate call). + +**Close-call rule, out of sample.** The 0.1 rule was chosen on the Phase 9 +trials. On the fresh Sonnet 5 re-run it flags **4/6 wrong picks with 7/66 +false alarms** (in-sample it was 9/13 and 9/131), so it holds. On Haiku 4.5 it +flags 7/14 wrong but 22/58 right picks, and Haiku's raw confidence is identical +for right and wrong picks (0.93 / 0.93): the rule is calibrated for Sonnet 5 +and should be re-checked if the model changes. diff --git a/core/services/llm.py b/core/services/llm.py index 456e4fb..24a3605 100644 --- a/core/services/llm.py +++ b/core/services/llm.py @@ -5,7 +5,7 @@ from dotenv import load_dotenv load_dotenv() -MODEL = "claude-sonnet-5" +MODEL = os.getenv("SENTINEL_MODEL", "claude-sonnet-5") @cache diff --git a/eval/report.py b/eval/report.py index 425d2cc..37021ee 100644 --- a/eval/report.py +++ b/eval/report.py @@ -40,9 +40,24 @@ def summarize(records: list[dict]) -> dict: "n_right": len(right), "n_wrong": len(wrong), "latency_median_s": statistics.median(r["rank_latency_s"] for r in records) if records else None, + # runs before token metering have no usage; report None rather than a fake zero + "tokens_in_mean": _mean_usage(records, "input_tokens"), + "tokens_out_mean": _mean_usage(records, "output_tokens"), } +def _mean_usage(records: list[dict], key: str): + vals = [r["usage"][key] for r in records if r.get("usage")] + return round(statistics.mean(vals)) if vals else None + + +def _label(a: dict, b: dict) -> tuple[str, str]: + """Name the two runs by whatever differs: condition for an A/B, model for a model comparison.""" + if a["condition"] != b["condition"]: + return a["condition"], b["condition"] + return a["model"], b["model"] + + def retrieval(records: list[dict]) -> dict: """Retrieval is deterministic per case, so score one record per case.""" per_case = {} @@ -114,6 +129,8 @@ def print_summary(name: str, res: dict): f"wrong {_fmt_conf(s['conf_wrong'])} (n={s['n_wrong']})" ) print(f" rank latency median {s['latency_median_s']}s (n={s['n']})") + if s["tokens_in_mean"] is not None: + print(f" tokens/call in {s['tokens_in_mean']}, out {s['tokens_out_mean']} (mean)") if res.get("aborted"): print(f" WARNING run ABORTED, not a valid measurement: {res['aborted']}") truncated = sorted({r["case_id"] for r in res["records"] if r["truncated_diffs"]}) @@ -137,20 +154,21 @@ def print_retrieval(res: dict): def print_comparison(a: dict, b: dict): rows = paired(a["records"], b["records"]) + la, lb = _label(a, b) wins = sum(hb / tb > ha / ta for _, ha, ta, hb, tb in rows) losses = sum(hb / tb < ha / ta for _, ha, ta, hb, tb in rows) - print(f"\n== Paired per-case top-1 hits ({a['condition']} vs {b['condition']})") - print(f" {'case':<34} {a['condition']:>9} {b['condition']:>9}") + print(f"\n== Paired per-case top-1 hits ({la} vs {lb})") + print(f" {'case':<34} {la:>15} {lb:>15}") for cid, ha, ta, hb, tb in rows: mark = "+" if hb / tb > ha / ta else "-" if hb / tb < ha / ta else " " - print(f" {cid:<34} {ha:>5}/{ta:<3} {hb:>5}/{tb:<3} {mark}") + print(f" {cid:<34} {ha:>11}/{ta:<3} {hb:>11}/{tb:<3} {mark}") ties = len(rows) - wins - losses - print(f" {b['condition']} wins {wins}, losses {losses}, ties {ties} (n={len(rows)} cases)") + print(f" {lb} wins {wins}, losses {losses}, ties {ties} (n={len(rows)} cases)") print("\n== Null-runbook cases only (does irrelevant context hurt?)") - for res in (a, b): + for label, res in ((la, a), (lb, b)): s = summarize([r for r in res["records"] if not r["expected_runbook_id"]]) - print(f" {res['condition']:<9} top-1 {_pct(s['top1'], s['n'])}, MRR {s['mrr']:.3f}") + print(f" {label:<15} top-1 {_pct(s['top1'], s['n'])}, MRR {s['mrr']:.3f}") def main(argv=None): diff --git a/eval/results/README.md b/eval/results/README.md index 22d69cc..e2c2dac 100644 --- a/eval/results/README.md +++ b/eval/results/README.md @@ -6,4 +6,6 @@ Each file records `sentinel_git_sha`, the commit the run used. Phase 9 was rebas those SHAs (`7b4dc90`, `2e1eb8d`, `530b73f`) live on the preserved branch `feat/phase9-rag-eval`, not on `main`. Do not delete that branch. +The 2026-10-07 model-comparison runs record `dfd2797`, preserved on branch `feat/model-comparison`; keep it too. + Runs under `aborted/` are kept for the record and are never scored as measurements. diff --git a/eval/results/baseline_v2_haiku45_2026-10-07.json b/eval/results/baseline_v2_haiku45_2026-10-07.json new file mode 100644 index 0000000..35872f9 --- /dev/null +++ b/eval/results/baseline_v2_haiku45_2026-10-07.json @@ -0,0 +1,3736 @@ +{ + "condition": "baseline", + "dry_run": false, + "eval_set_version": "v2", + "runbook_corpus_version": "v1", + "runbooks_ingested": 10, + "runbook_floor": 0.35, + "model": "claude-haiku-4-5", + "sentinel_git_sha": "dfd27973fd1c0e2bb514413e4e258782cc7b48fd", + "started_at": "2026-10-07T21:11:11.972865+00:00", + "trials": 3, + "n_cases": 24, + "records": [ + { + "trial": 0, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.95 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.25 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.05 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.02 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.78, + "usage": { + "input_tokens": 2729, + "output_tokens": 556 + } + }, + { + "trial": 0, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.95 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.15 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.05 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.02 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.12, + "usage": { + "input_tokens": 2582, + "output_tokens": 507 + } + }, + { + "trial": 0, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.95 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.85 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.25 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.05 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.36, + "usage": { + "input_tokens": 2657, + "output_tokens": 533 + } + }, + { + "trial": 0, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.85 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.15 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.0 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.75, + "usage": { + "input_tokens": 2254, + "output_tokens": 448 + } + }, + { + "trial": 0, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.95 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.65 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.35 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.15 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.11, + "usage": { + "input_tokens": 2852, + "output_tokens": 564 + } + }, + { + "trial": 0, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "3af442c", + "confidence_score": 0.85 + }, + { + "commit_hash": "c3629ef", + "confidence_score": 0.8 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.15 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.05 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.44, + "usage": { + "input_tokens": 2834, + "output_tokens": 551 + } + }, + { + "trial": 0, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.85 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.05 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.0 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.09, + "usage": { + "input_tokens": 2788, + "output_tokens": 509 + } + }, + { + "trial": 0, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "5cedb21", + "confidence_score": 0.95 + }, + { + "commit_hash": "6eea0fc", + "confidence_score": 0.72 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.15 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.02 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.01 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.59, + "usage": { + "input_tokens": 3050, + "output_tokens": 535 + } + }, + { + "trial": 0, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.95 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.15 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.0 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.0 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.95, + "usage": { + "input_tokens": 2596, + "output_tokens": 513 + } + }, + { + "trial": 0, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "741f699", + "confidence_score": 0.95 + }, + { + "commit_hash": "b35322f", + "confidence_score": 0.3 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.0 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 3.72, + "usage": { + "input_tokens": 2496, + "output_tokens": 436 + } + }, + { + "trial": 0, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.95 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.7 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.05 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.01 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.2, + "usage": { + "input_tokens": 2934, + "output_tokens": 512 + } + }, + { + "trial": 0, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.95 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.15 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.1 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.05 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.64, + "usage": { + "input_tokens": 2867, + "output_tokens": 548 + } + }, + { + "trial": 0, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2e11931", + "confidence_score": 0.85 + }, + { + "commit_hash": "2eeb222", + "confidence_score": 0.75 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.6 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.1 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.05 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.94, + "usage": { + "input_tokens": 2676, + "output_tokens": 572 + } + }, + { + "trial": 0, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.99 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.15 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.05 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.17, + "usage": { + "input_tokens": 2610, + "output_tokens": 535 + } + }, + { + "trial": 0, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.95 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.85 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.7 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.15 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.41, + "usage": { + "input_tokens": 2816, + "output_tokens": 532 + } + }, + { + "trial": 0, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.85 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.75 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.6 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.1 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.19, + "usage": { + "input_tokens": 2726, + "output_tokens": 523 + } + }, + { + "trial": 0, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.95 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.85 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.15 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.05 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.06, + "usage": { + "input_tokens": 2600, + "output_tokens": 529 + } + }, + { + "trial": 0, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.95 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.1 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.05 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.5, + "usage": { + "input_tokens": 3001, + "output_tokens": 556 + } + }, + { + "trial": 0, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.95 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.85 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.05 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.0 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.23, + "usage": { + "input_tokens": 2944, + "output_tokens": 524 + } + }, + { + "trial": 0, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "fd92e84", + "confidence_score": 0.85 + }, + { + "commit_hash": "0751831", + "confidence_score": 0.75 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.45 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.05 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.98, + "usage": { + "input_tokens": 2363, + "output_tokens": 599 + } + }, + { + "trial": 0, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.75 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.45 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.1 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.05 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.05 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.88, + "usage": { + "input_tokens": 2532, + "output_tokens": 568 + } + }, + { + "trial": 0, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "266c09c", + "confidence_score": 0.95 + }, + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.85 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.45 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.05 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.03, + "usage": { + "input_tokens": 2702, + "output_tokens": 526 + } + }, + { + "trial": 0, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "0f497ec", + "confidence_score": 0.95 + }, + { + "commit_hash": "f8d8258", + "confidence_score": 0.75 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.35 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.05 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.48, + "usage": { + "input_tokens": 2431, + "output_tokens": 559 + } + }, + { + "trial": 0, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.95 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.85 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.45 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.05 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.45, + "usage": { + "input_tokens": 2724, + "output_tokens": 555 + } + }, + { + "trial": 1, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.95 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.25 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.05 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.05 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.33, + "usage": { + "input_tokens": 2729, + "output_tokens": 553 + } + }, + { + "trial": 1, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.95 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.3 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.05 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.05 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.22, + "usage": { + "input_tokens": 2582, + "output_tokens": 516 + } + }, + { + "trial": 1, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.95 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.85 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.25 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.05 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.44, + "usage": { + "input_tokens": 2657, + "output_tokens": 548 + } + }, + { + "trial": 1, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.75 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.65 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.05 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.7, + "usage": { + "input_tokens": 2254, + "output_tokens": 453 + } + }, + { + "trial": 1, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.92 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.68 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.54 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.15 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.86, + "usage": { + "input_tokens": 2852, + "output_tokens": 568 + } + }, + { + "trial": 1, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "3af442c", + "confidence_score": 0.95 + }, + { + "commit_hash": "c3629ef", + "confidence_score": 0.85 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.3 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.1 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.88, + "usage": { + "input_tokens": 2834, + "output_tokens": 551 + } + }, + { + "trial": 1, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.7 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.1 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.05 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.03, + "usage": { + "input_tokens": 2788, + "output_tokens": 515 + } + }, + { + "trial": 1, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "5cedb21", + "confidence_score": 0.95 + }, + { + "commit_hash": "6eea0fc", + "confidence_score": 0.78 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.15 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.05 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 3.91, + "usage": { + "input_tokens": 3050, + "output_tokens": 481 + } + }, + { + "trial": 1, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.99 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.15 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.01 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.01 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.55, + "usage": { + "input_tokens": 2596, + "output_tokens": 534 + } + }, + { + "trial": 1, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "b35322f", + "confidence_score": 0.95 + }, + { + "commit_hash": "741f699", + "confidence_score": 0.92 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.05 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.44, + "usage": { + "input_tokens": 2496, + "output_tokens": 429 + } + }, + { + "trial": 1, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.95 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.72 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.05 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.02 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.91, + "usage": { + "input_tokens": 2934, + "output_tokens": 569 + } + }, + { + "trial": 1, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.85 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.65 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.1 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.05 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.5, + "usage": { + "input_tokens": 2867, + "output_tokens": 526 + } + }, + { + "trial": 1, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2e11931", + "confidence_score": 0.95 + }, + { + "commit_hash": "2eeb222", + "confidence_score": 0.85 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.15 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.05 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.59, + "usage": { + "input_tokens": 2676, + "output_tokens": 562 + } + }, + { + "trial": 1, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.95 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.05 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.02 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.98, + "usage": { + "input_tokens": 2610, + "output_tokens": 512 + } + }, + { + "trial": 1, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.95 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.85 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.6 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.05 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.69, + "usage": { + "input_tokens": 2816, + "output_tokens": 532 + } + }, + { + "trial": 1, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.95 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.75 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.15 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.05 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.2, + "usage": { + "input_tokens": 2726, + "output_tokens": 561 + } + }, + { + "trial": 1, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.95 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.85 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.15 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.05 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.11, + "usage": { + "input_tokens": 2600, + "output_tokens": 535 + } + }, + { + "trial": 1, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.95 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.05 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.02 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.59, + "usage": { + "input_tokens": 3001, + "output_tokens": 546 + } + }, + { + "trial": 1, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.95 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.85 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.1 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.05 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.77, + "usage": { + "input_tokens": 2944, + "output_tokens": 519 + } + }, + { + "trial": 1, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "0751831", + "confidence_score": 0.65 + }, + { + "commit_hash": "fd92e84", + "confidence_score": 0.35 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.25 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.05 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.2, + "usage": { + "input_tokens": 2363, + "output_tokens": 599 + } + }, + { + "trial": 1, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.95 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.3 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.05 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.02 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.64, + "usage": { + "input_tokens": 2532, + "output_tokens": 549 + } + }, + { + "trial": 1, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "266c09c", + "confidence_score": 0.95 + }, + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.85 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.05 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.02 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.45, + "usage": { + "input_tokens": 2702, + "output_tokens": 544 + } + }, + { + "trial": 1, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "0f497ec", + "confidence_score": 0.95 + }, + { + "commit_hash": "f8d8258", + "confidence_score": 0.75 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.2 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.05 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.3, + "usage": { + "input_tokens": 2431, + "output_tokens": 551 + } + }, + { + "trial": 1, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.95 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.45 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.4 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.05 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.59, + "usage": { + "input_tokens": 2724, + "output_tokens": 535 + } + }, + { + "trial": 2, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.95 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.35 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.1 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.05 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.05 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.53, + "usage": { + "input_tokens": 2729, + "output_tokens": 581 + } + }, + { + "trial": 2, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.95 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.4 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.05 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.02 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.39, + "usage": { + "input_tokens": 2582, + "output_tokens": 522 + } + }, + { + "trial": 2, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.95 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.85 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.4 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.05 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.38, + "usage": { + "input_tokens": 2657, + "output_tokens": 528 + } + }, + { + "trial": 2, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.85 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.25 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.05 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.66, + "usage": { + "input_tokens": 2254, + "output_tokens": 453 + } + }, + { + "trial": 2, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.85 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.65 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.55 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.15 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.05 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.03, + "usage": { + "input_tokens": 2852, + "output_tokens": 565 + } + }, + { + "trial": 2, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "3af442c", + "confidence_score": 0.95 + }, + { + "commit_hash": "c3629ef", + "confidence_score": 0.9 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.4 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.05 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 5.36, + "usage": { + "input_tokens": 2834, + "output_tokens": 615 + } + }, + { + "trial": 2, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.85 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.15 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.05 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.12, + "usage": { + "input_tokens": 2788, + "output_tokens": 513 + } + }, + { + "trial": 2, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "5cedb21", + "confidence_score": 0.95 + }, + { + "commit_hash": "6eea0fc", + "confidence_score": 0.75 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.15 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.05 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.53, + "usage": { + "input_tokens": 3050, + "output_tokens": 542 + } + }, + { + "trial": 2, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.95 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.05 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.0 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.0 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.86, + "usage": { + "input_tokens": 2596, + "output_tokens": 522 + } + }, + { + "trial": 2, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "741f699", + "confidence_score": 0.95 + }, + { + "commit_hash": "b35322f", + "confidence_score": 0.15 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.05 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 3.81, + "usage": { + "input_tokens": 2496, + "output_tokens": 439 + } + }, + { + "trial": 2, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.95 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.75 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.1 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.05 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.44, + "usage": { + "input_tokens": 2934, + "output_tokens": 549 + } + }, + { + "trial": 2, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.95 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.45 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.15 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.1 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.05 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.58, + "usage": { + "input_tokens": 2867, + "output_tokens": 538 + } + }, + { + "trial": 2, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2eeb222", + "confidence_score": 0.95 + }, + { + "commit_hash": "2e11931", + "confidence_score": 0.85 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.35 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.05 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.19, + "usage": { + "input_tokens": 2676, + "output_tokens": 579 + } + }, + { + "trial": 2, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.99 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.05 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.02 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 3.81, + "usage": { + "input_tokens": 2610, + "output_tokens": 498 + } + }, + { + "trial": 2, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.95 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.75 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.35 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.05 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.59, + "usage": { + "input_tokens": 2816, + "output_tokens": 553 + } + }, + { + "trial": 2, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.85 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.75 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.65 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.1 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.66, + "usage": { + "input_tokens": 2726, + "output_tokens": 550 + } + }, + { + "trial": 2, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.95 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.92 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.15 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.05 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.45, + "usage": { + "input_tokens": 2600, + "output_tokens": 558 + } + }, + { + "trial": 2, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.95 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.05 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.01 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.64, + "usage": { + "input_tokens": 3001, + "output_tokens": 548 + } + }, + { + "trial": 2, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.95 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.85 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.05 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.0 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.28, + "usage": { + "input_tokens": 2944, + "output_tokens": 540 + } + }, + { + "trial": 2, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "0751831", + "confidence_score": 0.85 + }, + { + "commit_hash": "fd92e84", + "confidence_score": 0.25 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.15 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.05 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.05 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.41, + "usage": { + "input_tokens": 2363, + "output_tokens": 566 + } + }, + { + "trial": 2, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.85 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.3 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.05 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.05 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.45, + "usage": { + "input_tokens": 2532, + "output_tokens": 561 + } + }, + { + "trial": 2, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "266c09c", + "confidence_score": 0.95 + }, + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.85 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.7 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.05 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.58, + "usage": { + "input_tokens": 2702, + "output_tokens": 563 + } + }, + { + "trial": 2, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "0f497ec", + "confidence_score": 0.95 + }, + { + "commit_hash": "f8d8258", + "confidence_score": 0.92 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.15 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.1 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.72, + "usage": { + "input_tokens": 2431, + "output_tokens": 558 + } + }, + { + "trial": 2, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.95 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.72 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.68 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.12 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 4.7, + "usage": { + "input_tokens": 2724, + "output_tokens": 550 + } + } + ], + "finished_at": "2026-10-07T21:21:23.757228+00:00" +} \ No newline at end of file diff --git a/eval/results/baseline_v2_sonnet5_rerun_2026-10-07.json b/eval/results/baseline_v2_sonnet5_rerun_2026-10-07.json new file mode 100644 index 0000000..e9d41ac --- /dev/null +++ b/eval/results/baseline_v2_sonnet5_rerun_2026-10-07.json @@ -0,0 +1,3736 @@ +{ + "condition": "baseline", + "dry_run": false, + "eval_set_version": "v2", + "runbook_corpus_version": "v1", + "runbooks_ingested": 10, + "runbook_floor": 0.35, + "model": "claude-sonnet-5", + "sentinel_git_sha": "dfd27973fd1c0e2bb514413e4e258782cc7b48fd", + "started_at": "2026-10-07T21:21:32.348350+00:00", + "trials": 3, + "n_cases": 24, + "records": [ + { + "trial": 0, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.97 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.05 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.03 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.02 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 8.81, + "usage": { + "input_tokens": 3108, + "output_tokens": 701 + } + }, + { + "trial": 0, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.97 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.15 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.02 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.02 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.22, + "usage": { + "input_tokens": 2900, + "output_tokens": 659 + } + }, + { + "trial": 0, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.97 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.55 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.05 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.01 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.17, + "usage": { + "input_tokens": 3034, + "output_tokens": 707 + } + }, + { + "trial": 0, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.75 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.4 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.08, + "usage": { + "input_tokens": 2555, + "output_tokens": 643 + } + }, + { + "trial": 0, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.85 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.55 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.15 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.05 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.59, + "usage": { + "input_tokens": 3255, + "output_tokens": 626 + } + }, + { + "trial": 0, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "c3629ef", + "confidence_score": 0.9 + }, + { + "commit_hash": "3af442c", + "confidence_score": 0.2 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.1 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.05 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.53, + "usage": { + "input_tokens": 3170, + "output_tokens": 685 + } + }, + { + "trial": 0, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.4 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.05 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.02 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.42, + "usage": { + "input_tokens": 3161, + "output_tokens": 679 + } + }, + { + "trial": 0, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "6eea0fc", + "confidence_score": 0.8 + }, + { + "commit_hash": "5cedb21", + "confidence_score": 0.75 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.3 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.05 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.56, + "usage": { + "input_tokens": 3504, + "output_tokens": 687 + } + }, + { + "trial": 0, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.97 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.1 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.02 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.01 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.8, + "usage": { + "input_tokens": 2902, + "output_tokens": 737 + } + }, + { + "trial": 0, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "741f699", + "confidence_score": 0.85 + }, + { + "commit_hash": "b35322f", + "confidence_score": 0.8 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.02 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.01 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 4.88, + "usage": { + "input_tokens": 2773, + "output_tokens": 534 + } + }, + { + "trial": 0, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.85 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.55 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.05 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.02 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.41, + "usage": { + "input_tokens": 3344, + "output_tokens": 759 + } + }, + { + "trial": 0, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.85 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.1 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.1 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.05 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.73, + "usage": { + "input_tokens": 3259, + "output_tokens": 615 + } + }, + { + "trial": 0, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2eeb222", + "confidence_score": 0.95 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.3 + }, + { + "commit_hash": "2e11931", + "confidence_score": 0.15 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.03 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 8.55, + "usage": { + "input_tokens": 3060, + "output_tokens": 644 + } + }, + { + "trial": 0, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.97 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.1 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.03 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.42, + "usage": { + "input_tokens": 2971, + "output_tokens": 667 + } + }, + { + "trial": 0, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.85 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.55 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.5 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.05 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.56, + "usage": { + "input_tokens": 3260, + "output_tokens": 790 + } + }, + { + "trial": 0, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.9 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.15 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.05 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.02 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.14, + "usage": { + "input_tokens": 3174, + "output_tokens": 656 + } + }, + { + "trial": 0, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.85 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.8 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.1 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.02 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.86, + "usage": { + "input_tokens": 2930, + "output_tokens": 641 + } + }, + { + "trial": 0, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.97 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.05 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.01 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.06, + "usage": { + "input_tokens": 3442, + "output_tokens": 555 + } + }, + { + "trial": 0, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.85 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.8 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.05 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.02 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.36, + "usage": { + "input_tokens": 3376, + "output_tokens": 691 + } + }, + { + "trial": 0, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "0751831", + "confidence_score": 0.85 + }, + { + "commit_hash": "fd92e84", + "confidence_score": 0.2 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.15 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.02 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.86, + "usage": { + "input_tokens": 2647, + "output_tokens": 724 + } + }, + { + "trial": 0, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.85 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.05 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.03 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.02 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.53, + "usage": { + "input_tokens": 2831, + "output_tokens": 718 + } + }, + { + "trial": 0, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "266c09c", + "confidence_score": 0.9 + }, + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.55 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.1 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.02 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.42, + "usage": { + "input_tokens": 3051, + "output_tokens": 615 + } + }, + { + "trial": 0, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "f8d8258", + "confidence_score": 0.92 + }, + { + "commit_hash": "0f497ec", + "confidence_score": 0.2 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.05 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.02 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.64, + "usage": { + "input_tokens": 2774, + "output_tokens": 665 + } + }, + { + "trial": 0, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.96 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.25 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.05 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.02 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.91, + "usage": { + "input_tokens": 3095, + "output_tokens": 614 + } + }, + { + "trial": 1, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.97 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.05 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.03 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.02 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.2, + "usage": { + "input_tokens": 3108, + "output_tokens": 691 + } + }, + { + "trial": 1, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.97 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.2 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.05 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.02 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.27, + "usage": { + "input_tokens": 2900, + "output_tokens": 580 + } + }, + { + "trial": 1, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.9 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.35 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.05 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.02 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.55, + "usage": { + "input_tokens": 3034, + "output_tokens": 729 + } + }, + { + "trial": 1, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.6 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.35 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.02 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.52, + "usage": { + "input_tokens": 2555, + "output_tokens": 559 + } + }, + { + "trial": 1, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.85 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.55 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.3 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.1 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.73, + "usage": { + "input_tokens": 3255, + "output_tokens": 707 + } + }, + { + "trial": 1, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "c3629ef", + "confidence_score": 0.85 + }, + { + "commit_hash": "3af442c", + "confidence_score": 0.2 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.1 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.05 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.75, + "usage": { + "input_tokens": 3170, + "output_tokens": 732 + } + }, + { + "trial": 1, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.55 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.05 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.02 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.84, + "usage": { + "input_tokens": 3161, + "output_tokens": 749 + } + }, + { + "trial": 1, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "5cedb21", + "confidence_score": 0.85 + }, + { + "commit_hash": "6eea0fc", + "confidence_score": 0.35 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.2 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.05 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 6.72, + "usage": { + "input_tokens": 3504, + "output_tokens": 656 + } + }, + { + "trial": 1, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.97 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.1 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.01 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.01 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.81, + "usage": { + "input_tokens": 2902, + "output_tokens": 635 + } + }, + { + "trial": 1, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "741f699", + "confidence_score": 0.9 + }, + { + "commit_hash": "b35322f", + "confidence_score": 0.85 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.02 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.01 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 5.02, + "usage": { + "input_tokens": 2773, + "output_tokens": 549 + } + }, + { + "trial": 1, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.95 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.35 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.05 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.02 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.5, + "usage": { + "input_tokens": 3344, + "output_tokens": 598 + } + }, + { + "trial": 1, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.85 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.15 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.1 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.05 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.28, + "usage": { + "input_tokens": 3259, + "output_tokens": 624 + } + }, + { + "trial": 1, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2eeb222", + "confidence_score": 0.97 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.25 + }, + { + "commit_hash": "2e11931", + "confidence_score": 0.2 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.03 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.52, + "usage": { + "input_tokens": 3060, + "output_tokens": 703 + } + }, + { + "trial": 1, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.98 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.05 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.02 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.64, + "usage": { + "input_tokens": 2971, + "output_tokens": 614 + } + }, + { + "trial": 1, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.85 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.3 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.25 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.03 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.17, + "usage": { + "input_tokens": 3260, + "output_tokens": 765 + } + }, + { + "trial": 1, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.85 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.3 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.05 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.02 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.67, + "usage": { + "input_tokens": 3174, + "output_tokens": 684 + } + }, + { + "trial": 1, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.85 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.8 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.05 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.02 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.73, + "usage": { + "input_tokens": 2930, + "output_tokens": 613 + } + }, + { + "trial": 1, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.97 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.05 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.03 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.25, + "usage": { + "input_tokens": 3442, + "output_tokens": 599 + } + }, + { + "trial": 1, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.8 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.75 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.02 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.0 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.72, + "usage": { + "input_tokens": 3376, + "output_tokens": 590 + } + }, + { + "trial": 1, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "0751831", + "confidence_score": 0.85 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.45 + }, + { + "commit_hash": "fd92e84", + "confidence_score": 0.3 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.05 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.59, + "usage": { + "input_tokens": 2647, + "output_tokens": 686 + } + }, + { + "trial": 1, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.9 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.05 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.02 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.02 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.06, + "usage": { + "input_tokens": 2831, + "output_tokens": 733 + } + }, + { + "trial": 1, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "266c09c", + "confidence_score": 0.85 + }, + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.6 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.1 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.02 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.44, + "usage": { + "input_tokens": 3051, + "output_tokens": 714 + } + }, + { + "trial": 1, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "f8d8258", + "confidence_score": 0.85 + }, + { + "commit_hash": "0f497ec", + "confidence_score": 0.25 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.1 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.03 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.39, + "usage": { + "input_tokens": 2774, + "output_tokens": 729 + } + }, + { + "trial": 1, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.95 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.2 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.05 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.02 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.69, + "usage": { + "input_tokens": 3095, + "output_tokens": 594 + } + }, + { + "trial": 2, + "case_id": "auth_issuer_mesh_hosts", + "category": "auth_rejection", + "culprit_hash": "5e68f8a", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.317 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.245 + }, + { + "id": "null_field_handling", + "similarity_score": 0.236 + } + ], + "ranking": [ + { + "commit_hash": "5e68f8a", + "confidence_score": 0.97 + }, + { + "commit_hash": "45a24d5", + "confidence_score": 0.05 + }, + { + "commit_hash": "df0ec4c", + "confidence_score": 0.03 + }, + { + "commit_hash": "298be8d", + "confidence_score": 0.02 + }, + { + "commit_hash": "d20025a", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.62, + "usage": { + "input_tokens": 3108, + "output_tokens": 677 + } + }, + { + "trial": 2, + "case_id": "cache_contains_recursion", + "category": "infinite_recursion", + "culprit_hash": "4584056", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "memory_leaks", + "similarity_score": 0.333 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.31 + }, + { + "id": "db_failures", + "similarity_score": 0.233 + } + ], + "ranking": [ + { + "commit_hash": "4584056", + "confidence_score": 0.97 + }, + { + "commit_hash": "6484609", + "confidence_score": 0.2 + }, + { + "commit_hash": "7e63ea3", + "confidence_score": 0.03 + }, + { + "commit_hash": "6cc141a", + "confidence_score": 0.02 + }, + { + "commit_hash": "a98de00", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.5, + "usage": { + "input_tokens": 2900, + "output_tokens": 538 + } + }, + { + "trial": 2, + "case_id": "cart_price_field_v2", + "category": "null_or_missing_field", + "culprit_hash": "d0ab213", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.26 + }, + { + "id": "pagination_errors", + "similarity_score": 0.207 + }, + { + "id": "memory_leaks", + "similarity_score": 0.179 + } + ], + "ranking": [ + { + "commit_hash": "d0ab213", + "confidence_score": 0.95 + }, + { + "commit_hash": "73e1933", + "confidence_score": 0.4 + }, + { + "commit_hash": "4fbde4c", + "confidence_score": 0.05 + }, + { + "commit_hash": "2bbbf3f", + "confidence_score": 0.02 + }, + { + "commit_hash": "f618f59", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.62, + "usage": { + "input_tokens": 3034, + "output_tokens": 729 + } + }, + { + "trial": 2, + "case_id": "cart_sku_lowercased", + "category": "identifier_format", + "culprit_hash": "c2ec824", + "expected_runbook_id": null, + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.293 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.247 + }, + { + "id": "null_field_handling", + "similarity_score": 0.229 + } + ], + "ranking": [ + { + "commit_hash": "c2ec824", + "confidence_score": 0.6 + }, + { + "commit_hash": "64dde29", + "confidence_score": 0.55 + }, + { + "commit_hash": "d0cb0b3", + "confidence_score": 0.05 + }, + { + "commit_hash": "91f8d36", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.06, + "usage": { + "input_tokens": 2555, + "output_tokens": 621 + } + }, + { + "trial": 2, + "case_id": "checkout_deadline_extra_call", + "category": "latency_timeout", + "culprit_hash": "efebc07", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.487 + }, + { + "id": "db_failures", + "similarity_score": 0.369 + }, + { + "id": "rate_limiting", + "similarity_score": 0.353 + } + ], + "ranking": [ + { + "commit_hash": "efebc07", + "confidence_score": 0.85 + }, + { + "commit_hash": "a035477", + "confidence_score": 0.55 + }, + { + "commit_hash": "2b765f4", + "confidence_score": 0.25 + }, + { + "commit_hash": "be85cc9", + "confidence_score": 0.05 + }, + { + "commit_hash": "91b8b07", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.67, + "usage": { + "input_tokens": 3255, + "output_tokens": 689 + } + }, + { + "trial": 2, + "case_id": "config_reformat_hidden_string", + "category": "config_type_error", + "culprit_hash": "c3629ef", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.246 + }, + { + "id": "pagination_errors", + "similarity_score": 0.19 + }, + { + "id": "memory_leaks", + "similarity_score": 0.184 + } + ], + "ranking": [ + { + "commit_hash": "c3629ef", + "confidence_score": 0.85 + }, + { + "commit_hash": "3af442c", + "confidence_score": 0.3 + }, + { + "commit_hash": "976c988", + "confidence_score": 0.1 + }, + { + "commit_hash": "a6a3c41", + "confidence_score": 0.1 + }, + { + "commit_hash": "89436d3", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.41, + "usage": { + "input_tokens": 3170, + "output_tokens": 700 + } + }, + { + "trial": 2, + "case_id": "db_fetch_one_leak", + "category": "db_pool_exhaustion", + "culprit_hash": "af9b861", + "expected_runbook_id": "db_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.631 + }, + { + "id": "memory_leaks", + "similarity_score": 0.35 + }, + { + "id": "request_timeouts", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "af9b861", + "confidence_score": 0.95 + }, + { + "commit_hash": "f46cd8c", + "confidence_score": 0.5 + }, + { + "commit_hash": "1618815", + "confidence_score": 0.05 + }, + { + "commit_hash": "6d50d20", + "confidence_score": 0.02 + }, + { + "commit_hash": "1bf1024", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.77, + "usage": { + "input_tokens": 3161, + "output_tokens": 649 + } + }, + { + "trial": 2, + "case_id": "export_tempfile_never_closed", + "category": "file_handle_leak", + "culprit_hash": "6eea0fc", + "expected_runbook_id": "file_handle_exhaustion", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.625 + }, + { + "id": "memory_leaks", + "similarity_score": 0.432 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.291 + } + ], + "ranking": [ + { + "commit_hash": "5cedb21", + "confidence_score": 0.55 + }, + { + "commit_hash": "6eea0fc", + "confidence_score": 0.5 + }, + { + "commit_hash": "b245524", + "confidence_score": 0.3 + }, + { + "commit_hash": "0be651d", + "confidence_score": 0.05 + }, + { + "commit_hash": "7453733", + "confidence_score": 0.0 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 7.12, + "usage": { + "input_tokens": 3504, + "output_tokens": 695 + } + }, + { + "trial": 2, + "case_id": "guest_email_normalise", + "category": "null_or_missing_field", + "culprit_hash": "7dd25fe", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.241 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.208 + }, + { + "id": "pagination_errors", + "similarity_score": 0.161 + } + ], + "ranking": [ + { + "commit_hash": "7dd25fe", + "confidence_score": 0.97 + }, + { + "commit_hash": "9519064", + "confidence_score": 0.1 + }, + { + "commit_hash": "1c0135b", + "confidence_score": 0.02 + }, + { + "commit_hash": "30a7434", + "confidence_score": 0.01 + }, + { + "commit_hash": "cde392a", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.62, + "usage": { + "input_tokens": 2902, + "output_tokens": 647 + } + }, + { + "trial": 2, + "case_id": "httpx_028_drops_proxies", + "category": "dependency_version", + "culprit_hash": "b35322f", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 4, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.294 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.285 + }, + { + "id": "memory_leaks", + "similarity_score": 0.263 + } + ], + "ranking": [ + { + "commit_hash": "741f699", + "confidence_score": 0.95 + }, + { + "commit_hash": "b35322f", + "confidence_score": 0.85 + }, + { + "commit_hash": "b45189f", + "confidence_score": 0.02 + }, + { + "commit_hash": "e3601f6", + "confidence_score": 0.01 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 5.19, + "usage": { + "input_tokens": 2773, + "output_tokens": 555 + } + }, + { + "trial": 2, + "case_id": "inventory_cache_key_per_request", + "category": "rate_limit_429", + "culprit_hash": "526f8dd", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.382 + }, + { + "id": "auth_failures", + "similarity_score": 0.381 + }, + { + "id": "null_field_handling", + "similarity_score": 0.309 + } + ], + "ranking": [ + { + "commit_hash": "526f8dd", + "confidence_score": 0.95 + }, + { + "commit_hash": "bf4cc5f", + "confidence_score": 0.45 + }, + { + "commit_hash": "93320f2", + "confidence_score": 0.05 + }, + { + "commit_hash": "2e1b06b", + "confidence_score": 0.02 + }, + { + "commit_hash": "d9caa65", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.16, + "usage": { + "input_tokens": 3344, + "output_tokens": 747 + } + }, + { + "trial": 2, + "case_id": "inventory_v3_payload", + "category": "null_or_missing_field", + "culprit_hash": "1c29d94", + "expected_runbook_id": "null_field_handling", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.33 + }, + { + "id": "memory_leaks", + "similarity_score": 0.281 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.28 + } + ], + "ranking": [ + { + "commit_hash": "1c29d94", + "confidence_score": 0.85 + }, + { + "commit_hash": "c6f0797", + "confidence_score": 0.05 + }, + { + "commit_hash": "16d2191", + "confidence_score": 0.05 + }, + { + "commit_hash": "5823d1f", + "confidence_score": 0.02 + }, + { + "commit_hash": "fb9b710", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.52, + "usage": { + "input_tokens": 3259, + "output_tokens": 575 + } + }, + { + "trial": 2, + "case_id": "jwt_leeway_milliseconds", + "category": "auth_rejection", + "culprit_hash": "2eeb222", + "expected_runbook_id": "auth_failures", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.488 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.198 + }, + { + "id": "request_timeouts", + "similarity_score": 0.159 + } + ], + "ranking": [ + { + "commit_hash": "2eeb222", + "confidence_score": 0.95 + }, + { + "commit_hash": "2e11931", + "confidence_score": 0.25 + }, + { + "commit_hash": "e8548c5", + "confidence_score": 0.2 + }, + { + "commit_hash": "a028c62", + "confidence_score": 0.03 + }, + { + "commit_hash": "716e22d", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.8, + "usage": { + "input_tokens": 3060, + "output_tokens": 709 + } + }, + { + "trial": 2, + "case_id": "money_returns_decimal", + "category": "json_serialization", + "culprit_hash": "b14897b", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.298 + }, + { + "id": "null_field_handling", + "similarity_score": 0.151 + }, + { + "id": "auth_failures", + "similarity_score": 0.13 + } + ], + "ranking": [ + { + "commit_hash": "b14897b", + "confidence_score": 0.97 + }, + { + "commit_hash": "dedc41a", + "confidence_score": 0.05 + }, + { + "commit_hash": "626c22d", + "confidence_score": 0.02 + }, + { + "commit_hash": "cd4a49f", + "confidence_score": 0.01 + }, + { + "commit_hash": "92cb3da", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.31, + "usage": { + "input_tokens": 2971, + "output_tokens": 721 + } + }, + { + "trial": 2, + "case_id": "order_read_sync_audit", + "category": "latency_timeout", + "culprit_hash": "8aed86c", + "expected_runbook_id": "request_timeouts", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.539 + }, + { + "id": "rate_limiting", + "similarity_score": 0.369 + }, + { + "id": "auth_failures", + "similarity_score": 0.35 + } + ], + "ranking": [ + { + "commit_hash": "8aed86c", + "confidence_score": 0.85 + }, + { + "commit_hash": "d3d6f87", + "confidence_score": 0.4 + }, + { + "commit_hash": "94578cc", + "confidence_score": 0.25 + }, + { + "commit_hash": "5abe2e4", + "confidence_score": 0.05 + }, + { + "commit_hash": "b7ea08d", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 7.0, + "usage": { + "input_tokens": 3260, + "output_tokens": 794 + } + }, + { + "trial": 2, + "case_id": "order_sql_missing_param", + "category": "sql_param_mismatch", + "culprit_hash": "9ef79b2", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.322 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.23 + }, + { + "id": "memory_leaks", + "similarity_score": 0.196 + } + ], + "ranking": [ + { + "commit_hash": "9ef79b2", + "confidence_score": 0.85 + }, + { + "commit_hash": "661f871", + "confidence_score": 0.2 + }, + { + "commit_hash": "93ebe2f", + "confidence_score": 0.05 + }, + { + "commit_hash": "66d5f2f", + "confidence_score": 0.02 + }, + { + "commit_hash": "411ddfa", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.53, + "usage": { + "input_tokens": 3174, + "output_tokens": 681 + } + }, + { + "trial": 2, + "case_id": "page_size_zero_division", + "category": "pagination_boundary", + "culprit_hash": "6265f24", + "expected_runbook_id": "pagination_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "config_parse_errors", + "similarity_score": 0.146 + }, + { + "id": "memory_leaks", + "similarity_score": 0.129 + }, + { + "id": "pagination_errors", + "similarity_score": 0.127 + } + ], + "ranking": [ + { + "commit_hash": "6265f24", + "confidence_score": 0.9 + }, + { + "commit_hash": "ed3861b", + "confidence_score": 0.85 + }, + { + "commit_hash": "0d241ff", + "confidence_score": 0.1 + }, + { + "commit_hash": "ef48acf", + "confidence_score": 0.02 + }, + { + "commit_hash": "debe268", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.61, + "usage": { + "input_tokens": 2930, + "output_tokens": 649 + } + }, + { + "trial": 2, + "case_id": "payments_body_without_content_type", + "category": "http_content_type", + "culprit_hash": "33a9e2a", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "auth_failures", + "similarity_score": 0.274 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.164 + }, + { + "id": "null_field_handling", + "similarity_score": 0.143 + } + ], + "ranking": [ + { + "commit_hash": "33a9e2a", + "confidence_score": 0.95 + }, + { + "commit_hash": "e60edf2", + "confidence_score": 0.05 + }, + { + "commit_hash": "91f3040", + "confidence_score": 0.02 + }, + { + "commit_hash": "5430f37", + "confidence_score": 0.02 + }, + { + "commit_hash": "64ee5a8", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.28, + "usage": { + "input_tokens": 3442, + "output_tokens": 559 + } + }, + { + "trial": 2, + "case_id": "payments_retry_loop", + "category": "rate_limit_429", + "culprit_hash": "f07e927", + "expected_runbook_id": "rate_limiting", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "rate_limiting", + "similarity_score": 0.357 + }, + { + "id": "auth_failures", + "similarity_score": 0.352 + }, + { + "id": "request_timeouts", + "similarity_score": 0.323 + } + ], + "ranking": [ + { + "commit_hash": "f07e927", + "confidence_score": 0.6 + }, + { + "commit_hash": "07a85f8", + "confidence_score": 0.35 + }, + { + "commit_hash": "c1fc55f", + "confidence_score": 0.03 + }, + { + "commit_hash": "429c4dd", + "confidence_score": 0.0 + }, + { + "commit_hash": "51bc043", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.06, + "usage": { + "input_tokens": 3376, + "output_tokens": 601 + } + }, + { + "trial": 2, + "case_id": "redis_legacy_lrem_args", + "category": "dependency_version", + "culprit_hash": "0751831", + "expected_runbook_id": "dependency_regressions", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "null_field_handling", + "similarity_score": 0.324 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.317 + }, + { + "id": "auth_failures", + "similarity_score": 0.269 + } + ], + "ranking": [ + { + "commit_hash": "0751831", + "confidence_score": 0.85 + }, + { + "commit_hash": "3059910", + "confidence_score": 0.4 + }, + { + "commit_hash": "fd92e84", + "confidence_score": 0.15 + }, + { + "commit_hash": "7b9dd2c", + "confidence_score": 0.02 + }, + { + "commit_hash": "957a42e", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.5, + "usage": { + "input_tokens": 2647, + "output_tokens": 660 + } + }, + { + "trial": 2, + "case_id": "settings_env_quotes", + "category": "config_type_error", + "culprit_hash": "15aba07", + "expected_runbook_id": "config_parse_errors", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "db_failures", + "similarity_score": 0.36 + }, + { + "id": "config_parse_errors", + "similarity_score": 0.278 + }, + { + "id": "memory_leaks", + "similarity_score": 0.241 + } + ], + "ranking": [ + { + "commit_hash": "15aba07", + "confidence_score": 0.9 + }, + { + "commit_hash": "cae9a6c", + "confidence_score": 0.05 + }, + { + "commit_hash": "423c51f", + "confidence_score": 0.05 + }, + { + "commit_hash": "3e36aa4", + "confidence_score": 0.02 + }, + { + "commit_hash": "55cfa0d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.88, + "usage": { + "input_tokens": 2831, + "output_tokens": 753 + } + }, + { + "trial": 2, + "case_id": "since_filter_naive_cutoff", + "category": "timezone_mismatch", + "culprit_hash": "266c09c", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "pagination_errors", + "similarity_score": 0.279 + }, + { + "id": "dependency_regressions", + "similarity_score": 0.224 + }, + { + "id": "memory_leaks", + "similarity_score": 0.215 + } + ], + "ranking": [ + { + "commit_hash": "f3ad8b7", + "confidence_score": 0.85 + }, + { + "commit_hash": "266c09c", + "confidence_score": 0.6 + }, + { + "commit_hash": "7f71cc9", + "confidence_score": 0.1 + }, + { + "commit_hash": "ef6b4a1", + "confidence_score": 0.02 + }, + { + "commit_hash": "5948e6d", + "confidence_score": 0.02 + } + ], + "culprit_rank": 2, + "error": null, + "rank_latency_s": 7.41, + "usage": { + "input_tokens": 3051, + "output_tokens": 730 + } + }, + { + "trial": 2, + "case_id": "worker_queue_renamed", + "category": "queue_stall", + "culprit_hash": "f8d8258", + "expected_runbook_id": null, + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "request_timeouts", + "similarity_score": 0.411 + }, + { + "id": "rate_limiting", + "similarity_score": 0.409 + }, + { + "id": "file_handle_exhaustion", + "similarity_score": 0.383 + } + ], + "ranking": [ + { + "commit_hash": "f8d8258", + "confidence_score": 0.92 + }, + { + "commit_hash": "0f497ec", + "confidence_score": 0.15 + }, + { + "commit_hash": "110eec6", + "confidence_score": 0.05 + }, + { + "commit_hash": "aacbcd7", + "confidence_score": 0.03 + }, + { + "commit_hash": "3434d6b", + "confidence_score": 0.01 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 6.36, + "usage": { + "input_tokens": 2774, + "output_tokens": 733 + } + }, + { + "trial": 2, + "case_id": "worker_retains_job_payloads", + "category": "memory_growth", + "culprit_hash": "3037583", + "expected_runbook_id": "memory_leaks", + "commits_in_window": 5, + "truncated_diffs": 0, + "retrieved_runbooks": [ + { + "id": "file_handle_exhaustion", + "similarity_score": 0.48 + }, + { + "id": "memory_leaks", + "similarity_score": 0.478 + }, + { + "id": "rate_limiting", + "similarity_score": 0.424 + } + ], + "ranking": [ + { + "commit_hash": "3037583", + "confidence_score": 0.95 + }, + { + "commit_hash": "947ac52", + "confidence_score": 0.25 + }, + { + "commit_hash": "088e964", + "confidence_score": 0.15 + }, + { + "commit_hash": "15f34e0", + "confidence_score": 0.03 + }, + { + "commit_hash": "63c2633", + "confidence_score": 0.0 + } + ], + "culprit_rank": 1, + "error": null, + "rank_latency_s": 5.33, + "usage": { + "input_tokens": 3095, + "output_tokens": 593 + } + } + ], + "finished_at": "2026-10-07T21:33:51.939699+00:00" +} \ No newline at end of file diff --git a/eval/run_eval.py b/eval/run_eval.py index aa06c73..74161af 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -47,14 +47,24 @@ def run_trial(case: dict, condition: str) -> dict: "culprit_rank": None, "error": None, } + usage = {} + real_chat = llm_analyzer.chat + + def metered_chat(*args, **kwargs): # records token usage without touching production code + resp = real_chat(*args, **kwargs) + if getattr(resp, "usage", None): + usage.update(input_tokens=resp.usage.input_tokens, output_tokens=resp.usage.output_tokens) + return resp + t0 = time.monotonic() try: # the harness never builds a prompt itself; it calls the production function - ranked = ( - llm_analyzer.rank_suspect_commits(diffs, alert, runbooks=runbooks) - if rag - else llm_analyzer.rank_suspect_commits(diffs, alert) - ) + with patch.object(llm_analyzer, "chat", metered_chat): + ranked = ( + llm_analyzer.rank_suspect_commits(diffs, alert, runbooks=runbooks) + if rag + else llm_analyzer.rank_suspect_commits(diffs, alert) + ) record["ranking"] = [ {"commit_hash": r["commit_hash"][:7], "confidence_score": r["confidence_score"]} for r in ranked @@ -65,6 +75,7 @@ def run_trial(case: dict, condition: str) -> dict: except llm_analyzer.LLMUnavailable as e: record["error"] = f"LLMUnavailable: {e}" # counted as a miss by report.py, never dropped record["rank_latency_s"] = round(time.monotonic() - t0, 2) + record["usage"] = usage or None return record