From 1565cf637a8201805cf71b81bc8ee7e3fb36f2b3 Mon Sep 17 00:00:00 2001 From: TillQuandel Date: Sun, 12 Jul 2026 01:26:34 +0200 Subject: [PATCH] =?UTF-8?q?feat(dashboard):=20Lauf-Filter=20=E2=80=94=20ei?= =?UTF-8?q?nzelnen=20Pipeline-Run=20isolieren=20(closes=20#211)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Der Versions-Filter poolt alle Runs einer Version; "was hat mein letzter Durchlauf produziert?" war nicht beantwortbar. Neu: fuenftes Filter-Dropdown "Lauf" (run_id), oberster Eintrag = letzter Durchlauf. Server: build_data(run=...) filtert quality_rows, token_runs, all_log_runs (Log-Altbestaende ohne run_id fallen ehrlich raus) und die Versions-Dropdown-Basis (gleiche Konvention wie PDF-/Sprachfilter); agent_stats-allowed_run_ids + calibration-Bedingung um run erweitert; _exclude_foss beruecksichtigt run (expliziter Filter -> foss sichtbar). _run_options: letzte 15 Runs aus pipeline_runs, Zeit-Label aus der run_id (lokale Startzeit, Trace-Namensschema) statt DB-timestamp (UTC vom Lauf-Ende), Fallback fuer fremde run_id-Formate. Client: Dropdown nach _rebuildFilterSelect-Muster (Auswahl ueberlebt Options-Rebuild, verschwundene Option -> ehrlicher Reset), Badge, Clear, resetAllFilters, loadAndRender-Param, Optionen escaped. TDD: 6 Tests (Optionen-Sortierung/Label/Cap, Kaskaden-Isolation, AND-Komposition mit PDF, Ohne-run-unveraendert). Suite 5699 gruen, ruff sauber. Playwright: Dropdown zeigt die 3 Messlaeufe vom 12.07. (juengster oben), Auswahl isoliert auf 1 Run/1 Note ueber alle Panels, Clear stellt 52 Notes wieder her; keine JS-Fehler. --- generative/eval_dashboard_server.py | 69 ++++++++- generative/tests/test_dashboard_run_filter.py | 142 ++++++++++++++++++ internal/dashboard/eval_dashboard.html | 29 +++- 3 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 generative/tests/test_dashboard_run_filter.py diff --git a/generative/eval_dashboard_server.py b/generative/eval_dashboard_server.py index f87b395..de0e01b 100644 --- a/generative/eval_dashboard_server.py +++ b/generative/eval_dashboard_server.py @@ -12,6 +12,7 @@ import argparse import json +import re import webbrowser from datetime import datetime from http.server import BaseHTTPRequestHandler, HTTPServer @@ -112,6 +113,41 @@ def _current_db_version(db_runs: list[dict], current: str | None = None) -> str return max(gen_runs, key=lambda r: r.get("timestamp") or "")["pipeline_version"] +def _run_options(db_runs: list[dict], limit: int = 15) -> list[dict]: + """Dropdown-Optionen für den Lauf-Filter (#211): jüngste Runs zuerst. + + Label: „TT.MM. HH:MM · · " — der oberste Eintrag ist der + letzte Durchlauf. Runs ohne run_id (Alt-Daten) sind nicht filterbar und + fallen raus. + """ + runs = sorted( + (r for r in db_runs if r.get("run_id")), + key=lambda r: r.get("timestamp") or "", + reverse=True, + )[:limit] + opts = [] + for r in runs: + # Zeit bevorzugt aus der run_id (lokale STARTzeit, Format + # YYYYMMDD-HHMMSS — konsistent mit Trace-Dateinamen); der + # DB-timestamp ist UTC vom Lauf-Ende und würde verwirren. + rid = str(r["run_id"]) + if re.fullmatch(r"\d{8}-\d{6}", rid): + ts_label = f"{rid[6:8]}.{rid[4:6]}. {rid[9:11]}:{rid[11:13]}" + else: + ts = str(r.get("timestamp") or "") + ts_label = f"{ts[8:10]}.{ts[5:7]}. {ts[11:16]}" if len(ts) >= 16 else ts + pdf = str(r.get("pdf_label") or r.get("pdf_source") or "?").removesuffix(".pdf") + if len(pdf) > 30: + pdf = pdf[:28] + "…" + opts.append( + { + "id": r["run_id"], + "label": f"{ts_label} · {pdf} · {r.get('pipeline_version') or '?'}", + } + ) + return opts + + def _cache_pct_prompt_based(input_tokens: int, cache_r: int, cache_c: int) -> float: """Cache-Hit-Quote prompt-basiert: cache_r / (input + cache_r + cache_c). @@ -344,11 +380,13 @@ def build_data( language: str | None = None, model: str | None = None, pdf: str | None = None, + run: str | None = None, ) -> dict: """Baut alle Chart-Daten aus SQLite (note_evals) + Log-Files (runs). eval_version: wenn None → neueste verfuegbare Version pipeline_version: wenn None → alle Versionen + run: run_id — nur diesen Pipeline-Lauf zeigen (#211) """ from generative import db as _db @@ -384,6 +422,8 @@ def build_data( # quality_rows.pdf driftet über Namensräume (Volltitel/Kebab-Key, #202) # → Slug-Matching statt rohem startswith quality_rows = [r for r in quality_rows if D._pdf_matches(pdf, r.get("pdf"))] + if run: + quality_rows = [r for r in quality_rows if r.get("run_id") == run] all_log_runs = D._read_all_log_runs() token_runs = D._read_token_runs() @@ -484,6 +524,8 @@ def build_data( # pdf_label driftet über Pipeline-Versionen (Volltitel → "Bates" → # "bates-2017", #202): rohes startswith matchte nur die Alt-Versionen. token_runs = [tr for tr in token_runs if D._pdf_matches(pdf, tr.get("pdf_label"))] + if run: + token_runs = [tr for tr in token_runs if tr.get("run_id") == run] # run_ids der gefilterten token_runs → quality_rows auf selbe Runs einschränken # Wenn Filter aktiv aber keine run_ids matchen → quality_rows leeren (nicht überspringen) @@ -530,7 +572,7 @@ def build_data( # foss nur im ungefilterten Default-View ausblenden — sobald ein Modell-/ # Versions-Filter aktiv ist, bleibt foss einsehbar (#36, gleiche Bedingung # wie der foss-Ausschluss aus quality_rows/all_log_runs unten). - _exclude_foss = not (model or pipeline_version) + _exclude_foss = not (model or pipeline_version or run) # PDF-Dropdown: aus den Eval-Daten der aktiven eval_version statt aus # all_log_runs — so erscheinen nur PDFs mit echten Daten im View (sonst @@ -552,6 +594,8 @@ def build_data( _pver_rows = [r for r in _pver_rows if r.get("language") == language] if pdf: _pver_rows = [r for r in _pver_rows if D._pdf_matches(pdf, r.get("pdf"))] + if run: + _pver_rows = [r for r in _pver_rows if r.get("run_id") == run] _pver_counts: dict[str, int] = {} for r in _pver_rows: pv = r.get("version") or r.get("pipeline_version") @@ -578,6 +622,11 @@ def build_data( # driften unabhängig voneinander (#202). all_log_runs = [r for r in all_log_runs if D._pdf_matches(pdf, r.get("label"), r.get("key"))] log_data = D._build_log_data(all_log_runs) + if run: + # Log-Runs ohne run_id (reine .log-Altbestände) sind einem Lauf nicht + # zuordenbar und fallen beim Lauf-Filter ehrlich raus (#211). + all_log_runs = [r for r in all_log_runs if r.get("run_id") == run] + log_data = D._build_log_data(all_log_runs) if language: # pdf_label → sprache aus token_runs (nach language-Filter bereits korrekt gefiltert). # KEIN `if _lang_pdfs:`-Guard: eine leere Match-Menge heißt „keine Runs in @@ -732,6 +781,14 @@ def _pooled_accept(ver: str) -> float | None: m: D.version_delta(kpi_trend, m) for m in ("hall", "cov", "n", "accept", "dur", "tokens", "cost") } + # ── Lauf-Dropdown-Optionen (#211): immer ungefiltert, jüngste zuerst ── + try: + from generative import db as _db_runs + + _all_runs_opts = _run_options(_db_runs.query_pipeline_runs()) + except Exception: + _all_runs_opts = [] + # ── Warnungen (#196 P1): Waisen-Versionen + JSONL-Fallback ────── try: from generative import config as _cfg @@ -768,18 +825,19 @@ def _pooled_accept(ver: str) -> float | None: "all_models": _all_models_opts, "all_pvers": _all_pvers_opts, "all_pdfs": _all_pdfs_opts, + "all_runs": _all_runs_opts, "agent_stats": _read_agent_stats( # pipeline_version gehört in die Bedingung (Codex-Fund, #191/PR-192-Review): # token_runs sind oben auf die gewählte Version gefiltert — ohne sie hier # zeigte das Agent-Panel bei explizitem Versions-Filter weiter die # config-aktuelle Version. allowed_run_ids={tr.get("run_id") for tr in token_runs if tr.get("run_id")} - if (pdf or model or language or pipeline_version) + if (pdf or model or language or pipeline_version or run) else None ), "calibration": _read_calibration_data( allowed_note_paths={r.get("note_path") or r.get("note") for r in quality_rows} - if (pdf or language or model or pipeline_version) + if (pdf or language or model or pipeline_version or run) else None, eval_version=eval_version or "4.1", ), @@ -1129,8 +1187,11 @@ def do_GET(self): lang = params.get("language", [None])[0] mdl = params.get("model", [None])[0] pdf_f = params.get("pdf", [None])[0] + run_f = params.get("run", [None])[0] try: - data = build_data(eval_version=eval_ver, pipeline_version=pver, language=lang, model=mdl, pdf=pdf_f) + data = build_data( + eval_version=eval_ver, pipeline_version=pver, language=lang, model=mdl, pdf=pdf_f, run=run_f + ) body = json.dumps(data, ensure_ascii=False).encode("utf-8") self.send_response(200) self.send_header("Content-Type", "application/json; charset=utf-8") diff --git a/generative/tests/test_dashboard_run_filter.py b/generative/tests/test_dashboard_run_filter.py new file mode 100644 index 0000000..f82755b --- /dev/null +++ b/generative/tests/test_dashboard_run_filter.py @@ -0,0 +1,142 @@ +"""Tests für #211: Lauf-Filter — nur einen bestimmten Pipeline-Run anzeigen. + +Der Versions-Filter poolt alle Runs einer Version; ein einzelner Run +(typisch: der letzte) war nicht isolierbar. Neu: `build_data(run=)` +filtert die komplette Kaskade auf diesen Run; `all_runs` liefert die +Dropdown-Optionen (jüngste zuerst, gedeckelt). +""" + +from __future__ import annotations + +from generative.eval_dashboard_server import _run_options + + +def _pipeline_run(run_id, ver, ts, pdf_label="Testquelle - 2020 - Titel", cost=0.0): + return { + "run_id": run_id, + "timestamp": ts, + "pipeline_version": ver, + "pdf_source": f"{pdf_label}.pdf", + "pdf_key": "testquelle-2020", + "pdf_label": pdf_label, + "model": "test-model-x", + "cost_usd": cost, + "n_generated": 4, + "n_vault": 3, + "n_inbox": 1, + "n_merge": 0, + "n_words": 5000, + "n_dropped": 0, + "duration_s": 60.0, + } + + +def _eval_row(run_id, ver, note="n1", pdf="Testquelle - 2020 - Titel.pdf"): + return { + "run_id": run_id, + "note_path": f"notes/{run_id}-{note}.md", + "acceptance_status": "vault", + "hallucination_rate": 0.05, + "coverage_factual": 0.8, + "pipeline_version": ver, + "version": ver, + "pdf": pdf, + "language": "DE→DE", + "eval_version": "4.1", + "anchors_total": 10, + "anchors_hallucinated": 1, + } + + +# ── _run_options: Dropdown-Basis ──────────────────────────────────────────── + + +def test_run_options_newest_first_with_label(): + runs = [ + _pipeline_run("r-alt", "v0.3.139", "2026-07-01T10:00:00"), + _pipeline_run("r-neu", "v0.3.140", "2026-07-11T23:46:56"), + ] + opts = _run_options(runs) + assert [o["id"] for o in opts] == ["r-neu", "r-alt"] + assert "v0.3.140" in opts[0]["label"] + assert "Testquelle" in opts[0]["label"] + + +def test_run_options_time_from_run_id_not_db_timestamp(): + # run_id trägt die lokale STARTzeit (Trace-Namensschema); der DB-timestamp + # ist UTC vom Lauf-Ende und würde als Anzeige verwirren. + runs = [_pipeline_run("20260712-002220", "v0.3.140", "2026-07-11T22:36:41")] + opts = _run_options(runs) + assert opts[0]["label"].startswith("12.07. 00:22") + + +def test_run_options_capped_and_skips_missing_ids(): + runs = [_pipeline_run(f"r{i:02d}", "v1", f"2026-07-01T10:00:{i:02d}") for i in range(20)] + runs.append({"timestamp": "2026-07-11T00:00:00", "pipeline_version": "v1"}) # ohne run_id + opts = _run_options(runs, limit=15) + assert len(opts) == 15 + assert opts[0]["id"] == "r19" + + +# ── build_data-Integration ────────────────────────────────────────────────── + + +def _patched_build_data(monkeypatch, runs, evals, **kwargs): + from generative import config as _cfg + from generative import db as _gdb + from generative import eval_dashboard as D + from generative import eval_dashboard_server as S + + monkeypatch.setattr(_cfg, "AGENT_VERSION", "v0.3.140") + monkeypatch.setattr(_gdb, "query_pipeline_runs", lambda *a, **k: [dict(r) for r in runs]) + monkeypatch.setattr(_gdb, "query_note_evals", lambda *a, **k: [dict(r) for r in evals]) + monkeypatch.setattr(D, "_read_all_log_runs", lambda: []) + monkeypatch.setattr(D, "_read_token_runs", lambda: []) + return S.build_data(**kwargs) + + +def test_build_data_run_filter_isolates_single_run(monkeypatch): + runs = [ + _pipeline_run("r-alt", "v0.3.140", "2026-07-01T10:00:00", cost=1.0), + _pipeline_run("r-neu", "v0.3.140", "2026-07-11T23:46:56", cost=2.0), + ] + evals = [ + _eval_row("r-alt", "v0.3.140", "alt-a"), + _eval_row("r-alt", "v0.3.140", "alt-b"), + _eval_row("r-neu", "v0.3.140", "neu-a"), + ] + data = _patched_build_data(monkeypatch, runs, evals, run="r-neu") + + # Nur der gewählte Run in der Kaskade: 1 Eval-Zeile, 1 Log-Run, Kosten nur r-neu + assert data["quality_by_version"]["v0.3.140"]["n"] == 1 + assert data["runs_by_version"]["v0.3.140"]["n_runs"] == 1 + cost_by_ver = dict(zip(data["kpi_trend"]["versions"], data["kpi_trend"]["cost"])) + assert cost_by_ver["v0.3.140"] == 2.0 + + # Dropdown-Optionen bleiben ungefiltert (alle Runs anwählbar) + assert [o["id"] for o in data["all_runs"]] == ["r-neu", "r-alt"] + + +def test_build_data_run_filter_composes_with_pdf(monkeypatch): + runs = [ + _pipeline_run("r-a", "v0.3.140", "2026-07-01T10:00:00"), + _pipeline_run("r-b", "v0.3.140", "2026-07-02T10:00:00", pdf_label="Andere - 2021 - Quelle"), + ] + evals = [ + _eval_row("r-a", "v0.3.140"), + _eval_row("r-b", "v0.3.140", pdf="Andere - 2021 - Quelle.pdf"), + ] + # Run r-b (andere Quelle) + PDF-Filter auf die Testquelle → leere Schnittmenge + data = _patched_build_data(monkeypatch, runs, evals, run="r-b", pdf="testquelle - 2020 - titel") + assert data["kpis"]["n_notes"] in (0, None) + + +def test_build_data_without_run_unchanged(monkeypatch): + runs = [ + _pipeline_run("r-alt", "v0.3.140", "2026-07-01T10:00:00"), + _pipeline_run("r-neu", "v0.3.140", "2026-07-11T23:46:56"), + ] + evals = [_eval_row("r-alt", "v0.3.140"), _eval_row("r-neu", "v0.3.140", "neu-a")] + data = _patched_build_data(monkeypatch, runs, evals) + assert data["quality_by_version"]["v0.3.140"]["n"] == 2 + assert data["runs_by_version"]["v0.3.140"]["n_runs"] == 2 diff --git a/internal/dashboard/eval_dashboard.html b/internal/dashboard/eval_dashboard.html index 42116f4..2424464 100644 --- a/internal/dashboard/eval_dashboard.html +++ b/internal/dashboard/eval_dashboard.html @@ -613,6 +613,12 @@ +
+ Lauf + +
PDF