M2.5 Phase 3: Bootstrap confidence intervals for eval nDCG@3

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project are documented here.
 
 ## Unreleased
 
+- Bootstrap confidence intervals: conversation-level percentile bootstrap for
+  nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable
+  runs). CIs appear in JSON reports and the markdown table.
 - Full-ranking eval: the decoder now exposes the current turn's candidates
   ranked by trajectory (final-turn cumulative) score via `decode_ranked` and
   `RetrievalResult.candidate_ranking` / `CandidateScore`. The eval re-ranks the

docs/eval.md

@@ -97,6 +97,21 @@ the follow-up-lift claim than TopiOCQA's topic switches.
 - `--candidate-limit` (default 100) sizes the re-rankable window; the gold
   passage must be within it to be re-ranked (otherwise recall bounds the score).
 
+## Bootstrap confidence intervals
+
+Use `--bootstrap-samples` to resample conversations and compute 95% percentile
+CIs for nDCG@3 on each slice. Disabled by default (`0`) for fast smoke runs;
+use `1000` for publishable numbers. `--bootstrap-seed` (default 42) keeps runs
+reproducible.
+
+```console
+python -m mapmatched.eval --benchmark synthetic \
+    --bootstrap-samples 200 --output eval-report.json
+```
+
+The markdown table adds an `nDCG@3 95% CI` column; JSON reports include
+`ndcg_at_3_ci` as `[lower, upper]` on each slice.
+
 ## Reproducing headline numbers
 
 The README headline table uses **Tier B dev-slice** results with a

src/mapmatched/eval/__main__.py

@@ -59,6 +59,18 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--recall-k", type=int, default=100)
     parser.add_argument("--standalone-tolerance", type=float, default=0.02)
     parser.add_argument("--follow-up-min-delta", type=float, default=0.0)
+    parser.add_argument(
+        "--bootstrap-samples",
+        type=int,
+        default=0,
+        help="Conversation-level bootstrap resamples for nDCG@3 95%% CIs (0 = disabled).",
+    )
+    parser.add_argument(
+        "--bootstrap-seed",
+        type=int,
+        default=42,
+        help="Random seed for bootstrap resampling.",
+    )
     parser.add_argument("--include-resolved-oracle", action="store_true")
     return parser
 
@@ -105,6 +117,8 @@ def main(argv: list[str] | None = None) -> int:
         follow_up_min_delta=args.follow_up_min_delta,
         ranking_mode=args.ranking_mode,
         graph_source=args.graph_source,
+        bootstrap_samples=args.bootstrap_samples,
+        bootstrap_seed=args.bootstrap_seed,
     )
     report = run_ablation_grid(
         conversations=conversations,

src/mapmatched/eval/bootstrap.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import math
+import random
+from collections.abc import Sequence
+
+from .metrics import mean
+from .types import SliceName, TurnMetrics
+
+
+def bootstrap_ndcg_at_3_ci(
+    conversation_turns: Sequence[Sequence[TurnMetrics]],
+    slice_name: SliceName,
+    *,
+    num_samples: int,
+    seed: int | None = None,
+    lower_percentile: float = 2.5,
+    upper_percentile: float = 97.5,
+) -> tuple[float, float] | None:
+    if num_samples <= 0:
+        return None
+    conversation_count = len(conversation_turns)
+    if conversation_count == 0:
+        return None
+
+    rng = random.Random(seed)
+    bootstrap_means: list[float] = []
+    for _ in range(num_samples):
+        resampled = [
+            conversation_turns[rng.randrange(conversation_count)] for _ in range(conversation_count)
+        ]
+        selected_turns = [
+            turn
+            for conversation in resampled
+            for turn in conversation
+            if turn.slice_name == slice_name
+        ]
+        bootstrap_means.append(mean([turn.ndcg_at_3 for turn in selected_turns]))
+
+    sorted_means = sorted(bootstrap_means)
+    return (
+        _percentile(sorted_means, lower_percentile),
+        _percentile(sorted_means, upper_percentile),
+    )
+
+
+def bootstrap_slice_cis(
+    conversation_turns: Sequence[Sequence[TurnMetrics]],
+    *,
+    num_samples: int,
+    seed: int | None = None,
+) -> dict[SliceName, tuple[float, float] | None]:
+    slice_names: tuple[SliceName, ...] = ("follow_up", "standalone", "all")
+    return {
+        slice_name: bootstrap_ndcg_at_3_ci(
+            conversation_turns,
+            slice_name,
+            num_samples=num_samples,
+            seed=seed,
+        )
+        for slice_name in slice_names
+    }
+
+
+def _percentile(sorted_values: Sequence[float], percentile: float) -> float:
+    if not sorted_values:
+        return 0.0
+    if len(sorted_values) == 1:
+        return sorted_values[0]
+    rank = (len(sorted_values) - 1) * (percentile / 100.0)
+    lower_index = math.floor(rank)
+    upper_index = math.ceil(rank)
+    if lower_index == upper_index:
+        return sorted_values[int(rank)]
+    weight = rank - lower_index
+    return sorted_values[lower_index] * (1.0 - weight) + sorted_values[upper_index] * weight

src/mapmatched/eval/report.py

@@ -12,8 +12,8 @@ def render_markdown_table(report: EvalReport) -> str:
         f"Benchmark: `{report.config.benchmark}` · Tier: `{report.config.tier}` · "
         f"Embedder: `{report.config.embedder_name}`",
         "",
-        "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@5 | Recall | Δ vs β=0 |",
-        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+        "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
     ]
     baseline_by_slice = _baseline_ndcg(report.methods)
     for method in report.methods:
@@ -36,6 +36,7 @@ def render_markdown_table(report: EvalReport) -> str:
                         method.method_name,
                         beta,
                         f"{slice_metrics.ndcg_at_3:.3f}",
+                        _format_ci(slice_metrics.ndcg_at_3_ci),
                         f"{slice_metrics.ndcg_at_5:.3f}",
                         f"{slice_metrics.recall_at_k:.3f}",
                         delta,
@@ -86,3 +87,9 @@ def _format_delta(value: float, baseline: float | None) -> str:
     if baseline is None:
         return "—"
     return f"{value - baseline:+.3f}"
+
+
+def _format_ci(ci: tuple[float, float] | None) -> str:
+    if ci is None:
+        return "—"
+    return f"[{ci[0]:.3f}, {ci[1]:.3f}]"

src/mapmatched/eval/runner.py

@@ -6,6 +6,7 @@
 
 from .baselines import MapMatchedMethodConfig, run_mapmatched_conversation
 from .baselines.methods import run_maximal_marginal_relevance_conversation
+from .bootstrap import bootstrap_slice_cis
 from .corpus import (
     BruteForceProvider,
     build_knn_graph,
@@ -164,7 +165,7 @@ def evaluate_method(
     entropy_threshold: float,
     graph_mode: str,
 ) -> MethodMetrics:
-    turn_metrics: list[TurnMetrics] = []
+    conversation_turns: list[list[TurnMetrics]] = []
     for conversation, trace_entropies_for_conversation in zip(
         conversations,
         trace_entropies,
@@ -178,14 +179,15 @@ def evaluate_method(
             config=config,
             ranking_mode=eval_config.ranking_mode,
         )
+        per_conversation: list[TurnMetrics] = []
         for turn, ranking, trace_entropy in zip(
             conversation.turns,
             rankings,
             trace_entropies_for_conversation,
             strict=True,
         ):
             relevances = turn_ranked_relevances(ranking, turn.qrels)
-            turn_metrics.append(
+            per_conversation.append(
                 TurnMetrics(
                     turn_index=turn.turn_index,
                     ndcg_at_3=ndcg_at_k(relevances, 3),
@@ -195,23 +197,40 @@ def evaluate_method(
                     slice_name="all",
                 )
             )
+        conversation_turns.append(per_conversation)
 
-    classified_turns = tuple(
-        TurnMetrics(
-            turn_index=turn.turn_index,
-            ndcg_at_3=turn.ndcg_at_3,
-            ndcg_at_5=turn.ndcg_at_5,
-            recall_at_k=turn.recall_at_k,
-            emission_entropy=turn.emission_entropy,
-            slice_name=classify_turn_slice(
-                turn_index=turn.turn_index,
-                emission_entropy=turn.emission_entropy,
-                entropy_threshold=entropy_threshold,
-            ),
+    classified_conversation_turns: list[list[TurnMetrics]] = []
+    for per_conversation in conversation_turns:
+        classified_conversation_turns.append(
+            [
+                TurnMetrics(
+                    turn_index=turn.turn_index,
+                    ndcg_at_3=turn.ndcg_at_3,
+                    ndcg_at_5=turn.ndcg_at_5,
+                    recall_at_k=turn.recall_at_k,
+                    emission_entropy=turn.emission_entropy,
+                    slice_name=classify_turn_slice(
+                        turn_index=turn.turn_index,
+                        emission_entropy=turn.emission_entropy,
+                        entropy_threshold=entropy_threshold,
+                    ),
+                )
+                for turn in per_conversation
+            ]
         )
-        for turn in turn_metrics
+
+    classified_turns = tuple(
+        turn for conversation in classified_conversation_turns for turn in conversation
     )
 
+    ndcg_at_3_cis = None
+    if eval_config.bootstrap_samples > 0:
+        ndcg_at_3_cis = bootstrap_slice_cis(
+            classified_conversation_turns,
+            num_samples=eval_config.bootstrap_samples,
+            seed=eval_config.bootstrap_seed,
+        )
+
     effective_transition_weight: float | None
     if method.name == "pointwise":
         effective_transition_weight = 0.0
@@ -231,6 +250,7 @@ def evaluate_method(
         fixed_lag=method.fixed_lag if method.fixed_lag is not None else config.fixed_lag,
         graph_mode=graph_mode,
         turns=classified_turns,
+        ndcg_at_3_cis=ndcg_at_3_cis,
     )
 
 

src/mapmatched/eval/slices.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from math import isclose
 
 from .metrics import mean
@@ -36,6 +36,8 @@ def classify_turn_slice(
 def aggregate_slice_metrics(
     turns: Sequence[TurnMetrics],
     slice_name: SliceName,
+    *,
+    ndcg_at_3_ci: tuple[float, float] | None = None,
 ) -> SliceMetrics:
     selected = [turn for turn in turns if turn.slice_name == slice_name]
     return SliceMetrics(
@@ -44,6 +46,7 @@ def aggregate_slice_metrics(
         ndcg_at_3=mean([turn.ndcg_at_3 for turn in selected]),
         ndcg_at_5=mean([turn.ndcg_at_5 for turn in selected]),
         recall_at_k=mean([turn.recall_at_k for turn in selected]),
+        ndcg_at_3_ci=ndcg_at_3_ci,
     )
 
 
@@ -55,17 +58,20 @@ def build_method_metrics(
     fixed_lag: int | None,
     graph_mode: str,
     turns: Sequence[TurnMetrics],
+    ndcg_at_3_cis: Mapping[SliceName, tuple[float, float] | None] | None = None,
 ) -> MethodMetrics:
+    cis = ndcg_at_3_cis or {}
     all_slice = SliceMetrics(
         slice_name="all",
         turn_count=len(turns),
         ndcg_at_3=mean([turn.ndcg_at_3 for turn in turns]),
         ndcg_at_5=mean([turn.ndcg_at_5 for turn in turns]),
         recall_at_k=mean([turn.recall_at_k for turn in turns]),
+        ndcg_at_3_ci=cis.get("all"),
     )
     slice_metrics = (
-        aggregate_slice_metrics(turns, "follow_up"),
-        aggregate_slice_metrics(turns, "standalone"),
+        aggregate_slice_metrics(turns, "follow_up", ndcg_at_3_ci=cis.get("follow_up")),
+        aggregate_slice_metrics(turns, "standalone", ndcg_at_3_ci=cis.get("standalone")),
         all_slice,
     )
     return MethodMetrics(

src/mapmatched/eval/types.py

@@ -70,6 +70,7 @@ class SliceMetrics:
     ndcg_at_3: float
     ndcg_at_5: float
     recall_at_k: float
+    ndcg_at_3_ci: tuple[float, float] | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -97,6 +98,8 @@ class EvalConfig:
     ranking_mode: str = "full"
     # "knn" (embedding fallback) or "section" (structured group_key graph).
     graph_source: str = "knn"
+    bootstrap_samples: int = 0
+    bootstrap_seed: int | None = 42
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -109,6 +112,8 @@ def to_dict(self) -> dict[str, object]:
             "follow_up_min_delta": self.follow_up_min_delta,
             "ranking_mode": self.ranking_mode,
             "graph_source": self.graph_source,
+            "bootstrap_samples": self.bootstrap_samples,
+            "bootstrap_seed": self.bootstrap_seed,
         }
 
 
@@ -144,6 +149,9 @@ def to_dict(self) -> dict[str, object]:
                             "ndcg_at_3": slice_metrics.ndcg_at_3,
                             "ndcg_at_5": slice_metrics.ndcg_at_5,
                             "recall_at_k": slice_metrics.recall_at_k,
+                            "ndcg_at_3_ci": None
+                            if slice_metrics.ndcg_at_3_ci is None
+                            else list(slice_metrics.ndcg_at_3_ci),
                         }
                         for slice_metrics in method.slices
                     ],