From fb9df42c61f1bdd09e42417f46266a436103a0c9 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 15:38:06 +0000
Subject: [PATCH 01/12] Add paired bootstrap intervals for eval deltas

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 src/mapmatched/eval/__init__.py     |  11 +-
 src/mapmatched/eval/bootstrap.py    | 161 +++++++++++++++++++++++++---
 src/mapmatched/eval/report.py       |  78 ++++++++++++--
 src/mapmatched/eval/runner.py       | 111 ++++++++++++++++++-
 src/mapmatched/eval/types.py        |  62 +++++++++++
 tests/test_eval_bootstrap.py        | 108 ++++++++++++++++++-
 tests/test_eval_report.py           |  40 ++++++-
 tests/test_eval_slices.py           |   4 +-
 tests/test_eval_synthetic_runner.py |  14 +++
 9 files changed, 557 insertions(+), 32 deletions(-)

diff --git a/src/mapmatched/eval/__init__.py b/src/mapmatched/eval/__init__.py
index f843060..43c5000 100644
--- a/src/mapmatched/eval/__init__.py
+++ b/src/mapmatched/eval/__init__.py
@@ -5,14 +5,23 @@
 from .loaders import load_cast2019_micro, load_synthetic_fixture, load_topiocqa_micro
 from .report import render_json, render_markdown_table
 from .runner import MethodSpec, run_eval
-from .types import EvalConfig, EvalConversation, EvalReport, Passage
+from .types import (
+    ComparisonSliceMetrics,
+    EvalConfig,
+    EvalConversation,
+    EvalReport,
+    MethodComparison,
+    Passage,
+)
 
 __all__ = [
     "DeterministicHashEmbedder",
+    "ComparisonSliceMetrics",
     "EvalConfig",
     "EvalConversation",
     "EvalReport",
     "MethodSpec",
+    "MethodComparison",
     "Passage",
     "QueryEmbedder",
     "default_method_grid",
diff --git a/src/mapmatched/eval/bootstrap.py b/src/mapmatched/eval/bootstrap.py
index 98977d6..e41c864 100644
--- a/src/mapmatched/eval/bootstrap.py
+++ b/src/mapmatched/eval/bootstrap.py
@@ -2,7 +2,7 @@
 
 import math
 import random
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 
 from .metrics import mean
 from .types import SliceName, TurnMetrics
@@ -19,7 +19,8 @@ def bootstrap_ndcg_at_3_ci(
 ) -> tuple[float, float] | None:
     if num_samples <= 0:
         return None
-    conversation_count = len(conversation_turns)
+    contributing_conversations = _contributing_conversations(conversation_turns, slice_name)
+    conversation_count = len(contributing_conversations)
     if conversation_count == 0:
         return None
 
@@ -27,20 +28,16 @@ def bootstrap_ndcg_at_3_ci(
     bootstrap_means: list[float] = []
     for _ in range(num_samples):
         resampled = [
-            conversation_turns[rng.randrange(conversation_count)] for _ in range(conversation_count)
-        ]
-        selected_turns = [
-            turn
-            for conversation in resampled
-            for turn in conversation
-            if turn.slice_name == slice_name
+            contributing_conversations[rng.randrange(conversation_count)]
+            for _ in range(conversation_count)
         ]
+        selected_turns = _selected_turns(resampled, slice_name)
         bootstrap_means.append(mean([turn.ndcg_at_3 for turn in selected_turns]))
 
-    sorted_means = sorted(bootstrap_means)
-    return (
-        _percentile(sorted_means, lower_percentile),
-        _percentile(sorted_means, upper_percentile),
+    return _percentile_interval(
+        bootstrap_means,
+        lower_percentile=lower_percentile,
+        upper_percentile=upper_percentile,
     )
 
 
@@ -62,6 +59,144 @@ def bootstrap_slice_cis(
     }
 
 
+def bootstrap_paired_ndcg_at_3_delta_ci(
+    treatment_conversation_turns: Sequence[Sequence[TurnMetrics]],
+    baseline_conversation_turns: Sequence[Sequence[TurnMetrics]],
+    slice_name: SliceName,
+    *,
+    num_samples: int,
+    seed: int | None = None,
+    lower_percentile: float = 2.5,
+    upper_percentile: float = 97.5,
+) -> tuple[float, float] | None:
+    if num_samples <= 0:
+        return None
+    treatment_by_id = _conversations_by_id(treatment_conversation_turns)
+    baseline_by_id = _conversations_by_id(baseline_conversation_turns)
+    _validate_alignment(treatment_by_id, baseline_by_id)
+    contributing_ids = tuple(
+        conversation_id
+        for conversation_id, turns in treatment_by_id.items()
+        if _selected_turns((turns,), slice_name)
+    )
+    if not contributing_ids:
+        return None
+
+    rng = random.Random(seed)
+    bootstrap_deltas: list[float] = []
+    for _ in range(num_samples):
+        sampled_ids = tuple(
+            contributing_ids[rng.randrange(len(contributing_ids))]
+            for _ in range(len(contributing_ids))
+        )
+        treatment_turns = _selected_turns(
+            tuple(treatment_by_id[conversation_id] for conversation_id in sampled_ids),
+            slice_name,
+        )
+        baseline_turns = _selected_turns(
+            tuple(baseline_by_id[conversation_id] for conversation_id in sampled_ids),
+            slice_name,
+        )
+        treatment_mean = mean([turn.ndcg_at_3 for turn in treatment_turns])
+        baseline_mean = mean([turn.ndcg_at_3 for turn in baseline_turns])
+        bootstrap_deltas.append(treatment_mean - baseline_mean)
+
+    return _percentile_interval(
+        bootstrap_deltas,
+        lower_percentile=lower_percentile,
+        upper_percentile=upper_percentile,
+    )
+
+
+def bootstrap_paired_slice_delta_cis(
+    treatment_conversation_turns: Sequence[Sequence[TurnMetrics]],
+    baseline_conversation_turns: Sequence[Sequence[TurnMetrics]],
+    *,
+    num_samples: int,
+    seed: int | None = None,
+) -> dict[SliceName, tuple[float, float] | None]:
+    slice_names: tuple[SliceName, ...] = ("follow_up", "standalone", "all")
+    return {
+        slice_name: bootstrap_paired_ndcg_at_3_delta_ci(
+            treatment_conversation_turns,
+            baseline_conversation_turns,
+            slice_name,
+            num_samples=num_samples,
+            seed=seed,
+        )
+        for slice_name in slice_names
+    }
+
+
+def _contributing_conversations(
+    conversation_turns: Sequence[Sequence[TurnMetrics]],
+    slice_name: SliceName,
+) -> tuple[Sequence[TurnMetrics], ...]:
+    return tuple(
+        turns for turns in conversation_turns if _selected_turns((turns,), slice_name)
+    )
+
+
+def _selected_turns(
+    conversation_turns: Sequence[Sequence[TurnMetrics]],
+    slice_name: SliceName,
+) -> list[TurnMetrics]:
+    return [
+        turn
+        for conversation in conversation_turns
+        for turn in conversation
+        if slice_name == "all" or turn.slice_name == slice_name
+    ]
+
+
+def _conversations_by_id(
+    conversation_turns: Sequence[Sequence[TurnMetrics]],
+) -> dict[str, tuple[TurnMetrics, ...]]:
+    by_id: dict[str, tuple[TurnMetrics, ...]] = {}
+    for turns in conversation_turns:
+        if not turns:
+            raise ValueError("conversation turn metrics must not be empty")
+        conversation_ids = {turn.conversation_id for turn in turns}
+        if len(conversation_ids) != 1:
+            raise ValueError("conversation turn metrics contain multiple conversation IDs")
+        conversation_id = next(iter(conversation_ids))
+        if conversation_id in by_id:
+            raise ValueError(f"duplicate conversation metrics: {conversation_id}")
+        by_id[conversation_id] = tuple(turns)
+    return by_id
+
+
+def _validate_alignment(
+    treatment_by_id: Mapping[str, Sequence[TurnMetrics]],
+    baseline_by_id: Mapping[str, Sequence[TurnMetrics]],
+) -> None:
+    if treatment_by_id.keys() != baseline_by_id.keys():
+        raise ValueError("treatment and baseline conversation IDs do not match")
+    for conversation_id, treatment_turns in treatment_by_id.items():
+        baseline_turns = baseline_by_id[conversation_id]
+        treatment_keys = tuple(
+            (turn.turn_index, turn.slice_name) for turn in treatment_turns
+        )
+        baseline_keys = tuple((turn.turn_index, turn.slice_name) for turn in baseline_turns)
+        if treatment_keys != baseline_keys:
+            raise ValueError(
+                f"treatment and baseline turns do not align for {conversation_id}"
+            )
+
+
+def _percentile_interval(
+    values: Sequence[float],
+    *,
+    lower_percentile: float,
+    upper_percentile: float,
+) -> tuple[float, float]:
+    sorted_values = sorted(values)
+    return (
+        _percentile(sorted_values, lower_percentile),
+        _percentile(sorted_values, upper_percentile),
+    )
+
+
 def _percentile(sorted_values: Sequence[float], percentile: float) -> float:
     if not sorted_values:
         return 0.0
diff --git a/src/mapmatched/eval/report.py b/src/mapmatched/eval/report.py
index 7d1179d..106bea7 100644
--- a/src/mapmatched/eval/report.py
+++ b/src/mapmatched/eval/report.py
@@ -2,20 +2,40 @@
 
 import json
 
-from .types import EvalReport, MethodMetrics, SliceName
+from .types import EvalReport, MethodComparison, MethodMetrics, SliceName
 
 
 def render_markdown_table(report: EvalReport) -> str:
+    metadata = [
+        f"Benchmark: `{report.config.benchmark}`",
+        f"Tier: `{report.config.tier}`",
+        f"Embedder: `{report.config.embedder_name}`",
+    ]
+    if report.config.profile is not None:
+        metadata.append(f"Profile: `{report.config.profile}`")
     lines = [
         "## Benchmark results (dev slice)",
         "",
-        f"Benchmark: `{report.config.benchmark}` · Tier: `{report.config.tier}` · "
-        f"Embedder: `{report.config.embedder_name}`",
+        " · ".join(metadata),
         "",
-        "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 |",
-        "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
     ]
+    if report.config.dataset_filename is not None:
+        lines.extend(
+            [
+                f"Data: `{report.config.dataset_filename}` · "
+                f"SHA-256: `{report.config.dataset_sha256}` · "
+                f"Conversations: `{len(report.config.conversation_ids)}`",
+                "",
+            ]
+        )
+    lines.extend(
+        [
+            "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 (95% CI) |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+        ]
+    )
     baseline_by_slice = _baseline_ndcg(report.methods)
+    comparison_by_slice = _comparison_delta_cis(report.comparisons)
     for method in report.methods:
         for slice_metrics in method.slices:
             if slice_metrics.slice_name == "all":
@@ -25,7 +45,17 @@ def render_markdown_table(report: EvalReport) -> str:
             baseline = baseline_by_slice.get(
                 (method.candidate_limit, method.fixed_lag, slice_metrics.slice_name),
             )
-            delta = _format_delta(slice_metrics.ndcg_at_3, baseline)
+            delta_ci = comparison_by_slice.get(
+                (
+                    method.method_name,
+                    method.transition_weight,
+                    method.candidate_limit,
+                    method.fixed_lag,
+                    method.graph_mode,
+                    slice_metrics.slice_name,
+                )
+            )
+            delta = _format_delta(slice_metrics.ndcg_at_3, baseline, delta_ci)
             beta = "—" if method.transition_weight is None else f"{method.transition_weight:.2f}"
             lines.append(
                 "| "
@@ -83,10 +113,42 @@ def _baseline_ndcg(
     return baseline
 
 
-def _format_delta(value: float, baseline: float | None) -> str:
+def _comparison_delta_cis(
+    comparisons: tuple[MethodComparison, ...],
+) -> dict[
+    tuple[str, float | None, int, int | None, str, SliceName],
+    tuple[float, float] | None,
+]:
+    by_slice: dict[
+        tuple[str, float | None, int, int | None, str, SliceName],
+        tuple[float, float] | None,
+    ] = {}
+    for comparison in comparisons:
+        for slice_metrics in comparison.slices:
+            by_slice[
+                (
+                    comparison.method_name,
+                    comparison.transition_weight,
+                    comparison.candidate_limit,
+                    comparison.fixed_lag,
+                    comparison.graph_mode,
+                    slice_metrics.slice_name,
+                )
+            ] = slice_metrics.ndcg_at_3_delta_ci
+    return by_slice
+
+
+def _format_delta(
+    value: float,
+    baseline: float | None,
+    ci: tuple[float, float] | None,
+) -> str:
     if baseline is None:
         return "—"
-    return f"{value - baseline:+.3f}"
+    delta = f"{value - baseline:+.3f}"
+    if ci is None:
+        return delta
+    return f"{delta} [{ci[0]:+.3f}, {ci[1]:+.3f}]"
 
 
 def _format_ci(ci: tuple[float, float] | None) -> str:
diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py
index e982dcd..fbbef15 100644
--- a/src/mapmatched/eval/runner.py
+++ b/src/mapmatched/eval/runner.py
@@ -6,7 +6,7 @@
 
 from .baselines import MapMatchedMethodConfig, run_mapmatched_conversation
 from .baselines.methods import run_maximal_marginal_relevance_conversation
-from .bootstrap import bootstrap_slice_cis
+from .bootstrap import bootstrap_paired_slice_delta_cis, bootstrap_slice_cis
 from .corpus import (
     BruteForceProvider,
     build_knn_graph,
@@ -21,7 +21,16 @@
     compute_entropy_threshold,
     evaluate_claim,
 )
-from .types import EvalConfig, EvalConversation, EvalReport, MethodMetrics, Passage, TurnMetrics
+from .types import (
+    ComparisonSliceMetrics,
+    EvalConfig,
+    EvalConversation,
+    EvalReport,
+    MethodComparison,
+    MethodMetrics,
+    Passage,
+    TurnMetrics,
+)
 
 
 class MethodSpec:
@@ -143,13 +152,18 @@ def _build_provider_and_graph(
     embedder: TextEmbedder,
     *,
     graph_source: str,
+    knn_neighbor_count: int,
 ) -> tuple[BruteForceProvider, CorpusGraph]:
     passage_ids, passage_embeddings = build_passage_embeddings(passages, embedder)
     provider = BruteForceProvider(passage_ids, passage_embeddings, embedder)
     if graph_source == "section":
         graph = build_section_graph(passages)
     else:
-        graph = build_knn_graph(passage_ids, passage_embeddings)
+        graph = build_knn_graph(
+            passage_ids,
+            passage_embeddings,
+            neighbor_count=knn_neighbor_count,
+        )
     return provider, graph
 
 
@@ -189,6 +203,7 @@ def evaluate_method(
             relevances = turn_ranked_relevances(ranking, turn.qrels)
             per_conversation.append(
                 TurnMetrics(
+                    conversation_id=conversation.conversation_id,
                     turn_index=turn.turn_index,
                     ndcg_at_3=ndcg_at_k(relevances, 3),
                     ndcg_at_5=ndcg_at_k(relevances, 5),
@@ -204,6 +219,7 @@ def evaluate_method(
         classified_conversation_turns.append(
             [
                 TurnMetrics(
+                    conversation_id=turn.conversation_id,
                     turn_index=turn.turn_index,
                     ndcg_at_3=turn.ndcg_at_3,
                     ndcg_at_5=turn.ndcg_at_5,
@@ -268,6 +284,7 @@ def run_eval(
         passages,
         embedder,
         graph_source=graph_mode,
+        knn_neighbor_count=eval_config.knn_neighbor_count,
     )
     trace_method = MethodSpec(name="pointwise")
     trace_entropies = tuple(
@@ -301,6 +318,7 @@ def run_eval(
         )
         for method in methods
     )
+    comparisons = _build_method_comparisons(method_metrics, eval_config)
     mapmatched_methods = [method for method in methods if method.name == "mapmatched"]
     if not mapmatched_methods:
         verdict = None
@@ -321,4 +339,89 @@ def run_eval(
             follow_up_min_delta=eval_config.follow_up_min_delta,
             mapmatched_transition_weight=best_mapmatched.transition_weight,
         )
-    return EvalReport(config=eval_config, methods=method_metrics, verdict=verdict)
+    return EvalReport(
+        config=eval_config,
+        methods=method_metrics,
+        verdict=verdict,
+        comparisons=comparisons,
+    )
+
+
+def _build_method_comparisons(
+    methods: Sequence[MethodMetrics],
+    eval_config: EvalConfig,
+) -> tuple[MethodComparison, ...]:
+    comparisons: list[MethodComparison] = []
+    comparable_method_names = {
+        "history_concat",
+        "mapmatched",
+        "maximal_marginal_relevance",
+    }
+    for method in methods:
+        if method.method_name not in comparable_method_names:
+            continue
+        baseline = _find_pointwise_baseline(methods, method)
+        if baseline is None:
+            continue
+        delta_cis = bootstrap_paired_slice_delta_cis(
+            _group_turns_by_conversation(method),
+            _group_turns_by_conversation(baseline),
+            num_samples=eval_config.bootstrap_samples,
+            seed=eval_config.bootstrap_seed,
+        )
+        baseline_slices = {
+            slice_metrics.slice_name: slice_metrics for slice_metrics in baseline.slices
+        }
+        comparison_slices = tuple(
+            ComparisonSliceMetrics(
+                slice_name=slice_metrics.slice_name,
+                turn_count=slice_metrics.turn_count,
+                ndcg_at_3_delta=(
+                    slice_metrics.ndcg_at_3
+                    - baseline_slices[slice_metrics.slice_name].ndcg_at_3
+                ),
+                ndcg_at_3_delta_ci=delta_cis[slice_metrics.slice_name],
+            )
+            for slice_metrics in method.slices
+        )
+        comparisons.append(
+            MethodComparison(
+                method_name=method.method_name,
+                transition_weight=method.transition_weight,
+                baseline_method_name=baseline.method_name,
+                baseline_transition_weight=baseline.transition_weight,
+                candidate_limit=method.candidate_limit,
+                fixed_lag=method.fixed_lag,
+                graph_mode=method.graph_mode,
+                slices=comparison_slices,
+            )
+        )
+    return tuple(comparisons)
+
+
+def _find_pointwise_baseline(
+    methods: Sequence[MethodMetrics],
+    treatment: MethodMetrics,
+) -> MethodMetrics | None:
+    for method in methods:
+        if method.method_name != "pointwise":
+            continue
+        if method.transition_weight not in (0.0, None):
+            continue
+        if method.candidate_limit != treatment.candidate_limit:
+            continue
+        if method.fixed_lag != treatment.fixed_lag:
+            continue
+        if method.graph_mode != treatment.graph_mode:
+            continue
+        return method
+    return None
+
+
+def _group_turns_by_conversation(
+    method: MethodMetrics,
+) -> tuple[tuple[TurnMetrics, ...], ...]:
+    grouped: dict[str, list[TurnMetrics]] = {}
+    for turn in method.turns:
+        grouped.setdefault(turn.conversation_id, []).append(turn)
+    return tuple(tuple(turns) for turns in grouped.values())
diff --git a/src/mapmatched/eval/types.py b/src/mapmatched/eval/types.py
index 4321cee..2953ffa 100644
--- a/src/mapmatched/eval/types.py
+++ b/src/mapmatched/eval/types.py
@@ -55,6 +55,7 @@ def __post_init__(self) -> None:
 
 @dataclass(frozen=True, slots=True)
 class TurnMetrics:
+    conversation_id: str
     turn_index: int
     ndcg_at_3: float
     ndcg_at_5: float
@@ -84,6 +85,26 @@ class MethodMetrics:
     turns: tuple[TurnMetrics, ...]
 
 
+@dataclass(frozen=True, slots=True)
+class ComparisonSliceMetrics:
+    slice_name: SliceName
+    turn_count: int
+    ndcg_at_3_delta: float
+    ndcg_at_3_delta_ci: tuple[float, float] | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class MethodComparison:
+    method_name: str
+    transition_weight: float | None
+    baseline_method_name: str
+    baseline_transition_weight: float | None
+    candidate_limit: int
+    fixed_lag: int | None
+    graph_mode: str
+    slices: tuple[ComparisonSliceMetrics, ...]
+
+
 @dataclass(frozen=True, slots=True)
 class EvalConfig:
     benchmark: str
@@ -98,8 +119,16 @@ class EvalConfig:
     ranking_mode: str = "full"
     # "knn" (embedding fallback) or "section" (structured group_key graph).
     graph_source: str = "knn"
+    knn_neighbor_count: int = 10
     bootstrap_samples: int = 0
     bootstrap_seed: int | None = 42
+    profile: str | None = None
+    dataset_filename: str | None = None
+    dataset_sha256: str | None = None
+    conversation_ids: tuple[str, ...] = ()
+    embedding_model: str | None = None
+    package_version: str | None = None
+    git_revision: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -112,8 +141,16 @@ def to_dict(self) -> dict[str, object]:
             "follow_up_min_delta": self.follow_up_min_delta,
             "ranking_mode": self.ranking_mode,
             "graph_source": self.graph_source,
+            "knn_neighbor_count": self.knn_neighbor_count,
             "bootstrap_samples": self.bootstrap_samples,
             "bootstrap_seed": self.bootstrap_seed,
+            "profile": self.profile,
+            "dataset_filename": self.dataset_filename,
+            "dataset_sha256": self.dataset_sha256,
+            "conversation_ids": list(self.conversation_ids),
+            "embedding_model": self.embedding_model,
+            "package_version": self.package_version,
+            "git_revision": self.git_revision,
         }
 
 
@@ -131,6 +168,7 @@ class EvalReport:
     config: EvalConfig
     methods: tuple[MethodMetrics, ...]
     verdict: ClaimVerdict | None
+    comparisons: tuple[MethodComparison, ...] = ()
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -157,6 +195,7 @@ def to_dict(self) -> dict[str, object]:
                     ],
                     "turns": [
                         {
+                            "conversation_id": turn.conversation_id,
                             "turn_index": turn.turn_index,
                             "ndcg_at_3": turn.ndcg_at_3,
                             "ndcg_at_5": turn.ndcg_at_5,
@@ -169,6 +208,29 @@ def to_dict(self) -> dict[str, object]:
                 }
                 for method in self.methods
             ],
+            "comparisons": [
+                {
+                    "method_name": comparison.method_name,
+                    "transition_weight": comparison.transition_weight,
+                    "baseline_method_name": comparison.baseline_method_name,
+                    "baseline_transition_weight": comparison.baseline_transition_weight,
+                    "candidate_limit": comparison.candidate_limit,
+                    "fixed_lag": comparison.fixed_lag,
+                    "graph_mode": comparison.graph_mode,
+                    "slices": [
+                        {
+                            "slice_name": slice_metrics.slice_name,
+                            "turn_count": slice_metrics.turn_count,
+                            "ndcg_at_3_delta": slice_metrics.ndcg_at_3_delta,
+                            "ndcg_at_3_delta_ci": None
+                            if slice_metrics.ndcg_at_3_delta_ci is None
+                            else list(slice_metrics.ndcg_at_3_delta_ci),
+                        }
+                        for slice_metrics in comparison.slices
+                    ],
+                }
+                for comparison in self.comparisons
+            ],
             "verdict": None
             if self.verdict is None
             else {
diff --git a/tests/test_eval_bootstrap.py b/tests/test_eval_bootstrap.py
index d271b83..90ee1c0 100644
--- a/tests/test_eval_bootstrap.py
+++ b/tests/test_eval_bootstrap.py
@@ -1,10 +1,23 @@
-from mapmatched.eval.bootstrap import bootstrap_ndcg_at_3_ci, bootstrap_slice_cis
+import pytest
+
+from mapmatched.eval.bootstrap import (
+    bootstrap_ndcg_at_3_ci,
+    bootstrap_paired_ndcg_at_3_delta_ci,
+    bootstrap_slice_cis,
+)
 from mapmatched.eval.types import SliceName, TurnMetrics
 
 
-def _turn(ndcg_at_3: float, slice_name: SliceName) -> TurnMetrics:
+def _turn(
+    ndcg_at_3: float,
+    slice_name: SliceName,
+    *,
+    conversation_id: str = "conversation-1",
+    turn_index: int = 0,
+) -> TurnMetrics:
     return TurnMetrics(
-        turn_index=0,
+        conversation_id=conversation_id,
+        turn_index=turn_index,
         ndcg_at_3=ndcg_at_3,
         ndcg_at_5=ndcg_at_3,
         recall_at_k=ndcg_at_3,
@@ -53,6 +66,94 @@ def test_bootstrap_slice_cis_returns_all_slices() -> None:
     assert set(cis) == {"follow_up", "standalone", "all"}
     assert cis["follow_up"] is not None
     assert cis["follow_up"][0] <= cis["follow_up"][1]
+    assert cis["all"] is not None
+    assert cis["all"] != (0.0, 0.0)
+
+
+def test_paired_bootstrap_positive_delta_excludes_zero() -> None:
+    treatment = (
+        (_turn(1.0, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.8, "follow_up", conversation_id="conversation-2"),),
+    )
+    baseline = (
+        (_turn(0.2, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.3, "follow_up", conversation_id="conversation-2"),),
+    )
+    ci = bootstrap_paired_ndcg_at_3_delta_ci(
+        treatment,
+        baseline,
+        "follow_up",
+        num_samples=200,
+        seed=7,
+    )
+    assert ci is not None
+    assert ci[0] > 0.0
+
+
+def test_paired_bootstrap_zero_delta_is_exact() -> None:
+    treatment = (
+        (_turn(0.2, "standalone", conversation_id="conversation-1"),),
+        (_turn(0.8, "standalone", conversation_id="conversation-2"),),
+    )
+    ci = bootstrap_paired_ndcg_at_3_delta_ci(
+        treatment,
+        treatment,
+        "standalone",
+        num_samples=100,
+        seed=11,
+    )
+    assert ci == (0.0, 0.0)
+
+
+def test_paired_bootstrap_negative_delta_excludes_zero() -> None:
+    treatment = (
+        (_turn(0.1, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.2, "follow_up", conversation_id="conversation-2"),),
+    )
+    baseline = (
+        (_turn(0.8, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.9, "follow_up", conversation_id="conversation-2"),),
+    )
+    ci = bootstrap_paired_ndcg_at_3_delta_ci(
+        treatment,
+        baseline,
+        "follow_up",
+        num_samples=100,
+        seed=3,
+    )
+    assert ci is not None
+    assert ci[1] < 0.0
+
+
+def test_paired_bootstrap_rejects_misaligned_conversations() -> None:
+    treatment = ((_turn(1.0, "follow_up", conversation_id="conversation-1"),),)
+    baseline = ((_turn(1.0, "follow_up", conversation_id="conversation-2"),),)
+    with pytest.raises(ValueError, match="conversation IDs do not match"):
+        bootstrap_paired_ndcg_at_3_delta_ci(
+            treatment,
+            baseline,
+            "follow_up",
+            num_samples=10,
+        )
+
+
+def test_paired_bootstrap_ignores_conversations_without_requested_slice() -> None:
+    treatment = (
+        (_turn(0.8, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.1, "standalone", conversation_id="conversation-2"),),
+    )
+    baseline = (
+        (_turn(0.3, "follow_up", conversation_id="conversation-1"),),
+        (_turn(0.9, "standalone", conversation_id="conversation-2"),),
+    )
+    ci = bootstrap_paired_ndcg_at_3_delta_ci(
+        treatment,
+        baseline,
+        "follow_up",
+        num_samples=50,
+        seed=5,
+    )
+    assert ci == (0.5, 0.5)
 
 
 def test_synthetic_eval_populates_bootstrap_cis() -> None:
@@ -92,3 +193,4 @@ def test_synthetic_eval_populates_bootstrap_cis() -> None:
     )
     assert follow_up.ndcg_at_3_ci is not None
     assert follow_up.ndcg_at_3_ci[0] <= follow_up.ndcg_at_3 <= follow_up.ndcg_at_3_ci[1]
+    assert report.methods[0].turns[0].conversation_id
diff --git a/tests/test_eval_report.py b/tests/test_eval_report.py
index 23c9bc2..42edbbf 100644
--- a/tests/test_eval_report.py
+++ b/tests/test_eval_report.py
@@ -1,5 +1,12 @@
 from mapmatched.eval.report import render_json, render_markdown_table
-from mapmatched.eval.types import EvalConfig, EvalReport, MethodMetrics, SliceMetrics
+from mapmatched.eval.types import (
+    ComparisonSliceMetrics,
+    EvalConfig,
+    EvalReport,
+    MethodComparison,
+    MethodMetrics,
+    SliceMetrics,
+)
 
 
 def test_report_json_and_markdown_are_deterministic() -> None:
@@ -27,11 +34,42 @@ def test_report_json_and_markdown_are_deterministic() -> None:
                 ),
                 turns=(),
             ),
+            MethodMetrics(
+                method_name="mapmatched",
+                transition_weight=0.5,
+                candidate_limit=20,
+                fixed_lag=None,
+                graph_mode="knn",
+                slices=(
+                    SliceMetrics("follow_up", 2, 0.5, 0.5, 0.5),
+                    SliceMetrics("standalone", 1, 0.8, 0.8, 0.8),
+                    SliceMetrics("all", 3, 0.6, 0.6, 0.6),
+                ),
+                turns=(),
+            ),
         ),
         verdict=None,
+        comparisons=(
+            MethodComparison(
+                method_name="mapmatched",
+                transition_weight=0.5,
+                baseline_method_name="pointwise",
+                baseline_transition_weight=0.0,
+                candidate_limit=20,
+                fixed_lag=None,
+                graph_mode="knn",
+                slices=(
+                    ComparisonSliceMetrics("follow_up", 2, 0.1, (0.02, 0.18)),
+                    ComparisonSliceMetrics("standalone", 1, 0.0, (0.0, 0.0)),
+                    ComparisonSliceMetrics("all", 3, 0.067, (0.01, 0.12)),
+                ),
+            ),
+        ),
     )
     json_payload = render_json(report)
     markdown = render_markdown_table(report)
     assert '"benchmark": "synthetic"' in json_payload
     assert "| synthetic | follow_up | pointwise |" in markdown
     assert "nDCG@3 95% CI" in markdown
+    assert "+0.100 [+0.020, +0.180]" in markdown
+    assert '"ndcg_at_3_delta_ci": [' in json_payload
diff --git a/tests/test_eval_slices.py b/tests/test_eval_slices.py
index b644cce..ccbffcd 100644
--- a/tests/test_eval_slices.py
+++ b/tests/test_eval_slices.py
@@ -61,8 +61,8 @@ def test_evaluate_claim_checks_follow_up_and_standalone_gates() -> None:
 
 def test_aggregate_slice_metrics_skips_other_slices() -> None:
     turns = (
-        TurnMetrics(0, 1.0, 1.0, 1.0, 0.0, "standalone"),
-        TurnMetrics(1, 0.5, 0.5, 0.5, 1.0, "follow_up"),
+        TurnMetrics("conversation-1", 0, 1.0, 1.0, 1.0, 0.0, "standalone"),
+        TurnMetrics("conversation-1", 1, 0.5, 0.5, 0.5, 1.0, "follow_up"),
     )
     follow_up = aggregate_slice_metrics(turns, "follow_up")
     assert follow_up.turn_count == 1
diff --git a/tests/test_eval_synthetic_runner.py b/tests/test_eval_synthetic_runner.py
index 5ff9a7b..3ee6ef5 100644
--- a/tests/test_eval_synthetic_runner.py
+++ b/tests/test_eval_synthetic_runner.py
@@ -31,9 +31,23 @@ def test_synthetic_eval_runs_end_to_end() -> None:
             entropy_threshold=None,
             standalone_tolerance=0.02,
             follow_up_min_delta=0.0,
+            bootstrap_samples=50,
+            bootstrap_seed=42,
         ),
     )
     assert report.methods
     markdown = render_markdown_table(report)
     assert "Benchmark results" in markdown
     assert "pointwise" in markdown
+    assert report.comparisons
+    mapmatched_comparison = next(
+        comparison
+        for comparison in report.comparisons
+        if comparison.method_name == "mapmatched"
+    )
+    follow_up = next(
+        slice_metrics
+        for slice_metrics in mapmatched_comparison.slices
+        if slice_metrics.slice_name == "follow_up"
+    )
+    assert follow_up.ndcg_at_3_delta_ci is not None

From 9943ef1aeaa998364ac08e8d7557ee4a55637c82 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 15:38:30 +0000
Subject: [PATCH 02/12] Add pinned TopiOCQA n25 reproduction profile

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 .gitignore                                 |   2 +
 CHANGELOG.md                               |   5 +
 README.md                                  |  11 ++-
 docs/eval.md                               |  55 ++++++-----
 scripts/reproduce_topiocqa_n25.sh          |  45 +++++++++
 src/mapmatched/eval/__main__.py            |  66 ++++++++++++-
 src/mapmatched/eval/loaders/topiocqa.py    |   6 +-
 tests/fixtures/topiocqa_micro_sample.jsonl |   4 +
 tests/test_eval_topiocqa_loader.py         | 102 +++++++++++++++++++++
 9 files changed, 269 insertions(+), 27 deletions(-)
 create mode 100755 scripts/reproduce_topiocqa_n25.sh
 create mode 100644 tests/fixtures/topiocqa_micro_sample.jsonl
 create mode 100644 tests/test_eval_topiocqa_loader.py

diff --git a/.gitignore b/.gitignore
index da3a6a9..96cdac9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,5 @@ __pycache__/
 build/
 dist/
 eval-report.json
+data/
+reports/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e2ca93b..8a1ad12 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project are documented here.
 
 ## Unreleased
 
+- Add paired conversation-level bootstrap intervals for method nDCG@3 deltas,
+  preserve conversation IDs in reports, and fix the aggregate `all`-slice
+  bootstrap interval.
+- Add a pinned TopiOCQA n=25 MiniLM/kNN reproduction script with dataset
+  checksum, selected conversation IDs, model, graph, package, and git provenance.
 - Bootstrap confidence intervals: conversation-level percentile bootstrap for
   nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable
   runs). CIs appear in JSON reports and the markdown table.
diff --git a/README.md b/README.md
index 262f5b8..ebcb8bf 100644
--- a/README.md
+++ b/README.md
@@ -194,7 +194,8 @@ context are separate outputs.
 The evaluation harness reports nDCG@3/5 and Recall@k separately for ambiguous
 follow-up turns and sharp standalone turns. It includes pointwise, history
 concatenation, Maximal Marginal Relevance, and resolved-query baselines, plus
-conversation-level percentile bootstrap confidence intervals.
+conversation-level percentile bootstrap confidence intervals. Method deltas use
+paired resampling of the same conversations.
 
 ```console
 python -m mapmatched.eval \
@@ -219,6 +220,14 @@ section graphs also underperform on topic-switch-heavy TopiOCQA, an important
 negative result rather than a hidden one. See [`docs/eval.md`](docs/eval.md) for
 benchmark tiers, methodology, limitations, and reproduction commands.
 
+Reproduce the pinned n=25 MiniLM/kNN profile after downloading the validation
+split:
+
+```console
+python -m pip install -e ".[eval,graph,st]"
+./scripts/reproduce_topiocqa_n25.sh data/topiocqa_valid.jsonl
+```
+
 ## Design choices and limits
 
 - Per-turn z-score normalization is the safe default; `center` and `none` are
diff --git a/docs/eval.md b/docs/eval.md
index 2fb8fe1..94576ec 100644
--- a/docs/eval.md
+++ b/docs/eval.md
@@ -1,13 +1,14 @@
 # Evaluation harness
 
 Map-matched retrieval makes a narrow claim: it should lift **underspecified
-follow-up turns** without materially harming **sharp standalone turns**. M2 adds
-an optional evaluation harness behind `pip install map-matched-retrieval[eval]`.
+follow-up turns** without materially harming **sharp standalone turns**. The
+evaluation harness measures that claim without making conversational RAG the
+library's API boundary.
 
 ## Install
 
 ```console
-pip install map-matched-retrieval[eval,graph]
+python -m pip install -e ".[eval,graph,st]"
 ```
 
 The harness uses NumPy for kNN graph construction and optional Hugging Face /
@@ -61,18 +62,26 @@ best map-matched configuration against the β=0 pointwise baseline:
 
 ### TopiOCQA (micro)
 
+Download `data/topiocqa_valid.jsonl` from the
+[TopiOCQA dataset repository](https://huggingface.co/datasets/McGill-NLP/TopiOCQA),
+then run the pinned profile:
+
 ```console
-# download a split first (HF datasets dropped the custom dataset script):
-#   https://huggingface.co/datasets/McGill-NLP/TopiOCQA -> data/topiocqa_valid.jsonl
-python -m mapmatched.eval --benchmark topiocqa \
-    --data-path topiocqa_valid.jsonl --conversation-limit 25 \
-    --embedder sentence-transformers --graph-source section --ranking-mode full
+./scripts/reproduce_topiocqa_n25.sh data/topiocqa_valid.jsonl
 ```
 
 Reads the released JSON/JSONL directly (`--data-path` or `MAPMATCHED_TOPIOCQA_PATH`)
-and builds a micro-corpus from gold passages and additional answers. The section
-graph keys on the Wikipedia article title. Note TopiOCQA is topic-switch heavy,
-so it stresses the standalone (H0) side as much as the follow-up (H1) side.
+and builds a micro-corpus from gold passages and additional answers. The pinned
+profile selects the first 25 conversations in file order, records the file
+SHA-256 and selected IDs, uses `sentence-transformers/all-MiniLM-L6-v2`, a
+10-neighbor kNN graph, full ranking, a 100-candidate window, and 1,000 bootstrap
+draws with seed 42.
+
+TopiOCQA is topic-switch heavy, so it stresses the standalone side as much as the
+follow-up side. It is licensed
+[CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/); the
+dataset is not redistributed by this repository. These runs are micro-corpus
+experiments, not full-Wikipedia retrieval.
 
 ### TREC CAsT 2019 (micro)
 
@@ -100,17 +109,20 @@ the follow-up-lift claim than TopiOCQA's topic switches.
 ## Bootstrap confidence intervals
 
 Use `--bootstrap-samples` to resample conversations and compute 95% percentile
-CIs for nDCG@3 on each slice. Disabled by default (`0`) for fast smoke runs;
-use `1000` for publishable numbers. `--bootstrap-seed` (default 42) keeps runs
-reproducible.
+CIs for nDCG@3 on each slice. Method comparisons resample the same conversations
+for treatment and pointwise retrieval, producing a paired CI on the nDCG@3
+delta. Disabled by default (`0`) for fast smoke runs; use `1000` for reported
+numbers. `--bootstrap-seed` (default 42) keeps runs reproducible.
 
 ```console
 python -m mapmatched.eval --benchmark synthetic \
     --bootstrap-samples 200 --output eval-report.json
 ```
 
-The markdown table adds an `nDCG@3 95% CI` column; JSON reports include
-`ndcg_at_3_ci` as `[lower, upper]` on each slice.
+The markdown table includes absolute and paired-delta 95% CIs. JSON reports
+include method-level `ndcg_at_3_ci` values and explicit `comparisons` with
+`ndcg_at_3_delta_ci`. The claim gate remains based on configured point-estimate
+thresholds; a paired interval excluding zero is the uncertainty check.
 
 ## Reproducing headline numbers
 
@@ -118,12 +130,11 @@ The README headline table uses **Tier B dev-slice** results with a
 caller-supplied embedder. The built-in `DeterministicHashEmbedder` is for tests
 and smoke runs only.
 
-For publishable numbers:
-
-1. Choose an embedding model and implement `QueryEmbedder` / `PassageEmbedder`.
-2. Build a micro-corpus or full corpus index.
-3. Run the ablation grid and record JSON + markdown output.
-4. Paste the markdown table into README with the embedder and tier noted.
+The TopiOCQA script writes full JSON and markdown reports under
+`reports/topiocqa-n25-minilm-knn-full/`. Generated reports are ignored because
+they contain machine-run detail; committed headline values must include the
+profile, dataset SHA-256, model, graph settings, conversation count, and paired
+interval.
 
 ## Output
 
diff --git a/scripts/reproduce_topiocqa_n25.sh b/scripts/reproduce_topiocqa_n25.sh
new file mode 100755
index 0000000..81e5bed
--- /dev/null
+++ b/scripts/reproduce_topiocqa_n25.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DATA_PATH="${1:-${MAPMATCHED_TOPIOCQA_PATH:-}}"
+PROFILE="topiocqa-n25-minilm-knn-full"
+REPORT_DIR="${MAPMATCHED_REPORT_DIR:-$ROOT_DIR/reports/$PROFILE}"
+
+if [[ -z "$DATA_PATH" ]]; then
+  echo "usage: $0 /path/to/topiocqa_valid.jsonl" >&2
+  echo "or set MAPMATCHED_TOPIOCQA_PATH" >&2
+  exit 2
+fi
+
+if [[ ! -f "$DATA_PATH" ]]; then
+  echo "TopiOCQA data file not found: $DATA_PATH" >&2
+  exit 2
+fi
+
+mkdir -p "$REPORT_DIR"
+
+export CUDA_VISIBLE_DEVICES=""
+export PYTHONHASHSEED=0
+export TOKENIZERS_PARALLELISM=false
+
+python3 -m mapmatched.eval \
+  --profile "$PROFILE" \
+  --benchmark topiocqa \
+  --tier micro \
+  --data-path "$DATA_PATH" \
+  --conversation-limit 25 \
+  --embedder sentence-transformers \
+  --st-model sentence-transformers/all-MiniLM-L6-v2 \
+  --graph-source knn \
+  --knn-neighbors 10 \
+  --ranking-mode full \
+  --candidate-limit 100 \
+  --recall-k 100 \
+  --bootstrap-samples 1000 \
+  --bootstrap-seed 42 \
+  --output "$REPORT_DIR/report.json" \
+  --markdown-output "$REPORT_DIR/report.md"
+
+echo "JSON report: $REPORT_DIR/report.json"
+echo "Markdown report: $REPORT_DIR/report.md"
diff --git a/src/mapmatched/eval/__main__.py b/src/mapmatched/eval/__main__.py
index 867398e..6d9a6c0 100644
--- a/src/mapmatched/eval/__main__.py
+++ b/src/mapmatched/eval/__main__.py
@@ -1,8 +1,13 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
+import os
+import subprocess
 from pathlib import Path
 
+from mapmatched import __version__
+
 from .ablations import run_ablation_grid
 from .embedder import DeterministicHashEmbedder, SentenceTransformerEmbedder
 from .loaders import load_cast2019_micro, load_synthetic_fixture, load_topiocqa_micro
@@ -44,6 +49,12 @@ def build_parser() -> argparse.ArgumentParser:
         default="knn",
         help="knn = embedding fallback graph; section = structured group_key graph.",
     )
+    parser.add_argument(
+        "--knn-neighbors",
+        type=int,
+        default=10,
+        help="Neighbors per passage when --graph-source knn.",
+    )
     parser.add_argument(
         "--ranking-mode",
         choices=("full", "rank1"),
@@ -71,6 +82,11 @@ def build_parser() -> argparse.ArgumentParser:
         default=42,
         help="Random seed for bootstrap resampling.",
     )
+    parser.add_argument(
+        "--profile",
+        default=None,
+        help="Stable run profile name stored in report metadata.",
+    )
     parser.add_argument("--include-resolved-oracle", action="store_true")
     return parser
 
@@ -97,10 +113,11 @@ def load_benchmark(
 def main(argv: list[str] | None = None) -> int:
     parser = build_parser()
     args = parser.parse_args(argv)
+    data_path = _effective_data_path(args.benchmark, args.data_path)
     conversations, passages = load_benchmark(
         args.benchmark,
         conversation_limit=args.conversation_limit,
-        data_path=args.data_path,
+        data_path=data_path,
     )
     embedder: DeterministicHashEmbedder | SentenceTransformerEmbedder
     if args.embedder == "sentence-transformers":
@@ -117,8 +134,20 @@ def main(argv: list[str] | None = None) -> int:
         follow_up_min_delta=args.follow_up_min_delta,
         ranking_mode=args.ranking_mode,
         graph_source=args.graph_source,
+        knn_neighbor_count=args.knn_neighbors,
         bootstrap_samples=args.bootstrap_samples,
         bootstrap_seed=args.bootstrap_seed,
+        profile=args.profile,
+        dataset_filename=None if data_path is None else data_path.name,
+        dataset_sha256=None if data_path is None else _sha256(data_path),
+        conversation_ids=tuple(
+            conversation.conversation_id for conversation in conversations
+        ),
+        embedding_model=args.st_model
+        if args.embedder == "sentence-transformers"
+        else embedder.name,
+        package_version=__version__,
+        git_revision=_git_revision(),
     )
     report = run_ablation_grid(
         conversations=conversations,
@@ -137,5 +166,40 @@ def main(argv: list[str] | None = None) -> int:
     return 0
 
 
+def _effective_data_path(benchmark: str, data_path: Path | None) -> Path | None:
+    if data_path is not None:
+        return data_path
+    if benchmark != "topiocqa":
+        return None
+    environment_path = os.environ.get("MAPMATCHED_TOPIOCQA_PATH")
+    if environment_path is None:
+        return None
+    return Path(environment_path)
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as data_file:
+        for block in iter(lambda: data_file.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _git_revision() -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+    except OSError:
+        return None
+    if result.returncode != 0:
+        return None
+    revision = result.stdout.strip()
+    return revision or None
+
+
 if __name__ == "__main__":
     raise SystemExit(main())
diff --git a/src/mapmatched/eval/loaders/topiocqa.py b/src/mapmatched/eval/loaders/topiocqa.py
index 710b6ec..3e0a77e 100644
--- a/src/mapmatched/eval/loaders/topiocqa.py
+++ b/src/mapmatched/eval/loaders/topiocqa.py
@@ -20,15 +20,15 @@ def _resolve_data_path(data_path: str | os.PathLike[str] | None) -> Path:
 
     HuggingFace `datasets` dropped support for the custom dataset *script* that
     `McGill-NLP/TopiOCQA` ships, so we read the released JSON directly instead.
-    Point this at a downloaded split (e.g. ``topiocqa_dev.json``) via the
+    Point this at the downloaded validation split (``topiocqa_valid.jsonl``) via the
     ``data_path`` argument or the ``MAPMATCHED_TOPIOCQA_PATH`` environment
     variable.
     """
     candidate = data_path if data_path is not None else os.environ.get(_PATH_ENV_VAR)
     if not candidate:
         raise EvalDependencyUnavailableError(
-            "TopiOCQA loader needs the dataset JSON. Download a split (e.g. "
-            "topiocqa_dev.json) from https://github.com/McGill-NLP/topiocqa and "
+            "TopiOCQA loader needs the dataset JSONL. Download data/topiocqa_valid.jsonl "
+            "from https://huggingface.co/datasets/McGill-NLP/TopiOCQA and "
             f"pass data_path=... or set {_PATH_ENV_VAR}."
         )
     path = Path(candidate)
diff --git a/tests/fixtures/topiocqa_micro_sample.jsonl b/tests/fixtures/topiocqa_micro_sample.jsonl
new file mode 100644
index 0000000..30017d4
--- /dev/null
+++ b/tests/fixtures/topiocqa_micro_sample.jsonl
@@ -0,0 +1,4 @@
+{"Conversation_no": 1, "Turn_no": 1, "Question": "What is orbital mechanics?", "Gold_passage": {"id": "orbit-1", "title": "Orbital mechanics", "text": "Orbital mechanics studies spacecraft motion."}, "Additional_answers": [{"Answer": "Gravity determines an orbit.", "Topic": "Gravity"}]}
+{"Conversation_no": 2, "Turn_no": 1, "Question": "What is photosynthesis?", "Gold_passage": {"id": "plant-1", "title": "Photosynthesis", "text": "Plants convert light into chemical energy."}, "Additional_answers": []}
+{"Conversation_no": 1, "Turn_no": 2, "Question": "How are transfers performed?", "Gold_passage": {"id": "orbit-2", "title": "Orbital mechanics", "text": "A transfer orbit moves between trajectories."}, "Additional_answers": []}
+{"Conversation_no": 3, "Turn_no": 1, "Question": "What is plate tectonics?", "Gold_passage": {"id": "earth-1", "title": "Plate tectonics", "text": "Tectonic plates move across Earth."}, "Additional_answers": []}
diff --git a/tests/test_eval_topiocqa_loader.py b/tests/test_eval_topiocqa_loader.py
new file mode 100644
index 0000000..b167d29
--- /dev/null
+++ b/tests/test_eval_topiocqa_loader.py
@@ -0,0 +1,102 @@
+import hashlib
+import json
+from pathlib import Path
+
+from mapmatched.eval import DeterministicHashEmbedder, EvalConfig, MethodSpec, run_eval
+from mapmatched.eval.__main__ import main
+from mapmatched.eval.baselines import MapMatchedMethodConfig
+from mapmatched.eval.loaders import load_topiocqa_micro
+
+
+FIXTURE_PATH = Path(__file__).parent / "fixtures" / "topiocqa_micro_sample.jsonl"
+
+
+def test_topiocqa_loader_limits_conversations_in_file_order() -> None:
+    conversations, passages = load_topiocqa_micro(
+        conversation_limit=2,
+        data_path=FIXTURE_PATH,
+    )
+    assert tuple(conversation.conversation_id for conversation in conversations) == (
+        "topiocqa-1",
+        "topiocqa-2",
+    )
+    assert tuple(len(conversation.turns) for conversation in conversations) == (2, 1)
+    assert {passage.passage_id for passage in passages} == {
+        "orbit-1",
+        "topiocqa-extra-1-0-1",
+        "orbit-2",
+        "plant-1",
+    }
+    assert conversations[0].turns[0].qrels == {
+        "orbit-1": 3,
+        "topiocqa-extra-1-0-1": 2,
+    }
+
+
+def test_topiocqa_fixture_runs_end_to_end_offline() -> None:
+    conversations, passages = load_topiocqa_micro(
+        conversation_limit=2,
+        data_path=FIXTURE_PATH,
+    )
+    embedder = DeterministicHashEmbedder()
+    report = run_eval(
+        conversations=conversations,
+        passages=passages,
+        embedder=embedder,
+        methods=(
+            MethodSpec(name="pointwise", transition_weight=0.0),
+            MethodSpec(name="mapmatched", transition_weight=0.5),
+        ),
+        config=MapMatchedMethodConfig(candidate_limit=4),
+        eval_config=EvalConfig(
+            benchmark="topiocqa",
+            tier="micro",
+            embedder_name=embedder.name,
+            recall_k=4,
+            entropy_threshold=None,
+            standalone_tolerance=0.02,
+            follow_up_min_delta=0.0,
+            bootstrap_samples=20,
+            bootstrap_seed=42,
+            conversation_ids=tuple(
+                conversation.conversation_id for conversation in conversations
+            ),
+        ),
+    )
+    assert report.comparisons
+    assert report.config.conversation_ids == ("topiocqa-1", "topiocqa-2")
+    assert all(turn.conversation_id for method in report.methods for turn in method.turns)
+
+
+def test_topiocqa_cli_records_reproduction_metadata(tmp_path: Path) -> None:
+    output_path = tmp_path / "report.json"
+    exit_code = main(
+        [
+            "--profile",
+            "fixture-profile",
+            "--benchmark",
+            "topiocqa",
+            "--data-path",
+            str(FIXTURE_PATH),
+            "--conversation-limit",
+            "2",
+            "--knn-neighbors",
+            "2",
+            "--candidate-limit",
+            "4",
+            "--bootstrap-samples",
+            "10",
+            "--output",
+            str(output_path),
+        ]
+    )
+    payload = json.loads(output_path.read_text(encoding="utf-8"))
+    expected_sha256 = hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest()
+    assert exit_code == 0
+    assert payload["config"]["profile"] == "fixture-profile"
+    assert payload["config"]["dataset_filename"] == FIXTURE_PATH.name
+    assert payload["config"]["dataset_sha256"] == expected_sha256
+    assert payload["config"]["conversation_ids"] == ["topiocqa-1", "topiocqa-2"]
+    assert payload["config"]["knn_neighbor_count"] == 2
+    assert payload["config"]["package_version"] == "0.1.0"
+    assert payload["config"]["git_revision"]

From ec908422ecc1017cc72536f229c3962ad4f46bb5 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 15:39:22 +0000
Subject: [PATCH 03/12] Fix evaluation quality checks

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 src/mapmatched/eval/__init__.py    | 4 ++--
 src/mapmatched/eval/report.py      | 5 +++--
 tests/test_eval_topiocqa_loader.py | 1 -
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/mapmatched/eval/__init__.py b/src/mapmatched/eval/__init__.py
index 43c5000..f0ad5ba 100644
--- a/src/mapmatched/eval/__init__.py
+++ b/src/mapmatched/eval/__init__.py
@@ -15,13 +15,13 @@
 )
 
 __all__ = [
-    "DeterministicHashEmbedder",
     "ComparisonSliceMetrics",
+    "DeterministicHashEmbedder",
     "EvalConfig",
     "EvalConversation",
     "EvalReport",
-    "MethodSpec",
     "MethodComparison",
+    "MethodSpec",
     "Passage",
     "QueryEmbedder",
     "default_method_grid",
diff --git a/src/mapmatched/eval/report.py b/src/mapmatched/eval/report.py
index 106bea7..c6e5b68 100644
--- a/src/mapmatched/eval/report.py
+++ b/src/mapmatched/eval/report.py
@@ -30,8 +30,9 @@ def render_markdown_table(report: EvalReport) -> str:
         )
     lines.extend(
         [
-            "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 (95% CI) |",
-        "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+            "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | "
+            "nDCG@5 | Recall | Δ vs β=0 (95% CI) |",
+            "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
         ]
     )
     baseline_by_slice = _baseline_ndcg(report.methods)
diff --git a/tests/test_eval_topiocqa_loader.py b/tests/test_eval_topiocqa_loader.py
index b167d29..14218e6 100644
--- a/tests/test_eval_topiocqa_loader.py
+++ b/tests/test_eval_topiocqa_loader.py
@@ -7,7 +7,6 @@
 from mapmatched.eval.baselines import MapMatchedMethodConfig
 from mapmatched.eval.loaders import load_topiocqa_micro
 
-
 FIXTURE_PATH = Path(__file__).parent / "fixtures" / "topiocqa_micro_sample.jsonl"
 
 

From 7810a19c6b2eba4181f2146378690845119f6da4 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 15:39:52 +0000
Subject: [PATCH 04/12] Format paired evaluation changes

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 src/mapmatched/eval/__main__.py     |  4 +---
 src/mapmatched/eval/bootstrap.py    | 12 +++---------
 src/mapmatched/eval/runner.py       |  3 +--
 tests/test_eval_synthetic_runner.py |  4 +---
 tests/test_eval_topiocqa_loader.py  |  4 +---
 5 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/src/mapmatched/eval/__main__.py b/src/mapmatched/eval/__main__.py
index 6d9a6c0..fe9ba3b 100644
--- a/src/mapmatched/eval/__main__.py
+++ b/src/mapmatched/eval/__main__.py
@@ -140,9 +140,7 @@ def main(argv: list[str] | None = None) -> int:
         profile=args.profile,
         dataset_filename=None if data_path is None else data_path.name,
         dataset_sha256=None if data_path is None else _sha256(data_path),
-        conversation_ids=tuple(
-            conversation.conversation_id for conversation in conversations
-        ),
+        conversation_ids=tuple(conversation.conversation_id for conversation in conversations),
         embedding_model=args.st_model
         if args.embedder == "sentence-transformers"
         else embedder.name,
diff --git a/src/mapmatched/eval/bootstrap.py b/src/mapmatched/eval/bootstrap.py
index e41c864..4aeee6d 100644
--- a/src/mapmatched/eval/bootstrap.py
+++ b/src/mapmatched/eval/bootstrap.py
@@ -132,9 +132,7 @@ def _contributing_conversations(
     conversation_turns: Sequence[Sequence[TurnMetrics]],
     slice_name: SliceName,
 ) -> tuple[Sequence[TurnMetrics], ...]:
-    return tuple(
-        turns for turns in conversation_turns if _selected_turns((turns,), slice_name)
-    )
+    return tuple(turns for turns in conversation_turns if _selected_turns((turns,), slice_name))
 
 
 def _selected_turns(
@@ -174,14 +172,10 @@ def _validate_alignment(
         raise ValueError("treatment and baseline conversation IDs do not match")
     for conversation_id, treatment_turns in treatment_by_id.items():
         baseline_turns = baseline_by_id[conversation_id]
-        treatment_keys = tuple(
-            (turn.turn_index, turn.slice_name) for turn in treatment_turns
-        )
+        treatment_keys = tuple((turn.turn_index, turn.slice_name) for turn in treatment_turns)
         baseline_keys = tuple((turn.turn_index, turn.slice_name) for turn in baseline_turns)
         if treatment_keys != baseline_keys:
-            raise ValueError(
-                f"treatment and baseline turns do not align for {conversation_id}"
-            )
+            raise ValueError(f"treatment and baseline turns do not align for {conversation_id}")
 
 
 def _percentile_interval(
diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py
index fbbef15..01d0b40 100644
--- a/src/mapmatched/eval/runner.py
+++ b/src/mapmatched/eval/runner.py
@@ -377,8 +377,7 @@ def _build_method_comparisons(
                 slice_name=slice_metrics.slice_name,
                 turn_count=slice_metrics.turn_count,
                 ndcg_at_3_delta=(
-                    slice_metrics.ndcg_at_3
-                    - baseline_slices[slice_metrics.slice_name].ndcg_at_3
+                    slice_metrics.ndcg_at_3 - baseline_slices[slice_metrics.slice_name].ndcg_at_3
                 ),
                 ndcg_at_3_delta_ci=delta_cis[slice_metrics.slice_name],
             )
diff --git a/tests/test_eval_synthetic_runner.py b/tests/test_eval_synthetic_runner.py
index 3ee6ef5..631a56e 100644
--- a/tests/test_eval_synthetic_runner.py
+++ b/tests/test_eval_synthetic_runner.py
@@ -41,9 +41,7 @@ def test_synthetic_eval_runs_end_to_end() -> None:
     assert "pointwise" in markdown
     assert report.comparisons
     mapmatched_comparison = next(
-        comparison
-        for comparison in report.comparisons
-        if comparison.method_name == "mapmatched"
+        comparison for comparison in report.comparisons if comparison.method_name == "mapmatched"
     )
     follow_up = next(
         slice_metrics
diff --git a/tests/test_eval_topiocqa_loader.py b/tests/test_eval_topiocqa_loader.py
index 14218e6..0621998 100644
--- a/tests/test_eval_topiocqa_loader.py
+++ b/tests/test_eval_topiocqa_loader.py
@@ -57,9 +57,7 @@ def test_topiocqa_fixture_runs_end_to_end_offline() -> None:
             follow_up_min_delta=0.0,
             bootstrap_samples=20,
             bootstrap_seed=42,
-            conversation_ids=tuple(
-                conversation.conversation_id for conversation in conversations
-            ),
+            conversation_ids=tuple(conversation.conversation_id for conversation in conversations),
         ),
     )
     assert report.comparisons

From 60a9d694b807c3eb49da2a00743c6f1a05d52e6d Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 15:58:08 +0000
Subject: [PATCH 05/12] Avoid repeated embeddings and graph searches in eval

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 CHANGELOG.md                    |  2 ++
 src/mapmatched/eval/corpus.py   |  6 +++++-
 src/mapmatched/graph.py         | 15 ++++++++++++---
 tests/test_eval_baselines.py    | 21 +++++++++++++++++++++
 tests/test_graph_and_scoring.py | 22 ++++++++++++++++++++++
 5 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8a1ad12..af406b2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ All notable changes to this project are documented here.
   bootstrap interval.
 - Add a pinned TopiOCQA n=25 MiniLM/kNN reproduction script with dataset
   checksum, selected conversation IDs, model, graph, package, and git provenance.
+- Cache repeated query embeddings in the eval provider and reuse one bounded
+  shortest-path search across all targets for a graph source.
 - Bootstrap confidence intervals: conversation-level percentile bootstrap for
   nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable
   runs). CIs appear in JSON reports and the markdown table.
diff --git a/src/mapmatched/eval/corpus.py b/src/mapmatched/eval/corpus.py
index c547136..08bc8ec 100644
--- a/src/mapmatched/eval/corpus.py
+++ b/src/mapmatched/eval/corpus.py
@@ -24,6 +24,7 @@ def __init__(
         self._passage_ids = ids
         self._passage_embeddings = tuple(tuple(values) for values in passage_embeddings)
         self._embed_query = embed_query
+        self._query_embedding_cache: dict[str, tuple[float, ...]] = {}
 
     @property
     def passage_count(self) -> int:
@@ -32,7 +33,10 @@ def passage_count(self) -> int:
     def candidates(self, query: str, limit: int) -> list[ScoredCandidate]:
         if limit <= 0:
             raise ValueError("limit must be greater than zero")
-        query_embedding = tuple(self._embed_query.embed_query(query))
+        query_embedding = self._query_embedding_cache.get(query)
+        if query_embedding is None:
+            query_embedding = tuple(self._embed_query.embed_query(query))
+            self._query_embedding_cache[query] = query_embedding
         scored = [
             (
                 dot_product(query_embedding, passage_embedding),
diff --git a/src/mapmatched/graph.py b/src/mapmatched/graph.py
index 805a0f9..b422d8d 100644
--- a/src/mapmatched/graph.py
+++ b/src/mapmatched/graph.py
@@ -96,9 +96,18 @@ def distance(self, source_chunk_id: str, target_chunk_id: str) -> float:
         if cached is not None:
             return cached
         distances = self._bounded_distances(source_chunk_id, self._maximum_distance)
-        result = min(distances.get(target_chunk_id, self._maximum_distance), self._maximum_distance)
-        self._distance_cache[cache_key] = result
-        return result
+        for chunk_id in self._adjacency:
+            distance = min(
+                distances.get(chunk_id, self._maximum_distance),
+                self._maximum_distance,
+            )
+            self._distance_cache[self._cache_key(source_chunk_id, chunk_id)] = distance
+        if cache_key not in self._distance_cache:
+            self._distance_cache[cache_key] = min(
+                distances.get(target_chunk_id, self._maximum_distance),
+                self._maximum_distance,
+            )
+        return self._distance_cache[cache_key]
 
     def neighborhood(self, chunk_id: str, radius: float) -> tuple[str, ...]:
         if not chunk_id:
diff --git a/tests/test_eval_baselines.py b/tests/test_eval_baselines.py
index 5a2d806..fe02700 100644
--- a/tests/test_eval_baselines.py
+++ b/tests/test_eval_baselines.py
@@ -9,6 +9,27 @@
 from mapmatched.eval.loaders.synthetic import load_synthetic_fixture
 
 
+def test_brute_force_provider_caches_query_embeddings() -> None:
+    class CountingEmbedder:
+        def __init__(self) -> None:
+            self.query_count = 0
+
+        def embed_query(self, query: str) -> tuple[float, float]:
+            del query
+            self.query_count += 1
+            return (1.0, 0.0)
+
+    embedder = CountingEmbedder()
+    provider = BruteForceProvider(
+        ("first", "second"),
+        ((1.0, 0.0), (0.0, 1.0)),
+        embedder,
+    )
+    provider.candidates("same query", limit=2)
+    provider.candidates("same query", limit=1)
+    assert embedder.query_count == 1
+
+
 def test_history_concat_changes_query_sequence() -> None:
     _, passages = load_synthetic_fixture()
     embedder = DeterministicHashEmbedder()
diff --git a/tests/test_graph_and_scoring.py b/tests/test_graph_and_scoring.py
index 07496d1..78a13a3 100644
--- a/tests/test_graph_and_scoring.py
+++ b/tests/test_graph_and_scoring.py
@@ -45,6 +45,28 @@ def test_directed_graph_and_shortest_path_cutoff() -> None:
     assert graph.distance("a", "c") == 1.5
 
 
+def test_distance_reuses_single_source_shortest_paths() -> None:
+    class CountingGraph(InMemoryCorpusGraph):
+        def __init__(self) -> None:
+            super().__init__(
+                [
+                    GraphEdge("a", "b", 1.0),
+                    GraphEdge("b", "c", 1.0),
+                ]
+            )
+            self.search_count = 0
+
+        def _bounded_distances(self, source: str, cutoff: float) -> dict[str, float]:
+            self.search_count += 1
+            return super()._bounded_distances(source, cutoff)
+
+    graph = CountingGraph()
+    assert graph.distance("a", "b") == 1.0
+    assert graph.distance("a", "c") == 2.0
+    assert graph.distance("c", "a") == 2.0
+    assert graph.search_count == 1
+
+
 @pytest.mark.parametrize(
     ("method", "expected"),
     [

From 3170598433cbbca94a4e174b4dd488fcabe518c1 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 16:05:29 +0000
Subject: [PATCH 06/12] Publish reproducible TopiOCQA paired-CI results

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 CHANGELOG.md                       |  3 +++
 README.md                          | 34 +++++++++++++++++-------------
 docs/eval.md                       | 15 +++++++++++++
 results/topiocqa_n25_minilm_knn.md | 32 ++++++++++++++++++++++++++++
 4 files changed, 69 insertions(+), 15 deletions(-)
 create mode 100644 results/topiocqa_n25_minilm_knn.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index af406b2..2801956 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,9 @@ All notable changes to this project are documented here.
   checksum, selected conversation IDs, model, graph, package, and git provenance.
 - Cache repeated query embeddings in the eval provider and reuse one bounded
   shortest-path search across all targets for a graph source.
+- Publish the reproducible TopiOCQA n=25 MiniLM/kNN micro-corpus result:
+  map-matched β=1.0 lifts follow-up nDCG@3 by `+0.084` with paired 95% CI
+  `[+0.046, +0.128]`; two runs produced byte-identical reports.
 - Bootstrap confidence intervals: conversation-level percentile bootstrap for
   nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable
   runs). CIs appear in JSON reports and the markdown table.
diff --git a/README.md b/README.md
index ebcb8bf..6561555 100644
--- a/README.md
+++ b/README.md
@@ -204,21 +204,25 @@ python -m mapmatched.eval \
 ```
 
 The synthetic benchmark is a deterministic smoke test, not research evidence.
-The current preliminary TopiOCQA micro-corpus result uses 25 conversations,
-MiniLM embeddings, a kNN graph, full candidate ranking, and no bootstrap CI:
-
-| Measurement | nDCG@3 |
-| --- | ---: |
-| Pointwise follow-up | 0.150 |
-| Map-matched follow-up | 0.234 |
-| Follow-up delta | +0.084 |
-| Standalone delta | +0.031 |
-
-These numbers demonstrate that the pipeline can produce measurable lift, but
-they are not a full-corpus or statistically conclusive benchmark. Structured
-section graphs also underperform on topic-switch-heavy TopiOCQA, an important
-negative result rather than a hidden one. See [`docs/eval.md`](docs/eval.md) for
-benchmark tiers, methodology, limitations, and reproduction commands.
+The pinned TopiOCQA micro-corpus profile uses 25 conversations, MiniLM
+embeddings, a 10-neighbor kNN graph, full candidate ranking, and 1,000 paired
+conversation-level bootstrap draws:
+
+| Slice | Method | nDCG@3 | Delta vs pointwise | Paired delta 95% CI |
+| --- | --- | ---: | ---: | ---: |
+| Follow-up | Pointwise | 0.150 | +0.000 | — |
+| Follow-up | Map-matched β=0.5 | 0.195 | +0.045 | [+0.018, +0.077] |
+| Follow-up | Map-matched β=1.0 | 0.234 | +0.084 | [+0.046, +0.128] |
+| Follow-up | MMR | 0.151 | +0.001 | [+0.000, +0.003] |
+| Standalone | Map-matched β=1.0 | 0.373 | +0.031 | [+0.009, +0.055] |
+
+Both runs produced byte-identical reports. The positive paired intervals are
+evidence for this fixed micro-corpus, not a full-Wikipedia or cross-benchmark
+claim. Structured section graphs also underperform on topic-switch-heavy
+TopiOCQA, an important negative result rather than a hidden one. See the
+[`committed result`](results/topiocqa_n25_minilm_knn.md) and
+[`evaluation guide`](docs/eval.md) for provenance, all baselines, limitations,
+and reproduction commands.
 
 Reproduce the pinned n=25 MiniLM/kNN profile after downloading the validation
 split:
diff --git a/docs/eval.md b/docs/eval.md
index 94576ec..8788512 100644
--- a/docs/eval.md
+++ b/docs/eval.md
@@ -83,6 +83,21 @@ follow-up side. It is licensed
 dataset is not redistributed by this repository. These runs are micro-corpus
 experiments, not full-Wikipedia retrieval.
 
+The pinned run at git revision `60a9d694b807c3eb49da2a00743c6f1a05d52e6d`
+produced byte-identical reports twice:
+
+| Slice | Method | nDCG@3 | Delta vs pointwise | Paired delta 95% CI |
+| --- | --- | ---: | ---: | ---: |
+| Follow-up | Map-matched β=0.5 | 0.195 | +0.045 | [+0.018, +0.077] |
+| Follow-up | Map-matched β=1.0 | 0.234 | +0.084 | [+0.046, +0.128] |
+| Follow-up | MMR | 0.151 | +0.001 | [+0.000, +0.003] |
+| Standalone | Map-matched β=1.0 | 0.373 | +0.031 | [+0.009, +0.055] |
+
+The complete aggregate table and provenance are committed in
+[`results/topiocqa_n25_minilm_knn.md`](../results/topiocqa_n25_minilm_knn.md).
+Positive intervals support the claim on this fixed 25-conversation micro-corpus;
+they do not establish full-corpus or cross-benchmark generalization.
+
 ### TREC CAsT 2019 (micro)
 
 ```console
diff --git a/results/topiocqa_n25_minilm_knn.md b/results/topiocqa_n25_minilm_knn.md
new file mode 100644
index 0000000..693619e
--- /dev/null
+++ b/results/topiocqa_n25_minilm_knn.md
@@ -0,0 +1,32 @@
+# TopiOCQA n=25 MiniLM/kNN result
+
+Profile: `topiocqa-n25-minilm-knn-full`
+
+- Dataset: `topiocqa_valid.jsonl`
+- Dataset SHA-256: `1bba9512b24b2e5de22704766dd80b1dc497bb7262799f66c3912e6f413ac1c6`
+- Conversations: 25
+- Embedder: `sentence-transformers/all-MiniLM-L6-v2`
+- Graph: 10-neighbor kNN
+- Ranking: full, candidate limit 100
+- Bootstrap: 1,000 conversation-level draws, seed 42
+- Package: `map-matched-retrieval==0.1.0`
+- Git revision: `60a9d694b807c3eb49da2a00743c6f1a05d52e6d`
+- Full JSON report SHA-256: `35a1801b2d7502a7bc1036f90b53818783f56ca9f2ffeebdfad66fc41056ce9e`
+
+| Slice | Method | β | nDCG@3 | nDCG@3 95% CI | Delta vs pointwise | Paired delta 95% CI |
+| --- | --- | ---: | ---: | ---: | ---: | ---: |
+| Follow-up | Pointwise | 0.00 | 0.150 | [0.109, 0.196] | +0.000 | — |
+| Follow-up | Map-matched | 0.50 | 0.195 | [0.147, 0.249] | +0.045 | [+0.018, +0.077] |
+| Follow-up | Map-matched | 1.00 | 0.234 | [0.186, 0.294] | +0.084 | [+0.046, +0.128] |
+| Follow-up | History concat | — | 0.071 | [0.043, 0.099] | -0.079 | [-0.137, -0.027] |
+| Follow-up | MMR | — | 0.151 | [0.109, 0.197] | +0.001 | [+0.000, +0.003] |
+| Standalone | Pointwise | 0.00 | 0.342 | [0.282, 0.405] | +0.000 | — |
+| Standalone | Map-matched | 0.50 | 0.359 | [0.295, 0.424] | +0.017 | [+0.004, +0.034] |
+| Standalone | Map-matched | 1.00 | 0.373 | [0.305, 0.440] | +0.031 | [+0.009, +0.055] |
+| Standalone | History concat | — | 0.134 | [0.110, 0.159] | -0.208 | [-0.261, -0.158] |
+| Standalone | MMR | — | 0.340 | [0.280, 0.404] | -0.002 | [-0.004, +0.000] |
+
+Both repeated runs produced byte-identical JSON and markdown reports. This is a
+gold-passage micro-corpus result, not full-Wikipedia retrieval. The paired
+intervals quantify uncertainty for these 25 conversations and should not be
+generalized to other corpora without additional evaluation.

From 20f518596dd5c8797f29fe443d39f1ef7f2a4edd Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:04:34 +0000
Subject: [PATCH 07/12] Retrigger CI after billing recovery

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>

From b1de30eac502297c50ca8c068c1711be5ddd4870 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:31:04 +0000
Subject: [PATCH 08/12] Add Gemini query rewrite baseline for CAsT

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 CHANGELOG.md                              |   3 +
 README.md                                 |   7 +-
 docs/eval.md                              |  18 ++-
 pyproject.toml                            |   3 +
 scripts/reproduce_cast2019_gemini.sh      |  40 +++++++
 src/mapmatched/eval/__main__.py           |  28 ++++-
 src/mapmatched/eval/ablations.py          |   9 +-
 src/mapmatched/eval/baselines/__init__.py |  17 +++
 src/mapmatched/eval/baselines/gemini.py   |  85 ++++++++++++++
 src/mapmatched/eval/baselines/rewrite.py  |  28 +++++
 src/mapmatched/eval/runner.py             |  21 +++-
 src/mapmatched/eval/types.py              |   6 +
 tests/test_eval_gemini_rewrite.py         | 128 ++++++++++++++++++++++
 13 files changed, 385 insertions(+), 8 deletions(-)
 create mode 100755 scripts/reproduce_cast2019_gemini.sh
 create mode 100644 src/mapmatched/eval/baselines/gemini.py
 create mode 100644 src/mapmatched/eval/baselines/rewrite.py
 create mode 100644 tests/test_eval_gemini_rewrite.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2801956..4a18045 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@ All notable changes to this project are documented here.
 
 ## Unreleased
 
+- Add an opt-in Gemini conversational query rewrite baseline for TREC CAsT,
+  configured by `GEMINI_API_KEY`, with paired pointwise comparisons and
+  model/prompt provenance in evaluation reports.
 - Add paired conversation-level bootstrap intervals for method nDCG@3 deltas,
   preserve conversation IDs in reports, and fix the aggregate `all`-slice
   bootstrap interval.
diff --git a/README.md b/README.md
index 6561555..7d09a1c 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,7 @@ python -m pip install -e ".[st]"
 | `graph` | Embedding-derived kNN graphs |
 | `faiss` | kNN graphs and the FAISS candidate provider |
 | `eval` | Benchmark loaders, baselines, metrics, and reports |
+| `gemini` | Gemini conversational query rewrite baseline |
 | `st` | Sentence-transformer embeddings for evaluation |
 
 ## Quickstart
@@ -193,9 +194,9 @@ context are separate outputs.
 
 The evaluation harness reports nDCG@3/5 and Recall@k separately for ambiguous
 follow-up turns and sharp standalone turns. It includes pointwise, history
-concatenation, Maximal Marginal Relevance, and resolved-query baselines, plus
-conversation-level percentile bootstrap confidence intervals. Method deltas use
-paired resampling of the same conversations.
+concatenation, optional Gemini query rewriting, Maximal Marginal Relevance, and
+resolved-query baselines, plus conversation-level percentile bootstrap confidence
+intervals. Method deltas use paired resampling of the same conversations.
 
 ```console
 python -m mapmatched.eval \
diff --git a/docs/eval.md b/docs/eval.md
index 8788512..9a62742 100644
--- a/docs/eval.md
+++ b/docs/eval.md
@@ -13,7 +13,8 @@ python -m pip install -e ".[eval,graph,st]"
 
 The harness uses NumPy for kNN graph construction and optional Hugging Face /
 ir-datasets loaders for benchmark metadata. It does **not** download embedding
-models.
+models. Install `.[eval,graph,st,gemini]` only when running the Gemini rewrite
+baseline.
 
 ## Tiers
 
@@ -55,6 +56,7 @@ best map-matched configuration against the β=0 pointwise baseline:
 | `pointwise` | `transition_weight=0` — independent per-turn top-1 |
 | `mapmatched` | Full trajectory decoder with configurable β |
 | `history_concat` | Dense retrieval over concatenated query history |
+| `gemini_rewrite` | Gemini rewrites each turn into a standalone query before dense retrieval |
 | `maximal_marginal_relevance` | Per-turn MMR re-ranking (not map-matched retrieval) |
 | `resolved_oracle` | CAsT resolved utterances (upper bound) |
 
@@ -101,7 +103,9 @@ they do not establish full-corpus or cross-benchmark generalization.
 ### TREC CAsT 2019 (micro)
 
 ```console
-python -m mapmatched.eval --benchmark cast2019 --embedder sentence-transformers
+export GEMINI_API_KEY="..."
+python -m pip install -e ".[eval,graph,st,gemini]"
+./scripts/reproduce_cast2019_gemini.sh
 ```
 
 Uses ir-datasets id `trec-cast/v1/2019/judged`. Real passage text comes from the
@@ -111,6 +115,16 @@ Without the collection the loader degrades to using doc ids as passage text
 (metrics not meaningful). CAsT's drill-down follow-ups are the fairer test for
 the follow-up-lift claim than TopiOCQA's topic switches.
 
+The script adds `gemini_rewrite` to the normal ablation grid. It sends each raw
+utterance and its prior user utterances to `gemini-2.5-flash` with temperature
+zero, retrieves with the returned standalone query, and compares it with both
+pointwise retrieval and CAsT's manual `resolved_oracle`. `GEMINI_API_KEY` is read
+from the environment and is never written to reports. Reports record the Gemini
+model and prompt version. The baseline is opt-in because it makes one paid,
+networked model request per selected turn; `--conversation-limit` bounds those
+requests. Temperature zero does not make hosted-model output immutable across
+model revisions.
+
 ## Graph source and ranking mode
 
 - `--graph-source knn` (default) builds the embedding-kNN fallback graph;
diff --git a/pyproject.toml b/pyproject.toml
index 244c226..3340e8b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,6 +36,9 @@ eval = [
   "numpy>=2.2.6; python_version < '3.11'",
   "numpy>=2.5.1; python_version >= '3.11'",
 ]
+gemini = [
+  "google-genai>=2.11.0",
+]
 st = [
   "sentence-transformers>=3.0",
 ]
diff --git a/scripts/reproduce_cast2019_gemini.sh b/scripts/reproduce_cast2019_gemini.sh
new file mode 100755
index 0000000..a6ba789
--- /dev/null
+++ b/scripts/reproduce_cast2019_gemini.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PROFILE="cast2019-gemini25flash-minilm-knn-full"
+REPORT_DIR="${MAPMATCHED_REPORT_DIR:-$ROOT_DIR/reports/$PROFILE}"
+GEMINI_MODEL="${MAPMATCHED_GEMINI_MODEL:-gemini-2.5-flash}"
+
+if [[ -z "${GEMINI_API_KEY:-}" ]]; then
+  echo "GEMINI_API_KEY must be set" >&2
+  exit 2
+fi
+
+mkdir -p "$REPORT_DIR"
+
+export CUDA_VISIBLE_DEVICES=""
+export PYTHONHASHSEED=0
+export TOKENIZERS_PARALLELISM=false
+
+python3 -m mapmatched.eval \
+  --profile "$PROFILE" \
+  --benchmark cast2019 \
+  --tier micro \
+  --conversation-limit 50 \
+  --embedder sentence-transformers \
+  --st-model sentence-transformers/all-MiniLM-L6-v2 \
+  --graph-source knn \
+  --knn-neighbors 10 \
+  --ranking-mode full \
+  --candidate-limit 100 \
+  --recall-k 100 \
+  --bootstrap-samples 1000 \
+  --bootstrap-seed 42 \
+  --include-gemini-rewrite \
+  --gemini-model "$GEMINI_MODEL" \
+  --output "$REPORT_DIR/report.json" \
+  --markdown-output "$REPORT_DIR/report.md"
+
+echo "JSON report: $REPORT_DIR/report.json"
+echo "Markdown report: $REPORT_DIR/report.md"
diff --git a/src/mapmatched/eval/__main__.py b/src/mapmatched/eval/__main__.py
index fe9ba3b..4ae5699 100644
--- a/src/mapmatched/eval/__main__.py
+++ b/src/mapmatched/eval/__main__.py
@@ -9,6 +9,12 @@
 from mapmatched import __version__
 
 from .ablations import run_ablation_grid
+from .baselines import (
+    DEFAULT_GEMINI_MODEL,
+    GEMINI_REWRITE_PROMPT_VERSION,
+    ConversationQueryRewriter,
+    create_gemini_query_rewriter,
+)
 from .embedder import DeterministicHashEmbedder, SentenceTransformerEmbedder
 from .loaders import load_cast2019_micro, load_synthetic_fixture, load_topiocqa_micro
 from .report import render_json, render_markdown_table
@@ -88,6 +94,16 @@ def build_parser() -> argparse.ArgumentParser:
         help="Stable run profile name stored in report metadata.",
     )
     parser.add_argument("--include-resolved-oracle", action="store_true")
+    parser.add_argument(
+        "--include-gemini-rewrite",
+        action="store_true",
+        help="Evaluate a Gemini conversational query rewrite baseline.",
+    )
+    parser.add_argument(
+        "--gemini-model",
+        default=DEFAULT_GEMINI_MODEL,
+        help="Gemini model used by --include-gemini-rewrite.",
+    )
     return parser
 
 
@@ -106,13 +122,16 @@ def load_benchmark(
         )
     if name == "cast2019":
         conversations, passages = load_cast2019_micro()
-        return conversations, passages
+        return conversations[:conversation_limit], passages
     raise ValueError(f"unsupported benchmark: {name}")
 
 
 def main(argv: list[str] | None = None) -> int:
     parser = build_parser()
     args = parser.parse_args(argv)
+    query_rewriter: ConversationQueryRewriter | None = None
+    if args.include_gemini_rewrite:
+        query_rewriter = create_gemini_query_rewriter(model=args.gemini_model)
     data_path = _effective_data_path(args.benchmark, args.data_path)
     conversations, passages = load_benchmark(
         args.benchmark,
@@ -146,6 +165,11 @@ def main(argv: list[str] | None = None) -> int:
         else embedder.name,
         package_version=__version__,
         git_revision=_git_revision(),
+        query_rewrite_provider="gemini" if args.include_gemini_rewrite else None,
+        query_rewrite_model=args.gemini_model if args.include_gemini_rewrite else None,
+        query_rewrite_prompt_version=GEMINI_REWRITE_PROMPT_VERSION
+        if args.include_gemini_rewrite
+        else None,
     )
     report = run_ablation_grid(
         conversations=conversations,
@@ -153,7 +177,9 @@ def main(argv: list[str] | None = None) -> int:
         embedder=embedder,
         eval_config=eval_config,
         candidate_limit=args.candidate_limit,
+        include_gemini_rewrite=args.include_gemini_rewrite,
         include_resolved_oracle=args.include_resolved_oracle or args.benchmark == "cast2019",
+        query_rewriter=query_rewriter,
     )
     args.output.write_text(render_json(report), encoding="utf-8")
     markdown = render_markdown_table(report)
diff --git a/src/mapmatched/eval/ablations.py b/src/mapmatched/eval/ablations.py
index d67c8d7..775cfda 100644
--- a/src/mapmatched/eval/ablations.py
+++ b/src/mapmatched/eval/ablations.py
@@ -2,7 +2,7 @@
 
 from collections.abc import Sequence
 
-from .baselines import MapMatchedMethodConfig
+from .baselines import ConversationQueryRewriter, MapMatchedMethodConfig
 from .embedder import TextEmbedder
 from .runner import MethodSpec, run_eval
 from .types import EvalConfig, EvalConversation, EvalReport, Passage
@@ -11,6 +11,7 @@
 def default_method_grid(
     *,
     transition_weights: Sequence[float] = (0.0, 0.5, 1.0),
+    include_gemini_rewrite: bool = False,
     include_mmr: bool = True,
     include_resolved_oracle: bool = False,
 ) -> tuple[MethodSpec, ...]:
@@ -22,6 +23,8 @@ def default_method_grid(
             continue
         methods.append(MethodSpec(name="mapmatched", transition_weight=beta))
     methods.append(MethodSpec(name="history_concat"))
+    if include_gemini_rewrite:
+        methods.append(MethodSpec(name="gemini_rewrite"))
     if include_mmr:
         methods.append(MethodSpec(name="maximal_marginal_relevance"))
     if include_resolved_oracle:
@@ -38,11 +41,14 @@ def run_ablation_grid(
     transition_weights: Sequence[float] = (0.0, 0.5, 1.0),
     candidate_limit: int = 20,
     fixed_lag: int | None = None,
+    include_gemini_rewrite: bool = False,
     include_mmr: bool = True,
     include_resolved_oracle: bool = False,
+    query_rewriter: ConversationQueryRewriter | None = None,
 ) -> EvalReport:
     methods = default_method_grid(
         transition_weights=transition_weights,
+        include_gemini_rewrite=include_gemini_rewrite,
         include_mmr=include_mmr,
         include_resolved_oracle=include_resolved_oracle,
     )
@@ -57,4 +63,5 @@ def run_ablation_grid(
         methods=methods,
         config=config,
         eval_config=eval_config,
+        query_rewriter=query_rewriter,
     )
diff --git a/src/mapmatched/eval/baselines/__init__.py b/src/mapmatched/eval/baselines/__init__.py
index f1357c3..2f58a49 100644
--- a/src/mapmatched/eval/baselines/__init__.py
+++ b/src/mapmatched/eval/baselines/__init__.py
@@ -1,5 +1,13 @@
 from __future__ import annotations
 
+from .gemini import (
+    DEFAULT_GEMINI_MODEL,
+    GEMINI_REWRITE_PROMPT_VERSION,
+    GeminiDependencyUnavailableError,
+    GeminiQueryRewriter,
+    GeminiRewriteConfig,
+    create_gemini_query_rewriter,
+)
 from .methods import (
     MapMatchedMethodConfig,
     run_history_concat_conversation,
@@ -8,9 +16,18 @@
     run_pointwise_conversation,
     run_resolved_oracle_conversation,
 )
+from .rewrite import ConversationQueryRewriter, rewrite_conversation_queries
 
 __all__ = [
+    "DEFAULT_GEMINI_MODEL",
+    "GEMINI_REWRITE_PROMPT_VERSION",
+    "ConversationQueryRewriter",
+    "GeminiDependencyUnavailableError",
+    "GeminiQueryRewriter",
+    "GeminiRewriteConfig",
     "MapMatchedMethodConfig",
+    "create_gemini_query_rewriter",
+    "rewrite_conversation_queries",
     "run_history_concat_conversation",
     "run_mapmatched_conversation",
     "run_maximal_marginal_relevance_conversation",
diff --git a/src/mapmatched/eval/baselines/gemini.py b/src/mapmatched/eval/baselines/gemini.py
new file mode 100644
index 0000000..b2e77ac
--- /dev/null
+++ b/src/mapmatched/eval/baselines/gemini.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import importlib
+import json
+import os
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+
+DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"
+GEMINI_REWRITE_PROMPT_VERSION = "cast-standalone-v1"
+
+_SYSTEM_INSTRUCTION = """\
+Rewrite the current conversational search utterance as one concise, standalone search query.
+Resolve references and omitted context using only the prior user utterances.
+Preserve the current information need and do not answer it.
+Return only the rewritten query with no explanation, label, quotation marks, or markdown."""
+
+
+class GeminiDependencyUnavailableError(ImportError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class GeminiRewriteConfig:
+    model: str = DEFAULT_GEMINI_MODEL
+    temperature: float = 0.0
+
+
+class GeminiQueryRewriter:
+    def __init__(
+        self,
+        *,
+        generate_content: Callable[..., object],
+        config: GeminiRewriteConfig,
+    ) -> None:
+        self._generate_content = generate_content
+        self.config = config
+
+    def rewrite(self, *, history: Sequence[str], query: str) -> str:
+        request = {
+            "prior_user_utterances": list(history),
+            "current_utterance": query,
+        }
+        response = self._generate_content(
+            model=self.config.model,
+            contents=f"{_SYSTEM_INSTRUCTION}\n\nConversation:\n{json.dumps(request, ensure_ascii=False)}",
+            config={"temperature": self.config.temperature},
+        )
+        response_text = getattr(response, "text", None)
+        if not isinstance(response_text, str) or not response_text.strip():
+            raise RuntimeError("Gemini returned no rewritten query")
+        return response_text.strip()
+
+
+def create_gemini_query_rewriter(
+    *,
+    api_key: str | None = None,
+    model: str = DEFAULT_GEMINI_MODEL,
+) -> GeminiQueryRewriter:
+    effective_api_key = api_key if api_key is not None else os.environ.get("GEMINI_API_KEY")
+    if not effective_api_key:
+        raise ValueError(
+            "GEMINI_API_KEY must be set when the Gemini rewrite baseline is enabled"
+        )
+    try:
+        genai = importlib.import_module("google.genai")
+    except ImportError as error:
+        raise GeminiDependencyUnavailableError(
+            "Gemini rewrite requires the 'gemini' extra: "
+            "pip install 'map-matched-retrieval[eval,gemini]'"
+        ) from error
+    client_factory = getattr(genai, "Client", None)
+    if not callable(client_factory):
+        raise GeminiDependencyUnavailableError("google.genai.Client is unavailable")
+    client = client_factory(api_key=effective_api_key)
+    models = getattr(client, "models", None)
+    generate_content = getattr(models, "generate_content", None)
+    if not callable(generate_content):
+        raise GeminiDependencyUnavailableError(
+            "google.genai Client does not provide models.generate_content"
+        )
+    return GeminiQueryRewriter(
+        generate_content=generate_content,
+        config=GeminiRewriteConfig(model=model),
+    )
diff --git a/src/mapmatched/eval/baselines/rewrite.py b/src/mapmatched/eval/baselines/rewrite.py
new file mode 100644
index 0000000..838f8ec
--- /dev/null
+++ b/src/mapmatched/eval/baselines/rewrite.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Protocol
+
+from ..types import EvalConversation
+
+
+class ConversationQueryRewriter(Protocol):
+    def rewrite(self, *, history: Sequence[str], query: str) -> str: ...
+
+
+def rewrite_conversation_queries(
+    conversation: EvalConversation,
+    rewriter: ConversationQueryRewriter,
+) -> tuple[str, ...]:
+    history: list[str] = []
+    rewritten_queries: list[str] = []
+    for turn in conversation.turns:
+        rewritten_query = rewriter.rewrite(history=tuple(history), query=turn.query).strip()
+        if not rewritten_query:
+            raise RuntimeError(
+                f"query rewriter returned an empty query for "
+                f"{conversation.conversation_id} turn {turn.turn_index}"
+            )
+        rewritten_queries.append(rewritten_query)
+        history.append(turn.query)
+    return tuple(rewritten_queries)
diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py
index 01d0b40..a6ac70f 100644
--- a/src/mapmatched/eval/runner.py
+++ b/src/mapmatched/eval/runner.py
@@ -4,7 +4,12 @@
 
 from mapmatched import CorpusGraph
 
-from .baselines import MapMatchedMethodConfig, run_mapmatched_conversation
+from .baselines import (
+    ConversationQueryRewriter,
+    MapMatchedMethodConfig,
+    rewrite_conversation_queries,
+    run_mapmatched_conversation,
+)
 from .baselines.methods import run_maximal_marginal_relevance_conversation
 from .bootstrap import bootstrap_paired_slice_delta_cis, bootstrap_slice_cis
 from .corpus import (
@@ -61,6 +66,7 @@ def run_method_on_conversation(
     method: MethodSpec,
     config: MapMatchedMethodConfig,
     ranking_mode: str = "full",
+    query_rewriter: ConversationQueryRewriter | None = None,
 ) -> tuple[tuple[tuple[str, ...], ...], tuple[float | None, ...]]:
     queries = [turn.query for turn in conversation.turns]
     method_config = MapMatchedMethodConfig(
@@ -100,6 +106,14 @@ def run_method_on_conversation(
             rankings.append(rank_full_corpus(provider, rewritten))
             history.append(query)
         return tuple(rankings), tuple(None for _ in queries)
+    if method.name == "gemini_rewrite":
+        if query_rewriter is None:
+            raise ValueError("gemini_rewrite requires a conversation query rewriter")
+        rewritten_queries = rewrite_conversation_queries(conversation, query_rewriter)
+        return tuple(
+            rank_full_corpus(provider, rewritten_query)
+            for rewritten_query in rewritten_queries
+        ), tuple(None for _ in queries)
     if method.name == "resolved_oracle":
         oracle_queries = tuple(
             turn.resolved_query if turn.resolved_query is not None else turn.query
@@ -178,6 +192,7 @@ def evaluate_method(
     trace_entropies: Sequence[Sequence[float | None]],
     entropy_threshold: float,
     graph_mode: str,
+    query_rewriter: ConversationQueryRewriter | None = None,
 ) -> MethodMetrics:
     conversation_turns: list[list[TurnMetrics]] = []
     for conversation, trace_entropies_for_conversation in zip(
@@ -192,6 +207,7 @@ def evaluate_method(
             method=method,
             config=config,
             ranking_mode=eval_config.ranking_mode,
+            query_rewriter=query_rewriter,
         )
         per_conversation: list[TurnMetrics] = []
         for turn, ranking, trace_entropy in zip(
@@ -278,6 +294,7 @@ def run_eval(
     methods: Sequence[MethodSpec],
     config: MapMatchedMethodConfig,
     eval_config: EvalConfig,
+    query_rewriter: ConversationQueryRewriter | None = None,
 ) -> EvalReport:
     graph_mode = eval_config.graph_source
     provider, graph = _build_provider_and_graph(
@@ -315,6 +332,7 @@ def run_eval(
             trace_entropies=trace_entropies,
             entropy_threshold=entropy_threshold,
             graph_mode=graph_mode,
+            query_rewriter=query_rewriter,
         )
         for method in methods
     )
@@ -353,6 +371,7 @@ def _build_method_comparisons(
 ) -> tuple[MethodComparison, ...]:
     comparisons: list[MethodComparison] = []
     comparable_method_names = {
+        "gemini_rewrite",
         "history_concat",
         "mapmatched",
         "maximal_marginal_relevance",
diff --git a/src/mapmatched/eval/types.py b/src/mapmatched/eval/types.py
index 2953ffa..a74bed3 100644
--- a/src/mapmatched/eval/types.py
+++ b/src/mapmatched/eval/types.py
@@ -129,6 +129,9 @@ class EvalConfig:
     embedding_model: str | None = None
     package_version: str | None = None
     git_revision: str | None = None
+    query_rewrite_provider: str | None = None
+    query_rewrite_model: str | None = None
+    query_rewrite_prompt_version: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -151,6 +154,9 @@ def to_dict(self) -> dict[str, object]:
             "embedding_model": self.embedding_model,
             "package_version": self.package_version,
             "git_revision": self.git_revision,
+            "query_rewrite_provider": self.query_rewrite_provider,
+            "query_rewrite_model": self.query_rewrite_model,
+            "query_rewrite_prompt_version": self.query_rewrite_prompt_version,
         }
 
 
diff --git a/tests/test_eval_gemini_rewrite.py b/tests/test_eval_gemini_rewrite.py
new file mode 100644
index 0000000..b0feaa2
--- /dev/null
+++ b/tests/test_eval_gemini_rewrite.py
@@ -0,0 +1,128 @@
+import json
+from collections.abc import Sequence
+from pathlib import Path
+
+import pytest
+
+from mapmatched.eval import DeterministicHashEmbedder, EvalConfig, MethodSpec, run_eval
+from mapmatched.eval.__main__ import main
+from mapmatched.eval.baselines import (
+    GeminiQueryRewriter,
+    GeminiRewriteConfig,
+    MapMatchedMethodConfig,
+    create_gemini_query_rewriter,
+)
+from mapmatched.eval.loaders.synthetic import load_synthetic_fixture
+
+
+class RecordingQueryRewriter:
+    def __init__(self) -> None:
+        self.calls: list[tuple[tuple[str, ...], str]] = []
+
+    def rewrite(self, *, history: Sequence[str], query: str) -> str:
+        self.calls.append((tuple(history), query))
+        return " ".join([*history, query])
+
+
+def test_gemini_query_rewriter_sends_history_and_returns_text() -> None:
+    requests: list[dict[str, object]] = []
+
+    class Response:
+        text = "  standalone query  "
+
+    def generate_content(**request: object) -> object:
+        requests.append(request)
+        return Response()
+
+    rewriter = GeminiQueryRewriter(
+        generate_content=generate_content,
+        config=GeminiRewriteConfig(model="test-model"),
+    )
+    rewritten = rewriter.rewrite(history=("first question",), query="what about it?")
+
+    assert rewritten == "standalone query"
+    assert requests[0]["model"] == "test-model"
+    assert requests[0]["config"] == {"temperature": 0.0}
+    assert '"prior_user_utterances": ["first question"]' in str(requests[0]["contents"])
+
+
+def test_create_gemini_query_rewriter_requires_api_key(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+    with pytest.raises(ValueError, match="GEMINI_API_KEY"):
+        create_gemini_query_rewriter()
+
+
+def test_gemini_rewrite_runs_end_to_end_with_paired_comparison() -> None:
+    conversations, passages = load_synthetic_fixture()
+    embedder = DeterministicHashEmbedder()
+    rewriter = RecordingQueryRewriter()
+    report = run_eval(
+        conversations=conversations,
+        passages=passages,
+        embedder=embedder,
+        methods=(
+            MethodSpec(name="pointwise", transition_weight=0.0),
+            MethodSpec(name="gemini_rewrite"),
+        ),
+        config=MapMatchedMethodConfig(candidate_limit=4),
+        eval_config=EvalConfig(
+            benchmark="synthetic",
+            tier="synthetic",
+            embedder_name=embedder.name,
+            recall_k=10,
+            entropy_threshold=None,
+            standalone_tolerance=0.02,
+            follow_up_min_delta=0.0,
+            bootstrap_samples=20,
+            bootstrap_seed=42,
+        ),
+        query_rewriter=rewriter,
+    )
+
+    expected_turn_count = sum(len(conversation.turns) for conversation in conversations)
+    assert len(rewriter.calls) == expected_turn_count
+    assert rewriter.calls[0][0] == ()
+    assert rewriter.calls[1][0] == (conversations[0].turns[0].query,)
+    comparison = next(
+        comparison
+        for comparison in report.comparisons
+        if comparison.method_name == "gemini_rewrite"
+    )
+    assert all(slice_metrics.ndcg_at_3_delta_ci is not None for slice_metrics in comparison.slices)
+
+
+def test_cli_records_gemini_rewrite_metadata(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    rewriter = RecordingQueryRewriter()
+    monkeypatch.setattr(
+        "mapmatched.eval.__main__.create_gemini_query_rewriter",
+        lambda *, model: rewriter,
+    )
+    output_path = tmp_path / "report.json"
+
+    exit_code = main(
+        [
+            "--benchmark",
+            "synthetic",
+            "--tier",
+            "synthetic",
+            "--include-gemini-rewrite",
+            "--gemini-model",
+            "test-model",
+            "--candidate-limit",
+            "4",
+            "--output",
+            str(output_path),
+        ]
+    )
+
+    payload = json.loads(output_path.read_text(encoding="utf-8"))
+    assert exit_code == 0
+    assert payload["config"]["query_rewrite_provider"] == "gemini"
+    assert payload["config"]["query_rewrite_model"] == "test-model"
+    assert payload["config"]["query_rewrite_prompt_version"] == "cast-standalone-v1"
+    assert any(method["method_name"] == "gemini_rewrite" for method in payload["methods"])

From a5ec616f093235c2037001e08d4e2f456aa79502 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:41:24 +0000
Subject: [PATCH 09/12] Update Gemini rewrite for current Flash model

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 docs/eval.md                            |  7 +++----
 scripts/reproduce_cast2019_gemini.sh    |  4 ++--
 src/mapmatched/eval/baselines/gemini.py | 19 ++++++++++++-------
 src/mapmatched/eval/runner.py           |  3 +--
 tests/test_eval_gemini_rewrite.py       |  4 +++-
 5 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/docs/eval.md b/docs/eval.md
index 9a62742..7af2620 100644
--- a/docs/eval.md
+++ b/docs/eval.md
@@ -116,14 +116,13 @@ Without the collection the loader degrades to using doc ids as passage text
 the follow-up-lift claim than TopiOCQA's topic switches.
 
 The script adds `gemini_rewrite` to the normal ablation grid. It sends each raw
-utterance and its prior user utterances to `gemini-2.5-flash` with temperature
-zero, retrieves with the returned standalone query, and compares it with both
+utterance and its prior user utterances to `gemini-3.5-flash` with minimal
+thinking, retrieves with the returned standalone query, and compares it with both
 pointwise retrieval and CAsT's manual `resolved_oracle`. `GEMINI_API_KEY` is read
 from the environment and is never written to reports. Reports record the Gemini
 model and prompt version. The baseline is opt-in because it makes one paid,
 networked model request per selected turn; `--conversation-limit` bounds those
-requests. Temperature zero does not make hosted-model output immutable across
-model revisions.
+requests. Hosted-model output is not immutable across model revisions.
 
 ## Graph source and ranking mode
 
diff --git a/scripts/reproduce_cast2019_gemini.sh b/scripts/reproduce_cast2019_gemini.sh
index a6ba789..9e3d633 100755
--- a/scripts/reproduce_cast2019_gemini.sh
+++ b/scripts/reproduce_cast2019_gemini.sh
@@ -2,9 +2,9 @@
 set -euo pipefail
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-PROFILE="cast2019-gemini25flash-minilm-knn-full"
+PROFILE="cast2019-gemini35flash-minilm-knn-full"
 REPORT_DIR="${MAPMATCHED_REPORT_DIR:-$ROOT_DIR/reports/$PROFILE}"
-GEMINI_MODEL="${MAPMATCHED_GEMINI_MODEL:-gemini-2.5-flash}"
+GEMINI_MODEL="${MAPMATCHED_GEMINI_MODEL:-gemini-3.5-flash}"
 
 if [[ -z "${GEMINI_API_KEY:-}" ]]; then
   echo "GEMINI_API_KEY must be set" >&2
diff --git a/src/mapmatched/eval/baselines/gemini.py b/src/mapmatched/eval/baselines/gemini.py
index b2e77ac..20ddef8 100644
--- a/src/mapmatched/eval/baselines/gemini.py
+++ b/src/mapmatched/eval/baselines/gemini.py
@@ -5,8 +5,9 @@
 import os
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass
+from typing import Literal
 
-DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"
+DEFAULT_GEMINI_MODEL = "gemini-3.5-flash"
 GEMINI_REWRITE_PROMPT_VERSION = "cast-standalone-v1"
 
 _SYSTEM_INSTRUCTION = """\
@@ -23,7 +24,7 @@ class GeminiDependencyUnavailableError(ImportError):
 @dataclass(frozen=True, slots=True)
 class GeminiRewriteConfig:
     model: str = DEFAULT_GEMINI_MODEL
-    temperature: float = 0.0
+    thinking_level: Literal["minimal", "low", "medium", "high"] = "minimal"
 
 
 class GeminiQueryRewriter:
@@ -32,7 +33,9 @@ def __init__(
         *,
         generate_content: Callable[..., object],
         config: GeminiRewriteConfig,
+        client: object | None = None,
     ) -> None:
+        self._client = client
         self._generate_content = generate_content
         self.config = config
 
@@ -41,10 +44,13 @@ def rewrite(self, *, history: Sequence[str], query: str) -> str:
             "prior_user_utterances": list(history),
             "current_utterance": query,
         }
+        prompt = (
+            f"{_SYSTEM_INSTRUCTION}\n\nConversation:\n{json.dumps(request, ensure_ascii=False)}"
+        )
         response = self._generate_content(
             model=self.config.model,
-            contents=f"{_SYSTEM_INSTRUCTION}\n\nConversation:\n{json.dumps(request, ensure_ascii=False)}",
-            config={"temperature": self.config.temperature},
+            contents=prompt,
+            config={"thinking_config": {"thinking_level": self.config.thinking_level}},
         )
         response_text = getattr(response, "text", None)
         if not isinstance(response_text, str) or not response_text.strip():
@@ -59,9 +65,7 @@ def create_gemini_query_rewriter(
 ) -> GeminiQueryRewriter:
     effective_api_key = api_key if api_key is not None else os.environ.get("GEMINI_API_KEY")
     if not effective_api_key:
-        raise ValueError(
-            "GEMINI_API_KEY must be set when the Gemini rewrite baseline is enabled"
-        )
+        raise ValueError("GEMINI_API_KEY must be set when the Gemini rewrite baseline is enabled")
     try:
         genai = importlib.import_module("google.genai")
     except ImportError as error:
@@ -82,4 +86,5 @@ def create_gemini_query_rewriter(
     return GeminiQueryRewriter(
         generate_content=generate_content,
         config=GeminiRewriteConfig(model=model),
+        client=client,
     )
diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py
index a6ac70f..8e49e75 100644
--- a/src/mapmatched/eval/runner.py
+++ b/src/mapmatched/eval/runner.py
@@ -111,8 +111,7 @@ def run_method_on_conversation(
             raise ValueError("gemini_rewrite requires a conversation query rewriter")
         rewritten_queries = rewrite_conversation_queries(conversation, query_rewriter)
         return tuple(
-            rank_full_corpus(provider, rewritten_query)
-            for rewritten_query in rewritten_queries
+            rank_full_corpus(provider, rewritten_query) for rewritten_query in rewritten_queries
         ), tuple(None for _ in queries)
     if method.name == "resolved_oracle":
         oracle_queries = tuple(
diff --git a/tests/test_eval_gemini_rewrite.py b/tests/test_eval_gemini_rewrite.py
index b0feaa2..0b843c3 100644
--- a/tests/test_eval_gemini_rewrite.py
+++ b/tests/test_eval_gemini_rewrite.py
@@ -42,7 +42,9 @@ def generate_content(**request: object) -> object:
 
     assert rewritten == "standalone query"
     assert requests[0]["model"] == "test-model"
-    assert requests[0]["config"] == {"temperature": 0.0}
+    assert requests[0]["config"] == {
+        "thinking_config": {"thinking_level": "minimal"}
+    }
     assert '"prior_user_utterances": ["first question"]' in str(requests[0]["contents"])
 
 

From 4df88eb6e469ea90c289b4d27d2dd177646a7886 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:41:24 +0000
Subject: [PATCH 10/12] Fix GitHub Actions version references

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 .github/workflows/ci.yml | 20 ++++++++++----------
 CHANGELOG.md             |  2 ++
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a49c669..d6c439b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,8 +20,8 @@ jobs:
     name: base package
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f
-      - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.12"
       - name: Install
@@ -44,8 +44,8 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f
-      - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
@@ -61,8 +61,8 @@ jobs:
     name: quality
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f
-      - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.12"
           cache: pip
@@ -89,8 +89,8 @@ jobs:
     name: cmg parity
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f
-      - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.12"
           cache: pip
@@ -110,8 +110,8 @@ jobs:
     name: eval offline
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f
-      - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.12"
           cache: pip
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a18045..3f5a313 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ All notable changes to this project are documented here.
 - Add an opt-in Gemini conversational query rewrite baseline for TREC CAsT,
   configured by `GEMINI_API_KEY`, with paired pointwise comparisons and
   model/prompt provenance in evaluation reports.
+- Replace unreachable GitHub Actions commit references with the current
+  `checkout@v7` and `setup-python@v6` major tags.
 - Add paired conversation-level bootstrap intervals for method nDCG@3 deltas,
   preserve conversation IDs in reports, and fix the aggregate `all`-slice
   bootstrap interval.

From 9ec82023af69fd03420e8a732cfae607a5c52b90 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:42:03 +0000
Subject: [PATCH 11/12] Format Gemini baseline tests

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 tests/test_eval_gemini_rewrite.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/test_eval_gemini_rewrite.py b/tests/test_eval_gemini_rewrite.py
index 0b843c3..9f6cf8c 100644
--- a/tests/test_eval_gemini_rewrite.py
+++ b/tests/test_eval_gemini_rewrite.py
@@ -42,9 +42,7 @@ def generate_content(**request: object) -> object:
 
     assert rewritten == "standalone query"
     assert requests[0]["model"] == "test-model"
-    assert requests[0]["config"] == {
-        "thinking_config": {"thinking_level": "minimal"}
-    }
+    assert requests[0]["config"] == {"thinking_config": {"thinking_level": "minimal"}}
     assert '"prior_user_utterances": ["first question"]' in str(requests[0]["contents"])
 
 

From be9a64585cb935e8503d1f77c0a8c4386d1488e2 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 13 Jul 2026 19:43:39 +0000
Subject: [PATCH 12/12] Fix optional dependency CI coverage

Co-authored-by: bigboateng <bigboateng@users.noreply.github.com>
---
 .github/workflows/ci.yml | 8 +++++++-
 CHANGELOG.md             | 3 ++-
 pyproject.toml           | 9 +++------
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d6c439b..a3dbe71 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,7 +29,13 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -e ".[dev]"
       - name: Test
-        run: python -m pytest
+        run: |
+          python -m pytest \
+            tests/test_decoder.py \
+            tests/test_graph_and_scoring.py \
+            tests/test_retrieval.py \
+            tests/test_section_graph.py \
+            tests/test_trace_render.py
       - name: Examples
         run: |
           for example in examples/*.py; do
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f5a313..32d53d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,8 @@ All notable changes to this project are documented here.
   configured by `GEMINI_API_KEY`, with paired pointwise comparisons and
   model/prompt provenance in evaluation reports.
 - Replace unreachable GitHub Actions commit references with the current
-  `checkout@v7` and `setup-python@v6` major tags.
+  `checkout@v7` and `setup-python@v6` major tags, restore Python 3.11 NumPy
+  compatibility, and keep the base-package job dependency-free.
 - Add paired conversation-level bootstrap intervals for method nDCG@3 deltas,
   preserve conversation IDs in reports, and fix the aggregate `all`-slice
   bootstrap interval.
diff --git a/pyproject.toml b/pyproject.toml
index 3340e8b..3124522 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,18 +23,15 @@ dependencies = []
 
 [project.optional-dependencies]
 graph = [
-  "numpy>=2.2.6; python_version < '3.11'",
-  "numpy>=2.5.1; python_version >= '3.11'",
+  "numpy>=2.2.6",
 ]
 faiss = [
   "faiss-cpu>=1.14.3",
-  "numpy>=2.2.6; python_version < '3.11'",
-  "numpy>=2.5.1; python_version >= '3.11'",
+  "numpy>=2.2.6",
 ]
 eval = [
   "ir-datasets>=0.5.11",
-  "numpy>=2.2.6; python_version < '3.11'",
-  "numpy>=2.5.1; python_version >= '3.11'",
+  "numpy>=2.2.6",
 ]
 gemini = [
   "google-genai>=2.11.0",