From d5044c75755cf46009ee8b0fa3f6b856186ec0ee Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 12 Jul 2026 16:54:21 +0000 Subject: [PATCH 1/3] Add conversation-level bootstrap CIs for eval nDCG@3 Resample conversations with replacement to compute 95% percentile intervals per slice. Wire into EvalConfig, runner, JSON/markdown reports, and the eval CLI via --bootstrap-samples and --bootstrap-seed. Co-authored-by: bigboateng --- CHANGELOG.md | 3 + docs/eval.md | 15 +++++ src/mapmatched/eval/__main__.py | 14 +++++ src/mapmatched/eval/bootstrap.py | 77 ++++++++++++++++++++++++++ src/mapmatched/eval/report.py | 11 +++- src/mapmatched/eval/runner.py | 50 ++++++++++++----- src/mapmatched/eval/slices.py | 12 +++- src/mapmatched/eval/types.py | 8 +++ tests/test_eval_bootstrap.py | 94 ++++++++++++++++++++++++++++++++ tests/test_eval_report.py | 1 + 10 files changed, 265 insertions(+), 20 deletions(-) create mode 100644 src/mapmatched/eval/bootstrap.py create mode 100644 tests/test_eval_bootstrap.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 33964de..e2ca93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project are documented here. ## Unreleased +- Bootstrap confidence intervals: conversation-level percentile bootstrap for + nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable + runs). CIs appear in JSON reports and the markdown table. - Full-ranking eval: the decoder now exposes the current turn's candidates ranked by trajectory (final-turn cumulative) score via `decode_ranked` and `RetrievalResult.candidate_ranking` / `CandidateScore`. The eval re-ranks the diff --git a/docs/eval.md b/docs/eval.md index b0ca668..2fb8fe1 100644 --- a/docs/eval.md +++ b/docs/eval.md @@ -97,6 +97,21 @@ the follow-up-lift claim than TopiOCQA's topic switches. - `--candidate-limit` (default 100) sizes the re-rankable window; the gold passage must be within it to be re-ranked (otherwise recall bounds the score). +## Bootstrap confidence intervals + +Use `--bootstrap-samples` to resample conversations and compute 95% percentile +CIs for nDCG@3 on each slice. Disabled by default (`0`) for fast smoke runs; +use `1000` for publishable numbers. `--bootstrap-seed` (default 42) keeps runs +reproducible. + +```console +python -m mapmatched.eval --benchmark synthetic \ + --bootstrap-samples 200 --output eval-report.json +``` + +The markdown table adds an `nDCG@3 95% CI` column; JSON reports include +`ndcg_at_3_ci` as `[lower, upper]` on each slice. + ## Reproducing headline numbers The README headline table uses **Tier B dev-slice** results with a diff --git a/src/mapmatched/eval/__main__.py b/src/mapmatched/eval/__main__.py index a98a3c2..867398e 100644 --- a/src/mapmatched/eval/__main__.py +++ b/src/mapmatched/eval/__main__.py @@ -59,6 +59,18 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--recall-k", type=int, default=100) parser.add_argument("--standalone-tolerance", type=float, default=0.02) parser.add_argument("--follow-up-min-delta", type=float, default=0.0) + parser.add_argument( + "--bootstrap-samples", + type=int, + default=0, + help="Conversation-level bootstrap resamples for nDCG@3 95%% CIs (0 = disabled).", + ) + parser.add_argument( + "--bootstrap-seed", + type=int, + default=42, + help="Random seed for bootstrap resampling.", + ) parser.add_argument("--include-resolved-oracle", action="store_true") return parser @@ -105,6 +117,8 @@ def main(argv: list[str] | None = None) -> int: follow_up_min_delta=args.follow_up_min_delta, ranking_mode=args.ranking_mode, graph_source=args.graph_source, + bootstrap_samples=args.bootstrap_samples, + bootstrap_seed=args.bootstrap_seed, ) report = run_ablation_grid( conversations=conversations, diff --git a/src/mapmatched/eval/bootstrap.py b/src/mapmatched/eval/bootstrap.py new file mode 100644 index 0000000..da5e7ad --- /dev/null +++ b/src/mapmatched/eval/bootstrap.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import math +import random +from collections.abc import Sequence + +from .metrics import mean +from .types import SliceName, TurnMetrics + + +def bootstrap_ndcg_at_3_ci( + conversation_turns: Sequence[Sequence[TurnMetrics]], + slice_name: SliceName, + *, + num_samples: int, + seed: int | None = None, + lower_percentile: float = 2.5, + upper_percentile: float = 97.5, +) -> tuple[float, float] | None: + if num_samples <= 0: + return None + conversation_count = len(conversation_turns) + if conversation_count == 0: + return None + + rng = random.Random(seed) + bootstrap_means: list[float] = [] + for _ in range(num_samples): + resampled = [ + conversation_turns[rng.randrange(conversation_count)] + for _ in range(conversation_count) + ] + selected_turns = [ + turn + for conversation in resampled + for turn in conversation + if turn.slice_name == slice_name + ] + bootstrap_means.append(mean([turn.ndcg_at_3 for turn in selected_turns])) + + sorted_means = sorted(bootstrap_means) + return ( + _percentile(sorted_means, lower_percentile), + _percentile(sorted_means, upper_percentile), + ) + + +def bootstrap_slice_cis( + conversation_turns: Sequence[Sequence[TurnMetrics]], + *, + num_samples: int, + seed: int | None = None, +) -> dict[SliceName, tuple[float, float] | None]: + slice_names: tuple[SliceName, ...] = ("follow_up", "standalone", "all") + return { + slice_name: bootstrap_ndcg_at_3_ci( + conversation_turns, + slice_name, + num_samples=num_samples, + seed=seed, + ) + for slice_name in slice_names + } + + +def _percentile(sorted_values: Sequence[float], percentile: float) -> float: + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return sorted_values[0] + rank = (len(sorted_values) - 1) * (percentile / 100.0) + lower_index = math.floor(rank) + upper_index = math.ceil(rank) + if lower_index == upper_index: + return sorted_values[int(rank)] + weight = rank - lower_index + return sorted_values[lower_index] * (1.0 - weight) + sorted_values[upper_index] * weight diff --git a/src/mapmatched/eval/report.py b/src/mapmatched/eval/report.py index 4747329..7d1179d 100644 --- a/src/mapmatched/eval/report.py +++ b/src/mapmatched/eval/report.py @@ -12,8 +12,8 @@ def render_markdown_table(report: EvalReport) -> str: f"Benchmark: `{report.config.benchmark}` · Tier: `{report.config.tier}` · " f"Embedder: `{report.config.embedder_name}`", "", - "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@5 | Recall | Δ vs β=0 |", - "| --- | --- | --- | --- | --- | --- | --- | --- |", + "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] baseline_by_slice = _baseline_ndcg(report.methods) for method in report.methods: @@ -36,6 +36,7 @@ def render_markdown_table(report: EvalReport) -> str: method.method_name, beta, f"{slice_metrics.ndcg_at_3:.3f}", + _format_ci(slice_metrics.ndcg_at_3_ci), f"{slice_metrics.ndcg_at_5:.3f}", f"{slice_metrics.recall_at_k:.3f}", delta, @@ -86,3 +87,9 @@ def _format_delta(value: float, baseline: float | None) -> str: if baseline is None: return "—" return f"{value - baseline:+.3f}" + + +def _format_ci(ci: tuple[float, float] | None) -> str: + if ci is None: + return "—" + return f"[{ci[0]:.3f}, {ci[1]:.3f}]" diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py index 1e1dd61..e982dcd 100644 --- a/src/mapmatched/eval/runner.py +++ b/src/mapmatched/eval/runner.py @@ -6,6 +6,7 @@ from .baselines import MapMatchedMethodConfig, run_mapmatched_conversation from .baselines.methods import run_maximal_marginal_relevance_conversation +from .bootstrap import bootstrap_slice_cis from .corpus import ( BruteForceProvider, build_knn_graph, @@ -164,7 +165,7 @@ def evaluate_method( entropy_threshold: float, graph_mode: str, ) -> MethodMetrics: - turn_metrics: list[TurnMetrics] = [] + conversation_turns: list[list[TurnMetrics]] = [] for conversation, trace_entropies_for_conversation in zip( conversations, trace_entropies, @@ -178,6 +179,7 @@ def evaluate_method( config=config, ranking_mode=eval_config.ranking_mode, ) + per_conversation: list[TurnMetrics] = [] for turn, ranking, trace_entropy in zip( conversation.turns, rankings, @@ -185,7 +187,7 @@ def evaluate_method( strict=True, ): relevances = turn_ranked_relevances(ranking, turn.qrels) - turn_metrics.append( + per_conversation.append( TurnMetrics( turn_index=turn.turn_index, ndcg_at_3=ndcg_at_k(relevances, 3), @@ -195,23 +197,40 @@ def evaluate_method( slice_name="all", ) ) + conversation_turns.append(per_conversation) - classified_turns = tuple( - TurnMetrics( - turn_index=turn.turn_index, - ndcg_at_3=turn.ndcg_at_3, - ndcg_at_5=turn.ndcg_at_5, - recall_at_k=turn.recall_at_k, - emission_entropy=turn.emission_entropy, - slice_name=classify_turn_slice( - turn_index=turn.turn_index, - emission_entropy=turn.emission_entropy, - entropy_threshold=entropy_threshold, - ), + classified_conversation_turns: list[list[TurnMetrics]] = [] + for per_conversation in conversation_turns: + classified_conversation_turns.append( + [ + TurnMetrics( + turn_index=turn.turn_index, + ndcg_at_3=turn.ndcg_at_3, + ndcg_at_5=turn.ndcg_at_5, + recall_at_k=turn.recall_at_k, + emission_entropy=turn.emission_entropy, + slice_name=classify_turn_slice( + turn_index=turn.turn_index, + emission_entropy=turn.emission_entropy, + entropy_threshold=entropy_threshold, + ), + ) + for turn in per_conversation + ] ) - for turn in turn_metrics + + classified_turns = tuple( + turn for conversation in classified_conversation_turns for turn in conversation ) + ndcg_at_3_cis = None + if eval_config.bootstrap_samples > 0: + ndcg_at_3_cis = bootstrap_slice_cis( + classified_conversation_turns, + num_samples=eval_config.bootstrap_samples, + seed=eval_config.bootstrap_seed, + ) + effective_transition_weight: float | None if method.name == "pointwise": effective_transition_weight = 0.0 @@ -231,6 +250,7 @@ def evaluate_method( fixed_lag=method.fixed_lag if method.fixed_lag is not None else config.fixed_lag, graph_mode=graph_mode, turns=classified_turns, + ndcg_at_3_cis=ndcg_at_3_cis, ) diff --git a/src/mapmatched/eval/slices.py b/src/mapmatched/eval/slices.py index 1aee595..06bb848 100644 --- a/src/mapmatched/eval/slices.py +++ b/src/mapmatched/eval/slices.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from math import isclose from .metrics import mean @@ -36,6 +36,8 @@ def classify_turn_slice( def aggregate_slice_metrics( turns: Sequence[TurnMetrics], slice_name: SliceName, + *, + ndcg_at_3_ci: tuple[float, float] | None = None, ) -> SliceMetrics: selected = [turn for turn in turns if turn.slice_name == slice_name] return SliceMetrics( @@ -44,6 +46,7 @@ def aggregate_slice_metrics( ndcg_at_3=mean([turn.ndcg_at_3 for turn in selected]), ndcg_at_5=mean([turn.ndcg_at_5 for turn in selected]), recall_at_k=mean([turn.recall_at_k for turn in selected]), + ndcg_at_3_ci=ndcg_at_3_ci, ) @@ -55,17 +58,20 @@ def build_method_metrics( fixed_lag: int | None, graph_mode: str, turns: Sequence[TurnMetrics], + ndcg_at_3_cis: Mapping[SliceName, tuple[float, float] | None] | None = None, ) -> MethodMetrics: + cis = ndcg_at_3_cis or {} all_slice = SliceMetrics( slice_name="all", turn_count=len(turns), ndcg_at_3=mean([turn.ndcg_at_3 for turn in turns]), ndcg_at_5=mean([turn.ndcg_at_5 for turn in turns]), recall_at_k=mean([turn.recall_at_k for turn in turns]), + ndcg_at_3_ci=cis.get("all"), ) slice_metrics = ( - aggregate_slice_metrics(turns, "follow_up"), - aggregate_slice_metrics(turns, "standalone"), + aggregate_slice_metrics(turns, "follow_up", ndcg_at_3_ci=cis.get("follow_up")), + aggregate_slice_metrics(turns, "standalone", ndcg_at_3_ci=cis.get("standalone")), all_slice, ) return MethodMetrics( diff --git a/src/mapmatched/eval/types.py b/src/mapmatched/eval/types.py index 5a8a2dd..4321cee 100644 --- a/src/mapmatched/eval/types.py +++ b/src/mapmatched/eval/types.py @@ -70,6 +70,7 @@ class SliceMetrics: ndcg_at_3: float ndcg_at_5: float recall_at_k: float + ndcg_at_3_ci: tuple[float, float] | None = None @dataclass(frozen=True, slots=True) @@ -97,6 +98,8 @@ class EvalConfig: ranking_mode: str = "full" # "knn" (embedding fallback) or "section" (structured group_key graph). graph_source: str = "knn" + bootstrap_samples: int = 0 + bootstrap_seed: int | None = 42 def to_dict(self) -> dict[str, object]: return { @@ -109,6 +112,8 @@ def to_dict(self) -> dict[str, object]: "follow_up_min_delta": self.follow_up_min_delta, "ranking_mode": self.ranking_mode, "graph_source": self.graph_source, + "bootstrap_samples": self.bootstrap_samples, + "bootstrap_seed": self.bootstrap_seed, } @@ -144,6 +149,9 @@ def to_dict(self) -> dict[str, object]: "ndcg_at_3": slice_metrics.ndcg_at_3, "ndcg_at_5": slice_metrics.ndcg_at_5, "recall_at_k": slice_metrics.recall_at_k, + "ndcg_at_3_ci": None + if slice_metrics.ndcg_at_3_ci is None + else list(slice_metrics.ndcg_at_3_ci), } for slice_metrics in method.slices ], diff --git a/tests/test_eval_bootstrap.py b/tests/test_eval_bootstrap.py new file mode 100644 index 0000000..d271b83 --- /dev/null +++ b/tests/test_eval_bootstrap.py @@ -0,0 +1,94 @@ +from mapmatched.eval.bootstrap import bootstrap_ndcg_at_3_ci, bootstrap_slice_cis +from mapmatched.eval.types import SliceName, TurnMetrics + + +def _turn(ndcg_at_3: float, slice_name: SliceName) -> TurnMetrics: + return TurnMetrics( + turn_index=0, + ndcg_at_3=ndcg_at_3, + ndcg_at_5=ndcg_at_3, + recall_at_k=ndcg_at_3, + emission_entropy=1.0, + slice_name=slice_name, + ) + + +def test_bootstrap_disabled_returns_none() -> None: + conversation_turns = ( + (_turn(1.0, "follow_up"), _turn(0.0, "standalone")), + (_turn(0.5, "follow_up"),), + ) + assert bootstrap_ndcg_at_3_ci(conversation_turns, "follow_up", num_samples=0) is None + + +def test_bootstrap_is_deterministic_with_seed() -> None: + conversation_turns = ( + (_turn(1.0, "follow_up"), _turn(0.0, "standalone")), + (_turn(0.5, "follow_up"),), + (_turn(0.25, "follow_up"), _turn(0.75, "standalone")), + ) + first = bootstrap_ndcg_at_3_ci( + conversation_turns, + "follow_up", + num_samples=200, + seed=7, + ) + second = bootstrap_ndcg_at_3_ci( + conversation_turns, + "follow_up", + num_samples=200, + seed=7, + ) + assert first == second + assert first is not None + assert first[0] <= first[1] + + +def test_bootstrap_slice_cis_returns_all_slices() -> None: + conversation_turns = ( + (_turn(1.0, "follow_up"), _turn(0.0, "standalone")), + (_turn(0.5, "follow_up"),), + ) + cis = bootstrap_slice_cis(conversation_turns, num_samples=50, seed=1) + assert set(cis) == {"follow_up", "standalone", "all"} + assert cis["follow_up"] is not None + assert cis["follow_up"][0] <= cis["follow_up"][1] + + +def test_synthetic_eval_populates_bootstrap_cis() -> None: + from mapmatched.eval import ( + DeterministicHashEmbedder, + EvalConfig, + MethodSpec, + load_synthetic_fixture, + run_eval, + ) + from mapmatched.eval.baselines import MapMatchedMethodConfig + + conversations, passages = load_synthetic_fixture() + embedder = DeterministicHashEmbedder() + report = run_eval( + conversations=conversations, + passages=passages, + embedder=embedder, + methods=(MethodSpec(name="pointwise", transition_weight=0.0),), + config=MapMatchedMethodConfig(candidate_limit=4), + eval_config=EvalConfig( + benchmark="synthetic", + tier="synthetic", + embedder_name=embedder.name, + recall_k=10, + entropy_threshold=None, + standalone_tolerance=0.02, + follow_up_min_delta=0.0, + bootstrap_samples=50, + bootstrap_seed=99, + ), + ) + follow_up = next( + slice_metrics + for slice_metrics in report.methods[0].slices + if slice_metrics.slice_name == "follow_up" + ) + assert follow_up.ndcg_at_3_ci is not None + assert follow_up.ndcg_at_3_ci[0] <= follow_up.ndcg_at_3 <= follow_up.ndcg_at_3_ci[1] diff --git a/tests/test_eval_report.py b/tests/test_eval_report.py index 3c51ac7..23c9bc2 100644 --- a/tests/test_eval_report.py +++ b/tests/test_eval_report.py @@ -34,3 +34,4 @@ def test_report_json_and_markdown_are_deterministic() -> None: markdown = render_markdown_table(report) assert '"benchmark": "synthetic"' in json_payload assert "| synthetic | follow_up | pointwise |" in markdown + assert "nDCG@3 95% CI" in markdown From ed1b4f7fb8af958bc913c98c06ac61f0768fb032 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 12 Jul 2026 19:31:41 +0000 Subject: [PATCH 2/3] Polish README for public alpha Co-authored-by: bigboateng --- README.md | 359 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 236 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index c7198ee..262f5b8 100644 --- a/README.md +++ b/README.md @@ -1,167 +1,280 @@ # Map-Matched Retrieval -Map-matched retrieval treats a multi-turn conversation as trajectory estimation. -Each turn's retrieval candidates are states in a trellis, corpus-graph distance is -the transition cost, and decoding returns the maximum-score path rather than an -independent winner for every turn. +[![CI](https://github.com/operatorstack/map-matched-retrieval/actions/workflows/ci.yml/badge.svg)](https://github.com/operatorstack/map-matched-retrieval/actions/workflows/ci.yml) +[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/) +[![Typed](https://img.shields.io/badge/typing-mypy_strict-blue)](https://mypy.readthedocs.io/) +[![Status: Alpha](https://img.shields.io/badge/status-alpha-orange)](#project-status) + +**Trajectory-aware retrieval for multi-turn conversations.** + +Most retrievers score every turn independently. Map-matched retrieval instead +decodes the conversation as a path through a corpus graph: retrieval scores say +where the conversation might be, while graph distance says how plausible each +move is. + +The result is a small, retriever-agnostic Python library that sits between your +candidate provider and your RAG pipeline. It returns both the selected chunks and +an inspectable trace showing the emission/transition trade-off behind each +decision. + +> [!IMPORTANT] +> This project is an **alpha**. The dependency-free core, typed public API, +> FAISS adapter, graph builders, traces, examples, and evaluation harness are +> implemented and tested. Benchmark coverage and third-party adapters are still +> being expanded, so APIs may evolve before 1.0. + +## Why trajectory decoding? + +| Pointwise retrieval | Map-matched retrieval | +| --- | --- | +| Chooses the highest-scoring chunk at each turn | Chooses the highest-scoring path across turns | +| Discards conversational location | Carries location through a corpus graph | +| Provides a score for the current result | Provides an emission/transition trace | +| Can jump on an ambiguous follow-up | Penalizes implausible jumps while preserving strong evidence | + +For candidates \(x_t\) at turn \(t\), the decoder maximizes: + +```text +Σ emission_weight × normalized_score(query_t, x_t) + − transition_weight × graph_distance(x_t−1, x_t) +``` + +Setting `transition_weight=0` exactly recovers deterministic pointwise retrieval. +Full Viterbi decoding can revise earlier turns; fixed-lag decoding provides a +bounded-revision streaming mode. + +## Highlights -It is a small retrieval decoder and trace library. It is not a vector store, -embedder, agent framework, query rewriter, graph builder for large corpora, or -context-ranking algorithm. In particular, the decoded MAP chunk and expanded -context are separate outputs; context expansion does not imply multiple decoded -paths. +- Standard-library-only core with no runtime dependencies +- Typed API checked with strict mypy +- Full and fixed-lag Viterbi decoders +- Bring-your-own retriever through a minimal `CandidateProvider` protocol +- In-memory weighted graphs and embedding-derived kNN graphs +- Optional FAISS adapter for cosine, inner-product, and L2 indexes +- JSON and terminal traces with scores, graph costs, entropy, and revisions +- Reproducible evaluation harness with ablations, baselines, and bootstrap CIs +- CI across Python 3.10, 3.11, and 3.12 ## Install -Python 3.10 or newer is required. +Python 3.10 or newer is required. The package is not yet published to PyPI; +install the public alpha from source: + +```console +git clone https://github.com/operatorstack/map-matched-retrieval.git +cd map-matched-retrieval +python -m pip install -e . +``` + +Optional extras keep the base package small: ```console -pip install map-matched-retrieval +python -m pip install -e ".[graph]" +python -m pip install -e ".[faiss]" +python -m pip install -e ".[eval,graph]" +python -m pip install -e ".[st]" ``` -The core has no runtime dependencies. Install `map-matched-retrieval[graph]` to -build a k-nearest-neighbor corpus graph from embeddings, or -`map-matched-retrieval[faiss]` to add both graph construction and FAISS -retrieval. Until composable-model-graph has a stable release, users who already -have a compatible installation may explicitly choose `CMGDecoder`; mapmatched -never installs or exposes its types. +| Extra | Adds | +| --- | --- | +| `graph` | Embedding-derived kNN graphs | +| `faiss` | kNN graphs and the FAISS candidate provider | +| `eval` | Benchmark loaders, baselines, metrics, and reports | +| `st` | Sentence-transformer embeddings for evaluation | + +## Quickstart -## Direct candidates +Supply scored candidates directly to see the decoder without a vector database: ```python from mapmatched import InMemoryCorpusGraph, MapMatchedRetriever, ScoredCandidate graph = InMemoryCorpusGraph.from_edges( - [("overview", "details"), ("details", "failure-modes")], + [("hmm", "noise"), ("noise", "road-jumps")], maximum_distance=4.0, ) session = MapMatchedRetriever( graph, - transition_weight=0.7, - score_normalization="zscore", + score_normalization="none", + transition_weight=1.0, ).session() -first = session.retrieve_candidates([ - ScoredCandidate("overview", 0.82), - ScoredCandidate("failure-modes", 0.78), -]) -second = session.retrieve_candidates([ - ScoredCandidate("details", 0.63), - ScoredCandidate("failure-modes", 0.65), -]) - -print(second.chunk_id) -print(second.context_chunk_ids) -print(second.trace.to_json(indent=2)) +session.retrieve_candidates( + [ + ScoredCandidate("hmm", 5.0), + ScoredCandidate("noise", 1.0), + ScoredCandidate("road-jumps", 0.0), + ] +) +result = session.retrieve_candidates( + [ + ScoredCandidate("hmm", 1.0), + ScoredCandidate("noise", 3.0), + ScoredCandidate("road-jumps", 3.5), + ] +) + +print(result.chunk_id) +print(result.context_chunk_ids) +print(result.trace.render()) ``` -For an existing retriever, implement `CandidateProvider.candidates(query, limit)` -and pass it as `provider=...`; then call `session.retrieve(query)`. +```text +noise +('noise', 'hmm', 'road-jumps') +``` -## FAISS session +The pointwise winner on the second turn is `road-jumps`, but the decoder selects +the adjacent `noise` chunk because its slightly lower emission score is offset by +a shorter graph move. The trace records the raw and normalized emissions, graph +distance, weighted transition cost, entropy, cumulative score, and any revisions +to prior turns. + +Run the complete example: + +```console +python examples/01_direct_candidates.py +``` -Mapmatched accepts caller-supplied embeddings but does not choose or download an -embedding model: +## Use an existing retriever + +Implement the two-argument candidate protocol and pass the provider into +`MapMatchedRetriever`: ```python -from mapmatched import FAISSProvider, KNNGraph, MapMatchedRetriever +from collections.abc import Sequence + +from mapmatched import MapMatchedRetriever, ScoredCandidate + + +class MyCandidateProvider: + def candidates(self, query: str, limit: int) -> Sequence[ScoredCandidate]: + return my_retriever.search(query, limit=limit) + -graph = KNNGraph.from_embeddings( - chunk_ids, - chunk_embeddings.tolist(), - neighbor_count=10, -) -provider = FAISSProvider( - faiss_index, - chunk_ids, - embed_query=my_embedding_function, -) session = MapMatchedRetriever( - graph, - provider=provider, + corpus_graph, + provider=MyCandidateProvider(), + candidate_limit=20, transition_weight=0.5, ).session() session.retrieve("How does token refresh work?") result = session.retrieve("What happens when it expires?") +``` -print(result.chunk_id) -print(result.context_chunk_ids) -print(result.trace.render()) +Providers return `ScoredCandidate` values with higher scores meaning better +matches. See +[`examples/02_custom_provider.py`](examples/02_custom_provider.py) for a complete +adapter and [`examples/04_faiss_session.py`](examples/04_faiss_session.py) for +FAISS with caller-supplied embeddings. + +## Architecture + +```text +query ──> CandidateProvider ──> scored candidate trellis + │ +corpus structure ──> CorpusGraph ─────┤ + ▼ + trajectory decoder + │ + ┌────────────┴────────────┐ + ▼ ▼ + RetrievalResult RetrievalTrace + chunk + ranked context scores + costs + revisions ``` -Use `score_mode="similarity"` for inner-product or cosine indexes. Use -`score_mode="distance"` for L2 indexes so lower FAISS distances become higher -retrieval scores. `FAISSProvider` verifies that index rows and chunk IDs stay -aligned. Normalize indexed and query vectors before using an inner-product index -as cosine search. - -`KNNGraph` normalizes embeddings and uses weighted cosine distance. Neighbor ties -are resolved by chunk ID, identical vectors receive a small positive edge -distance, and disconnected or over-cutoff paths still clamp to -`maximum_distance`. - -## Behavior and choices - -- The objective is `emission_weight * normalized_score - transition_weight * - graph_distance`, accumulated over the path. -- `zscore` is the safe normalization default. It makes each turn's score scale - comparable and maps a constant candidate set to zeros without division by zero. - `center` removes only the per-turn mean; `none` preserves the provider's scale - when scores are already calibrated. Raw and normalized scores remain in traces. -- Entropy is computed from a numerically stable softmax of weighted normalized - emissions. The margin is the chosen normalized emission minus the best - alternative; it can be negative when graph coherence overrules pointwise rank. -- `transition_weight=0` exactly reproduces deterministic per-turn argmax. - Candidate input order resolves score ties. -- Full decoding can revise any prior turn when evidence arrives. Set - `fixed_lag=L` to commit a turn after `L` later turns. The trace reports both - revised indices and the committed boundary. -- Context order is deterministic: current decoded chunk, graph neighbors ordered - by distance and ID, then current candidates in provider order, with stable - deduplication. - -## Suitable uses and limits - -This slice is intended for conversational documentation retrieval, linked -knowledge bases, section graphs, and other corpora where local movement has -meaning. Graph quality bounds retrieval quality. The in-memory graph uses bounded -Dijkstra searches and a distance cache; unreachable and beyond-cutoff pairs clamp -to `maximum_distance`. It is suitable for bounded candidate sets and modest -graphs, not an all-pairs graph service. - -Candidate providers should return a small, high-recall set. Decoding costs -`O(turns * candidates²)` graph lookups, reduced in practice by caching. A -standalone query in a long session can be over-smoothed by prior context; start a -new session for unrelated queries or reduce the transition weight. This library -deliberately has no adaptive weighting, asynchronous API, or multiple-path -decoding in the core package. - -## Evaluation (optional) - -Install the eval harness to run entropy-sliced benchmark reports: +Mapmatched does not replace a vector store, choose an embedding model, rewrite +queries, or run an agent framework. It owns one narrow boundary: graph-aware +sequential decoding over candidate sets. The decoded MAP chunk and expanded +context are separate outputs. -```console -pip install map-matched-retrieval[eval,graph] -python examples/05_eval_demo.py -``` +## Evaluation -See [`docs/eval.md`](docs/eval.md) for TopiOCQA / TREC CAsT micro-corpus runs, -ablation grids, and reproduction steps. The built-in hash embedder is for tests -only; published numbers require a caller-supplied embedding model. +The evaluation harness reports nDCG@3/5 and Recall@k separately for ambiguous +follow-up turns and sharp standalone turns. It includes pointwise, history +concatenation, Maximal Marginal Relevance, and resolved-query baselines, plus +conversation-level percentile bootstrap confidence intervals. -## Benchmark results (dev slice) +```console +python -m mapmatched.eval \ + --benchmark synthetic \ + --bootstrap-samples 200 +``` -| Benchmark | Slice | Method | β | nDCG@3 | nDCG@5 | Recall | Δ vs β=0 | -| --- | --- | --- | --- | --- | --- | --- | --- | -| synthetic | follow_up | mapmatched | 0.50 | TBD | TBD | TBD | TBD | -| synthetic | standalone | mapmatched | 0.50 | TBD | TBD | TBD | TBD | -| topiocqa (micro) | follow_up | mapmatched | 0.50 | TBD | TBD | TBD | TBD | -| cast2019 (micro) | follow_up | mapmatched | 0.50 | TBD | TBD | TBD | TBD | +The synthetic benchmark is a deterministic smoke test, not research evidence. +The current preliminary TopiOCQA micro-corpus result uses 25 conversations, +MiniLM embeddings, a kNN graph, full candidate ranking, and no bootstrap CI: + +| Measurement | nDCG@3 | +| --- | ---: | +| Pointwise follow-up | 0.150 | +| Map-matched follow-up | 0.234 | +| Follow-up delta | +0.084 | +| Standalone delta | +0.031 | + +These numbers demonstrate that the pipeline can produce measurable lift, but +they are not a full-corpus or statistically conclusive benchmark. Structured +section graphs also underperform on topic-switch-heavy TopiOCQA, an important +negative result rather than a hidden one. See [`docs/eval.md`](docs/eval.md) for +benchmark tiers, methodology, limitations, and reproduction commands. + +## Design choices and limits + +- Per-turn z-score normalization is the safe default; `center` and `none` are + available when provider scores already have a meaningful scale. +- Candidate providers should return a small, high-recall set. Decoding requires + `O(turns × candidates²)` graph-distance lookups, reduced by distance caching. +- Graph quality bounds retrieval quality. The in-memory graph uses bounded + Dijkstra search and clamps unreachable or over-cutoff distances. +- A long session can over-smooth unrelated queries. Start a new session or lower + `transition_weight` when the topic changes. +- The alpha does not yet provide adaptive weighting, an asynchronous API, + multiple-path decoding, or managed graph infrastructure. + +The strongest current use cases are conversational documentation retrieval, +linked knowledge bases, section graphs, and other corpora where local movement +has semantic meaning. + +## Project status + +- [x] Dependency-free decoder, graph protocol, and retrieval session +- [x] Full and fixed-lag decoding with inspectable traces +- [x] kNN graph builder and FAISS candidate provider +- [x] Synthetic, TopiOCQA, and TREC CAsT evaluation paths +- [x] Full candidate ranking and conversation-level bootstrap CIs +- [ ] Lock reproducible full-corpus benchmark results +- [ ] Add LangChain/LlamaIndex and hosted vector-store adapters +- [ ] Add graph construction tooling for larger corpora +- [ ] Stabilize the public API for a non-alpha release + +See [`PLAN.md`](PLAN.md) for the longer roadmap and [`CHANGELOG.md`](CHANGELOG.md) +for the implementation history. + +## Documentation and examples + +- [`docs/theory.md`](docs/theory.md) — objective, normalization, graph distance, + and decoding semantics +- [`docs/eval.md`](docs/eval.md) — benchmark tiers, baselines, and reproduction +- [`examples/01_direct_candidates.py`](examples/01_direct_candidates.py) — core + decoder without external dependencies +- [`examples/02_custom_provider.py`](examples/02_custom_provider.py) — custom + candidate provider +- [`examples/03_cmg_inspectable_run.py`](examples/03_cmg_inspectable_run.py) — + optional composable-model-graph backend +- [`examples/04_faiss_session.py`](examples/04_faiss_session.py) — FAISS and kNN + integration +- [`examples/05_eval_demo.py`](examples/05_eval_demo.py) — offline synthetic eval + +## Development -Run `python -m mapmatched.eval --benchmark synthetic` to populate the synthetic -row locally. Tier B rows require network access and a real embedder for -publishable values. +```console +python -m pip install -e ".[dev,faiss,eval,graph]" +python -m pytest +python -m ruff check . +python -m ruff format --check . +python -m mypy +``` -See [`docs/theory.md`](docs/theory.md) for the objective and semantics and -[`examples`](examples) for complete runs. +Issues and focused pull requests are welcome. For behavior changes, include tests +and update the changelog so design decisions remain visible. From 484e7012c4a0b8d614c89a00d653d1986615f019 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 12 Jul 2026 19:32:28 +0000 Subject: [PATCH 3/3] Format bootstrap evaluation module Co-authored-by: bigboateng --- src/mapmatched/eval/bootstrap.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mapmatched/eval/bootstrap.py b/src/mapmatched/eval/bootstrap.py index da5e7ad..98977d6 100644 --- a/src/mapmatched/eval/bootstrap.py +++ b/src/mapmatched/eval/bootstrap.py @@ -27,8 +27,7 @@ def bootstrap_ndcg_at_3_ci( bootstrap_means: list[float] = [] for _ in range(num_samples): resampled = [ - conversation_turns[rng.randrange(conversation_count)] - for _ in range(conversation_count) + conversation_turns[rng.randrange(conversation_count)] for _ in range(conversation_count) ] selected_turns = [ turn