diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a49c669..a3dbe71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,8 @@ jobs: name: base package runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f - - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install @@ -29,7 +29,13 @@ jobs: python -m pip install --upgrade pip python -m pip install -e ".[dev]" - name: Test - run: python -m pytest + run: | + python -m pytest \ + tests/test_decoder.py \ + tests/test_graph_and_scoring.py \ + tests/test_retrieval.py \ + tests/test_section_graph.py \ + tests/test_trace_render.py - name: Examples run: | for example in examples/*.py; do @@ -44,8 +50,8 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f - - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -61,8 +67,8 @@ jobs: name: quality runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f - - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.12" cache: pip @@ -89,8 +95,8 @@ jobs: name: cmg parity runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f - - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.12" cache: pip @@ -110,8 +116,8 @@ jobs: name: eval offline runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630cea06102a548caf8bcb44f - - uses: actions/setup-python@8d9ed9acaa204a42485415563790864b2b171d23 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.12" cache: pip diff --git a/.gitignore b/.gitignore index da3a6a9..96cdac9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ __pycache__/ build/ dist/ eval-report.json +data/ +reports/ diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ca93b..32d53d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project are documented here. ## Unreleased +- Add an opt-in Gemini conversational query rewrite baseline for TREC CAsT, + configured by `GEMINI_API_KEY`, with paired pointwise comparisons and + model/prompt provenance in evaluation reports. +- Replace unreachable GitHub Actions commit references with the current + `checkout@v7` and `setup-python@v6` major tags, restore Python 3.11 NumPy + compatibility, and keep the base-package job dependency-free. +- Add paired conversation-level bootstrap intervals for method nDCG@3 deltas, + preserve conversation IDs in reports, and fix the aggregate `all`-slice + bootstrap interval. +- Add a pinned TopiOCQA n=25 MiniLM/kNN reproduction script with dataset + checksum, selected conversation IDs, model, graph, package, and git provenance. +- Cache repeated query embeddings in the eval provider and reuse one bounded + shortest-path search across all targets for a graph source. +- Publish the reproducible TopiOCQA n=25 MiniLM/kNN micro-corpus result: + map-matched β=1.0 lifts follow-up nDCG@3 by `+0.084` with paired 95% CI + `[+0.046, +0.128]`; two runs produced byte-identical reports. - Bootstrap confidence intervals: conversation-level percentile bootstrap for nDCG@3 per slice (`--bootstrap-samples`, default 0; use 1000 for publishable runs). CIs appear in JSON reports and the markdown table. diff --git a/README.md b/README.md index 262f5b8..7d09a1c 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ python -m pip install -e ".[st]" | `graph` | Embedding-derived kNN graphs | | `faiss` | kNN graphs and the FAISS candidate provider | | `eval` | Benchmark loaders, baselines, metrics, and reports | +| `gemini` | Gemini conversational query rewrite baseline | | `st` | Sentence-transformer embeddings for evaluation | ## Quickstart @@ -193,8 +194,9 @@ context are separate outputs. The evaluation harness reports nDCG@3/5 and Recall@k separately for ambiguous follow-up turns and sharp standalone turns. It includes pointwise, history -concatenation, Maximal Marginal Relevance, and resolved-query baselines, plus -conversation-level percentile bootstrap confidence intervals. +concatenation, optional Gemini query rewriting, Maximal Marginal Relevance, and +resolved-query baselines, plus conversation-level percentile bootstrap confidence +intervals. Method deltas use paired resampling of the same conversations. ```console python -m mapmatched.eval \ @@ -203,21 +205,33 @@ python -m mapmatched.eval \ ``` The synthetic benchmark is a deterministic smoke test, not research evidence. -The current preliminary TopiOCQA micro-corpus result uses 25 conversations, -MiniLM embeddings, a kNN graph, full candidate ranking, and no bootstrap CI: - -| Measurement | nDCG@3 | -| --- | ---: | -| Pointwise follow-up | 0.150 | -| Map-matched follow-up | 0.234 | -| Follow-up delta | +0.084 | -| Standalone delta | +0.031 | - -These numbers demonstrate that the pipeline can produce measurable lift, but -they are not a full-corpus or statistically conclusive benchmark. Structured -section graphs also underperform on topic-switch-heavy TopiOCQA, an important -negative result rather than a hidden one. See [`docs/eval.md`](docs/eval.md) for -benchmark tiers, methodology, limitations, and reproduction commands. +The pinned TopiOCQA micro-corpus profile uses 25 conversations, MiniLM +embeddings, a 10-neighbor kNN graph, full candidate ranking, and 1,000 paired +conversation-level bootstrap draws: + +| Slice | Method | nDCG@3 | Delta vs pointwise | Paired delta 95% CI | +| --- | --- | ---: | ---: | ---: | +| Follow-up | Pointwise | 0.150 | +0.000 | — | +| Follow-up | Map-matched β=0.5 | 0.195 | +0.045 | [+0.018, +0.077] | +| Follow-up | Map-matched β=1.0 | 0.234 | +0.084 | [+0.046, +0.128] | +| Follow-up | MMR | 0.151 | +0.001 | [+0.000, +0.003] | +| Standalone | Map-matched β=1.0 | 0.373 | +0.031 | [+0.009, +0.055] | + +Both runs produced byte-identical reports. The positive paired intervals are +evidence for this fixed micro-corpus, not a full-Wikipedia or cross-benchmark +claim. Structured section graphs also underperform on topic-switch-heavy +TopiOCQA, an important negative result rather than a hidden one. See the +[`committed result`](results/topiocqa_n25_minilm_knn.md) and +[`evaluation guide`](docs/eval.md) for provenance, all baselines, limitations, +and reproduction commands. + +Reproduce the pinned n=25 MiniLM/kNN profile after downloading the validation +split: + +```console +python -m pip install -e ".[eval,graph,st]" +./scripts/reproduce_topiocqa_n25.sh data/topiocqa_valid.jsonl +``` ## Design choices and limits diff --git a/docs/eval.md b/docs/eval.md index 2fb8fe1..7af2620 100644 --- a/docs/eval.md +++ b/docs/eval.md @@ -1,18 +1,20 @@ # Evaluation harness Map-matched retrieval makes a narrow claim: it should lift **underspecified -follow-up turns** without materially harming **sharp standalone turns**. M2 adds -an optional evaluation harness behind `pip install map-matched-retrieval[eval]`. +follow-up turns** without materially harming **sharp standalone turns**. The +evaluation harness measures that claim without making conversational RAG the +library's API boundary. ## Install ```console -pip install map-matched-retrieval[eval,graph] +python -m pip install -e ".[eval,graph,st]" ``` The harness uses NumPy for kNN graph construction and optional Hugging Face / ir-datasets loaders for benchmark metadata. It does **not** download embedding -models. +models. Install `.[eval,graph,st,gemini]` only when running the Gemini rewrite +baseline. ## Tiers @@ -54,6 +56,7 @@ best map-matched configuration against the β=0 pointwise baseline: | `pointwise` | `transition_weight=0` — independent per-turn top-1 | | `mapmatched` | Full trajectory decoder with configurable β | | `history_concat` | Dense retrieval over concatenated query history | +| `gemini_rewrite` | Gemini rewrites each turn into a standalone query before dense retrieval | | `maximal_marginal_relevance` | Per-turn MMR re-ranking (not map-matched retrieval) | | `resolved_oracle` | CAsT resolved utterances (upper bound) | @@ -61,23 +64,48 @@ best map-matched configuration against the β=0 pointwise baseline: ### TopiOCQA (micro) +Download `data/topiocqa_valid.jsonl` from the +[TopiOCQA dataset repository](https://huggingface.co/datasets/McGill-NLP/TopiOCQA), +then run the pinned profile: + ```console -# download a split first (HF datasets dropped the custom dataset script): -# https://huggingface.co/datasets/McGill-NLP/TopiOCQA -> data/topiocqa_valid.jsonl -python -m mapmatched.eval --benchmark topiocqa \ - --data-path topiocqa_valid.jsonl --conversation-limit 25 \ - --embedder sentence-transformers --graph-source section --ranking-mode full +./scripts/reproduce_topiocqa_n25.sh data/topiocqa_valid.jsonl ``` Reads the released JSON/JSONL directly (`--data-path` or `MAPMATCHED_TOPIOCQA_PATH`) -and builds a micro-corpus from gold passages and additional answers. The section -graph keys on the Wikipedia article title. Note TopiOCQA is topic-switch heavy, -so it stresses the standalone (H0) side as much as the follow-up (H1) side. +and builds a micro-corpus from gold passages and additional answers. The pinned +profile selects the first 25 conversations in file order, records the file +SHA-256 and selected IDs, uses `sentence-transformers/all-MiniLM-L6-v2`, a +10-neighbor kNN graph, full ranking, a 100-candidate window, and 1,000 bootstrap +draws with seed 42. + +TopiOCQA is topic-switch heavy, so it stresses the standalone side as much as the +follow-up side. It is licensed +[CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/); the +dataset is not redistributed by this repository. These runs are micro-corpus +experiments, not full-Wikipedia retrieval. + +The pinned run at git revision `60a9d694b807c3eb49da2a00743c6f1a05d52e6d` +produced byte-identical reports twice: + +| Slice | Method | nDCG@3 | Delta vs pointwise | Paired delta 95% CI | +| --- | --- | ---: | ---: | ---: | +| Follow-up | Map-matched β=0.5 | 0.195 | +0.045 | [+0.018, +0.077] | +| Follow-up | Map-matched β=1.0 | 0.234 | +0.084 | [+0.046, +0.128] | +| Follow-up | MMR | 0.151 | +0.001 | [+0.000, +0.003] | +| Standalone | Map-matched β=1.0 | 0.373 | +0.031 | [+0.009, +0.055] | + +The complete aggregate table and provenance are committed in +[`results/topiocqa_n25_minilm_knn.md`](../results/topiocqa_n25_minilm_knn.md). +Positive intervals support the claim on this fixed 25-conversation micro-corpus; +they do not establish full-corpus or cross-benchmark generalization. ### TREC CAsT 2019 (micro) ```console -python -m mapmatched.eval --benchmark cast2019 --embedder sentence-transformers +export GEMINI_API_KEY="..." +python -m pip install -e ".[eval,graph,st,gemini]" +./scripts/reproduce_cast2019_gemini.sh ``` Uses ir-datasets id `trec-cast/v1/2019/judged`. Real passage text comes from the @@ -87,6 +115,15 @@ Without the collection the loader degrades to using doc ids as passage text (metrics not meaningful). CAsT's drill-down follow-ups are the fairer test for the follow-up-lift claim than TopiOCQA's topic switches. +The script adds `gemini_rewrite` to the normal ablation grid. It sends each raw +utterance and its prior user utterances to `gemini-3.5-flash` with minimal +thinking, retrieves with the returned standalone query, and compares it with both +pointwise retrieval and CAsT's manual `resolved_oracle`. `GEMINI_API_KEY` is read +from the environment and is never written to reports. Reports record the Gemini +model and prompt version. The baseline is opt-in because it makes one paid, +networked model request per selected turn; `--conversation-limit` bounds those +requests. Hosted-model output is not immutable across model revisions. + ## Graph source and ranking mode - `--graph-source knn` (default) builds the embedding-kNN fallback graph; @@ -100,17 +137,20 @@ the follow-up-lift claim than TopiOCQA's topic switches. ## Bootstrap confidence intervals Use `--bootstrap-samples` to resample conversations and compute 95% percentile -CIs for nDCG@3 on each slice. Disabled by default (`0`) for fast smoke runs; -use `1000` for publishable numbers. `--bootstrap-seed` (default 42) keeps runs -reproducible. +CIs for nDCG@3 on each slice. Method comparisons resample the same conversations +for treatment and pointwise retrieval, producing a paired CI on the nDCG@3 +delta. Disabled by default (`0`) for fast smoke runs; use `1000` for reported +numbers. `--bootstrap-seed` (default 42) keeps runs reproducible. ```console python -m mapmatched.eval --benchmark synthetic \ --bootstrap-samples 200 --output eval-report.json ``` -The markdown table adds an `nDCG@3 95% CI` column; JSON reports include -`ndcg_at_3_ci` as `[lower, upper]` on each slice. +The markdown table includes absolute and paired-delta 95% CIs. JSON reports +include method-level `ndcg_at_3_ci` values and explicit `comparisons` with +`ndcg_at_3_delta_ci`. The claim gate remains based on configured point-estimate +thresholds; a paired interval excluding zero is the uncertainty check. ## Reproducing headline numbers @@ -118,12 +158,11 @@ The README headline table uses **Tier B dev-slice** results with a caller-supplied embedder. The built-in `DeterministicHashEmbedder` is for tests and smoke runs only. -For publishable numbers: - -1. Choose an embedding model and implement `QueryEmbedder` / `PassageEmbedder`. -2. Build a micro-corpus or full corpus index. -3. Run the ablation grid and record JSON + markdown output. -4. Paste the markdown table into README with the embedder and tier noted. +The TopiOCQA script writes full JSON and markdown reports under +`reports/topiocqa-n25-minilm-knn-full/`. Generated reports are ignored because +they contain machine-run detail; committed headline values must include the +profile, dataset SHA-256, model, graph settings, conversation count, and paired +interval. ## Output diff --git a/pyproject.toml b/pyproject.toml index 244c226..3124522 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,18 +23,18 @@ dependencies = [] [project.optional-dependencies] graph = [ - "numpy>=2.2.6; python_version < '3.11'", - "numpy>=2.5.1; python_version >= '3.11'", + "numpy>=2.2.6", ] faiss = [ "faiss-cpu>=1.14.3", - "numpy>=2.2.6; python_version < '3.11'", - "numpy>=2.5.1; python_version >= '3.11'", + "numpy>=2.2.6", ] eval = [ "ir-datasets>=0.5.11", - "numpy>=2.2.6; python_version < '3.11'", - "numpy>=2.5.1; python_version >= '3.11'", + "numpy>=2.2.6", +] +gemini = [ + "google-genai>=2.11.0", ] st = [ "sentence-transformers>=3.0", diff --git a/results/topiocqa_n25_minilm_knn.md b/results/topiocqa_n25_minilm_knn.md new file mode 100644 index 0000000..693619e --- /dev/null +++ b/results/topiocqa_n25_minilm_knn.md @@ -0,0 +1,32 @@ +# TopiOCQA n=25 MiniLM/kNN result + +Profile: `topiocqa-n25-minilm-knn-full` + +- Dataset: `topiocqa_valid.jsonl` +- Dataset SHA-256: `1bba9512b24b2e5de22704766dd80b1dc497bb7262799f66c3912e6f413ac1c6` +- Conversations: 25 +- Embedder: `sentence-transformers/all-MiniLM-L6-v2` +- Graph: 10-neighbor kNN +- Ranking: full, candidate limit 100 +- Bootstrap: 1,000 conversation-level draws, seed 42 +- Package: `map-matched-retrieval==0.1.0` +- Git revision: `60a9d694b807c3eb49da2a00743c6f1a05d52e6d` +- Full JSON report SHA-256: `35a1801b2d7502a7bc1036f90b53818783f56ca9f2ffeebdfad66fc41056ce9e` + +| Slice | Method | β | nDCG@3 | nDCG@3 95% CI | Delta vs pointwise | Paired delta 95% CI | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| Follow-up | Pointwise | 0.00 | 0.150 | [0.109, 0.196] | +0.000 | — | +| Follow-up | Map-matched | 0.50 | 0.195 | [0.147, 0.249] | +0.045 | [+0.018, +0.077] | +| Follow-up | Map-matched | 1.00 | 0.234 | [0.186, 0.294] | +0.084 | [+0.046, +0.128] | +| Follow-up | History concat | — | 0.071 | [0.043, 0.099] | -0.079 | [-0.137, -0.027] | +| Follow-up | MMR | — | 0.151 | [0.109, 0.197] | +0.001 | [+0.000, +0.003] | +| Standalone | Pointwise | 0.00 | 0.342 | [0.282, 0.405] | +0.000 | — | +| Standalone | Map-matched | 0.50 | 0.359 | [0.295, 0.424] | +0.017 | [+0.004, +0.034] | +| Standalone | Map-matched | 1.00 | 0.373 | [0.305, 0.440] | +0.031 | [+0.009, +0.055] | +| Standalone | History concat | — | 0.134 | [0.110, 0.159] | -0.208 | [-0.261, -0.158] | +| Standalone | MMR | — | 0.340 | [0.280, 0.404] | -0.002 | [-0.004, +0.000] | + +Both repeated runs produced byte-identical JSON and markdown reports. This is a +gold-passage micro-corpus result, not full-Wikipedia retrieval. The paired +intervals quantify uncertainty for these 25 conversations and should not be +generalized to other corpora without additional evaluation. diff --git a/scripts/reproduce_cast2019_gemini.sh b/scripts/reproduce_cast2019_gemini.sh new file mode 100755 index 0000000..9e3d633 --- /dev/null +++ b/scripts/reproduce_cast2019_gemini.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROFILE="cast2019-gemini35flash-minilm-knn-full" +REPORT_DIR="${MAPMATCHED_REPORT_DIR:-$ROOT_DIR/reports/$PROFILE}" +GEMINI_MODEL="${MAPMATCHED_GEMINI_MODEL:-gemini-3.5-flash}" + +if [[ -z "${GEMINI_API_KEY:-}" ]]; then + echo "GEMINI_API_KEY must be set" >&2 + exit 2 +fi + +mkdir -p "$REPORT_DIR" + +export CUDA_VISIBLE_DEVICES="" +export PYTHONHASHSEED=0 +export TOKENIZERS_PARALLELISM=false + +python3 -m mapmatched.eval \ + --profile "$PROFILE" \ + --benchmark cast2019 \ + --tier micro \ + --conversation-limit 50 \ + --embedder sentence-transformers \ + --st-model sentence-transformers/all-MiniLM-L6-v2 \ + --graph-source knn \ + --knn-neighbors 10 \ + --ranking-mode full \ + --candidate-limit 100 \ + --recall-k 100 \ + --bootstrap-samples 1000 \ + --bootstrap-seed 42 \ + --include-gemini-rewrite \ + --gemini-model "$GEMINI_MODEL" \ + --output "$REPORT_DIR/report.json" \ + --markdown-output "$REPORT_DIR/report.md" + +echo "JSON report: $REPORT_DIR/report.json" +echo "Markdown report: $REPORT_DIR/report.md" diff --git a/scripts/reproduce_topiocqa_n25.sh b/scripts/reproduce_topiocqa_n25.sh new file mode 100755 index 0000000..81e5bed --- /dev/null +++ b/scripts/reproduce_topiocqa_n25.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DATA_PATH="${1:-${MAPMATCHED_TOPIOCQA_PATH:-}}" +PROFILE="topiocqa-n25-minilm-knn-full" +REPORT_DIR="${MAPMATCHED_REPORT_DIR:-$ROOT_DIR/reports/$PROFILE}" + +if [[ -z "$DATA_PATH" ]]; then + echo "usage: $0 /path/to/topiocqa_valid.jsonl" >&2 + echo "or set MAPMATCHED_TOPIOCQA_PATH" >&2 + exit 2 +fi + +if [[ ! -f "$DATA_PATH" ]]; then + echo "TopiOCQA data file not found: $DATA_PATH" >&2 + exit 2 +fi + +mkdir -p "$REPORT_DIR" + +export CUDA_VISIBLE_DEVICES="" +export PYTHONHASHSEED=0 +export TOKENIZERS_PARALLELISM=false + +python3 -m mapmatched.eval \ + --profile "$PROFILE" \ + --benchmark topiocqa \ + --tier micro \ + --data-path "$DATA_PATH" \ + --conversation-limit 25 \ + --embedder sentence-transformers \ + --st-model sentence-transformers/all-MiniLM-L6-v2 \ + --graph-source knn \ + --knn-neighbors 10 \ + --ranking-mode full \ + --candidate-limit 100 \ + --recall-k 100 \ + --bootstrap-samples 1000 \ + --bootstrap-seed 42 \ + --output "$REPORT_DIR/report.json" \ + --markdown-output "$REPORT_DIR/report.md" + +echo "JSON report: $REPORT_DIR/report.json" +echo "Markdown report: $REPORT_DIR/report.md" diff --git a/src/mapmatched/eval/__init__.py b/src/mapmatched/eval/__init__.py index f843060..f0ad5ba 100644 --- a/src/mapmatched/eval/__init__.py +++ b/src/mapmatched/eval/__init__.py @@ -5,13 +5,22 @@ from .loaders import load_cast2019_micro, load_synthetic_fixture, load_topiocqa_micro from .report import render_json, render_markdown_table from .runner import MethodSpec, run_eval -from .types import EvalConfig, EvalConversation, EvalReport, Passage +from .types import ( + ComparisonSliceMetrics, + EvalConfig, + EvalConversation, + EvalReport, + MethodComparison, + Passage, +) __all__ = [ + "ComparisonSliceMetrics", "DeterministicHashEmbedder", "EvalConfig", "EvalConversation", "EvalReport", + "MethodComparison", "MethodSpec", "Passage", "QueryEmbedder", diff --git a/src/mapmatched/eval/__main__.py b/src/mapmatched/eval/__main__.py index 867398e..4ae5699 100644 --- a/src/mapmatched/eval/__main__.py +++ b/src/mapmatched/eval/__main__.py @@ -1,9 +1,20 @@ from __future__ import annotations import argparse +import hashlib +import os +import subprocess from pathlib import Path +from mapmatched import __version__ + from .ablations import run_ablation_grid +from .baselines import ( + DEFAULT_GEMINI_MODEL, + GEMINI_REWRITE_PROMPT_VERSION, + ConversationQueryRewriter, + create_gemini_query_rewriter, +) from .embedder import DeterministicHashEmbedder, SentenceTransformerEmbedder from .loaders import load_cast2019_micro, load_synthetic_fixture, load_topiocqa_micro from .report import render_json, render_markdown_table @@ -44,6 +55,12 @@ def build_parser() -> argparse.ArgumentParser: default="knn", help="knn = embedding fallback graph; section = structured group_key graph.", ) + parser.add_argument( + "--knn-neighbors", + type=int, + default=10, + help="Neighbors per passage when --graph-source knn.", + ) parser.add_argument( "--ranking-mode", choices=("full", "rank1"), @@ -71,7 +88,22 @@ def build_parser() -> argparse.ArgumentParser: default=42, help="Random seed for bootstrap resampling.", ) + parser.add_argument( + "--profile", + default=None, + help="Stable run profile name stored in report metadata.", + ) parser.add_argument("--include-resolved-oracle", action="store_true") + parser.add_argument( + "--include-gemini-rewrite", + action="store_true", + help="Evaluate a Gemini conversational query rewrite baseline.", + ) + parser.add_argument( + "--gemini-model", + default=DEFAULT_GEMINI_MODEL, + help="Gemini model used by --include-gemini-rewrite.", + ) return parser @@ -90,17 +122,21 @@ def load_benchmark( ) if name == "cast2019": conversations, passages = load_cast2019_micro() - return conversations, passages + return conversations[:conversation_limit], passages raise ValueError(f"unsupported benchmark: {name}") def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) + query_rewriter: ConversationQueryRewriter | None = None + if args.include_gemini_rewrite: + query_rewriter = create_gemini_query_rewriter(model=args.gemini_model) + data_path = _effective_data_path(args.benchmark, args.data_path) conversations, passages = load_benchmark( args.benchmark, conversation_limit=args.conversation_limit, - data_path=args.data_path, + data_path=data_path, ) embedder: DeterministicHashEmbedder | SentenceTransformerEmbedder if args.embedder == "sentence-transformers": @@ -117,8 +153,23 @@ def main(argv: list[str] | None = None) -> int: follow_up_min_delta=args.follow_up_min_delta, ranking_mode=args.ranking_mode, graph_source=args.graph_source, + knn_neighbor_count=args.knn_neighbors, bootstrap_samples=args.bootstrap_samples, bootstrap_seed=args.bootstrap_seed, + profile=args.profile, + dataset_filename=None if data_path is None else data_path.name, + dataset_sha256=None if data_path is None else _sha256(data_path), + conversation_ids=tuple(conversation.conversation_id for conversation in conversations), + embedding_model=args.st_model + if args.embedder == "sentence-transformers" + else embedder.name, + package_version=__version__, + git_revision=_git_revision(), + query_rewrite_provider="gemini" if args.include_gemini_rewrite else None, + query_rewrite_model=args.gemini_model if args.include_gemini_rewrite else None, + query_rewrite_prompt_version=GEMINI_REWRITE_PROMPT_VERSION + if args.include_gemini_rewrite + else None, ) report = run_ablation_grid( conversations=conversations, @@ -126,7 +177,9 @@ def main(argv: list[str] | None = None) -> int: embedder=embedder, eval_config=eval_config, candidate_limit=args.candidate_limit, + include_gemini_rewrite=args.include_gemini_rewrite, include_resolved_oracle=args.include_resolved_oracle or args.benchmark == "cast2019", + query_rewriter=query_rewriter, ) args.output.write_text(render_json(report), encoding="utf-8") markdown = render_markdown_table(report) @@ -137,5 +190,40 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _effective_data_path(benchmark: str, data_path: Path | None) -> Path | None: + if data_path is not None: + return data_path + if benchmark != "topiocqa": + return None + environment_path = os.environ.get("MAPMATCHED_TOPIOCQA_PATH") + if environment_path is None: + return None + return Path(environment_path) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as data_file: + for block in iter(lambda: data_file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _git_revision() -> str | None: + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + except OSError: + return None + if result.returncode != 0: + return None + revision = result.stdout.strip() + return revision or None + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/mapmatched/eval/ablations.py b/src/mapmatched/eval/ablations.py index d67c8d7..775cfda 100644 --- a/src/mapmatched/eval/ablations.py +++ b/src/mapmatched/eval/ablations.py @@ -2,7 +2,7 @@ from collections.abc import Sequence -from .baselines import MapMatchedMethodConfig +from .baselines import ConversationQueryRewriter, MapMatchedMethodConfig from .embedder import TextEmbedder from .runner import MethodSpec, run_eval from .types import EvalConfig, EvalConversation, EvalReport, Passage @@ -11,6 +11,7 @@ def default_method_grid( *, transition_weights: Sequence[float] = (0.0, 0.5, 1.0), + include_gemini_rewrite: bool = False, include_mmr: bool = True, include_resolved_oracle: bool = False, ) -> tuple[MethodSpec, ...]: @@ -22,6 +23,8 @@ def default_method_grid( continue methods.append(MethodSpec(name="mapmatched", transition_weight=beta)) methods.append(MethodSpec(name="history_concat")) + if include_gemini_rewrite: + methods.append(MethodSpec(name="gemini_rewrite")) if include_mmr: methods.append(MethodSpec(name="maximal_marginal_relevance")) if include_resolved_oracle: @@ -38,11 +41,14 @@ def run_ablation_grid( transition_weights: Sequence[float] = (0.0, 0.5, 1.0), candidate_limit: int = 20, fixed_lag: int | None = None, + include_gemini_rewrite: bool = False, include_mmr: bool = True, include_resolved_oracle: bool = False, + query_rewriter: ConversationQueryRewriter | None = None, ) -> EvalReport: methods = default_method_grid( transition_weights=transition_weights, + include_gemini_rewrite=include_gemini_rewrite, include_mmr=include_mmr, include_resolved_oracle=include_resolved_oracle, ) @@ -57,4 +63,5 @@ def run_ablation_grid( methods=methods, config=config, eval_config=eval_config, + query_rewriter=query_rewriter, ) diff --git a/src/mapmatched/eval/baselines/__init__.py b/src/mapmatched/eval/baselines/__init__.py index f1357c3..2f58a49 100644 --- a/src/mapmatched/eval/baselines/__init__.py +++ b/src/mapmatched/eval/baselines/__init__.py @@ -1,5 +1,13 @@ from __future__ import annotations +from .gemini import ( + DEFAULT_GEMINI_MODEL, + GEMINI_REWRITE_PROMPT_VERSION, + GeminiDependencyUnavailableError, + GeminiQueryRewriter, + GeminiRewriteConfig, + create_gemini_query_rewriter, +) from .methods import ( MapMatchedMethodConfig, run_history_concat_conversation, @@ -8,9 +16,18 @@ run_pointwise_conversation, run_resolved_oracle_conversation, ) +from .rewrite import ConversationQueryRewriter, rewrite_conversation_queries __all__ = [ + "DEFAULT_GEMINI_MODEL", + "GEMINI_REWRITE_PROMPT_VERSION", + "ConversationQueryRewriter", + "GeminiDependencyUnavailableError", + "GeminiQueryRewriter", + "GeminiRewriteConfig", "MapMatchedMethodConfig", + "create_gemini_query_rewriter", + "rewrite_conversation_queries", "run_history_concat_conversation", "run_mapmatched_conversation", "run_maximal_marginal_relevance_conversation", diff --git a/src/mapmatched/eval/baselines/gemini.py b/src/mapmatched/eval/baselines/gemini.py new file mode 100644 index 0000000..20ddef8 --- /dev/null +++ b/src/mapmatched/eval/baselines/gemini.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib +import json +import os +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Literal + +DEFAULT_GEMINI_MODEL = "gemini-3.5-flash" +GEMINI_REWRITE_PROMPT_VERSION = "cast-standalone-v1" + +_SYSTEM_INSTRUCTION = """\ +Rewrite the current conversational search utterance as one concise, standalone search query. +Resolve references and omitted context using only the prior user utterances. +Preserve the current information need and do not answer it. +Return only the rewritten query with no explanation, label, quotation marks, or markdown.""" + + +class GeminiDependencyUnavailableError(ImportError): + pass + + +@dataclass(frozen=True, slots=True) +class GeminiRewriteConfig: + model: str = DEFAULT_GEMINI_MODEL + thinking_level: Literal["minimal", "low", "medium", "high"] = "minimal" + + +class GeminiQueryRewriter: + def __init__( + self, + *, + generate_content: Callable[..., object], + config: GeminiRewriteConfig, + client: object | None = None, + ) -> None: + self._client = client + self._generate_content = generate_content + self.config = config + + def rewrite(self, *, history: Sequence[str], query: str) -> str: + request = { + "prior_user_utterances": list(history), + "current_utterance": query, + } + prompt = ( + f"{_SYSTEM_INSTRUCTION}\n\nConversation:\n{json.dumps(request, ensure_ascii=False)}" + ) + response = self._generate_content( + model=self.config.model, + contents=prompt, + config={"thinking_config": {"thinking_level": self.config.thinking_level}}, + ) + response_text = getattr(response, "text", None) + if not isinstance(response_text, str) or not response_text.strip(): + raise RuntimeError("Gemini returned no rewritten query") + return response_text.strip() + + +def create_gemini_query_rewriter( + *, + api_key: str | None = None, + model: str = DEFAULT_GEMINI_MODEL, +) -> GeminiQueryRewriter: + effective_api_key = api_key if api_key is not None else os.environ.get("GEMINI_API_KEY") + if not effective_api_key: + raise ValueError("GEMINI_API_KEY must be set when the Gemini rewrite baseline is enabled") + try: + genai = importlib.import_module("google.genai") + except ImportError as error: + raise GeminiDependencyUnavailableError( + "Gemini rewrite requires the 'gemini' extra: " + "pip install 'map-matched-retrieval[eval,gemini]'" + ) from error + client_factory = getattr(genai, "Client", None) + if not callable(client_factory): + raise GeminiDependencyUnavailableError("google.genai.Client is unavailable") + client = client_factory(api_key=effective_api_key) + models = getattr(client, "models", None) + generate_content = getattr(models, "generate_content", None) + if not callable(generate_content): + raise GeminiDependencyUnavailableError( + "google.genai Client does not provide models.generate_content" + ) + return GeminiQueryRewriter( + generate_content=generate_content, + config=GeminiRewriteConfig(model=model), + client=client, + ) diff --git a/src/mapmatched/eval/baselines/rewrite.py b/src/mapmatched/eval/baselines/rewrite.py new file mode 100644 index 0000000..838f8ec --- /dev/null +++ b/src/mapmatched/eval/baselines/rewrite.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import Protocol + +from ..types import EvalConversation + + +class ConversationQueryRewriter(Protocol): + def rewrite(self, *, history: Sequence[str], query: str) -> str: ... + + +def rewrite_conversation_queries( + conversation: EvalConversation, + rewriter: ConversationQueryRewriter, +) -> tuple[str, ...]: + history: list[str] = [] + rewritten_queries: list[str] = [] + for turn in conversation.turns: + rewritten_query = rewriter.rewrite(history=tuple(history), query=turn.query).strip() + if not rewritten_query: + raise RuntimeError( + f"query rewriter returned an empty query for " + f"{conversation.conversation_id} turn {turn.turn_index}" + ) + rewritten_queries.append(rewritten_query) + history.append(turn.query) + return tuple(rewritten_queries) diff --git a/src/mapmatched/eval/bootstrap.py b/src/mapmatched/eval/bootstrap.py index 98977d6..4aeee6d 100644 --- a/src/mapmatched/eval/bootstrap.py +++ b/src/mapmatched/eval/bootstrap.py @@ -2,7 +2,7 @@ import math import random -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from .metrics import mean from .types import SliceName, TurnMetrics @@ -19,7 +19,8 @@ def bootstrap_ndcg_at_3_ci( ) -> tuple[float, float] | None: if num_samples <= 0: return None - conversation_count = len(conversation_turns) + contributing_conversations = _contributing_conversations(conversation_turns, slice_name) + conversation_count = len(contributing_conversations) if conversation_count == 0: return None @@ -27,20 +28,16 @@ def bootstrap_ndcg_at_3_ci( bootstrap_means: list[float] = [] for _ in range(num_samples): resampled = [ - conversation_turns[rng.randrange(conversation_count)] for _ in range(conversation_count) - ] - selected_turns = [ - turn - for conversation in resampled - for turn in conversation - if turn.slice_name == slice_name + contributing_conversations[rng.randrange(conversation_count)] + for _ in range(conversation_count) ] + selected_turns = _selected_turns(resampled, slice_name) bootstrap_means.append(mean([turn.ndcg_at_3 for turn in selected_turns])) - sorted_means = sorted(bootstrap_means) - return ( - _percentile(sorted_means, lower_percentile), - _percentile(sorted_means, upper_percentile), + return _percentile_interval( + bootstrap_means, + lower_percentile=lower_percentile, + upper_percentile=upper_percentile, ) @@ -62,6 +59,138 @@ def bootstrap_slice_cis( } +def bootstrap_paired_ndcg_at_3_delta_ci( + treatment_conversation_turns: Sequence[Sequence[TurnMetrics]], + baseline_conversation_turns: Sequence[Sequence[TurnMetrics]], + slice_name: SliceName, + *, + num_samples: int, + seed: int | None = None, + lower_percentile: float = 2.5, + upper_percentile: float = 97.5, +) -> tuple[float, float] | None: + if num_samples <= 0: + return None + treatment_by_id = _conversations_by_id(treatment_conversation_turns) + baseline_by_id = _conversations_by_id(baseline_conversation_turns) + _validate_alignment(treatment_by_id, baseline_by_id) + contributing_ids = tuple( + conversation_id + for conversation_id, turns in treatment_by_id.items() + if _selected_turns((turns,), slice_name) + ) + if not contributing_ids: + return None + + rng = random.Random(seed) + bootstrap_deltas: list[float] = [] + for _ in range(num_samples): + sampled_ids = tuple( + contributing_ids[rng.randrange(len(contributing_ids))] + for _ in range(len(contributing_ids)) + ) + treatment_turns = _selected_turns( + tuple(treatment_by_id[conversation_id] for conversation_id in sampled_ids), + slice_name, + ) + baseline_turns = _selected_turns( + tuple(baseline_by_id[conversation_id] for conversation_id in sampled_ids), + slice_name, + ) + treatment_mean = mean([turn.ndcg_at_3 for turn in treatment_turns]) + baseline_mean = mean([turn.ndcg_at_3 for turn in baseline_turns]) + bootstrap_deltas.append(treatment_mean - baseline_mean) + + return _percentile_interval( + bootstrap_deltas, + lower_percentile=lower_percentile, + upper_percentile=upper_percentile, + ) + + +def bootstrap_paired_slice_delta_cis( + treatment_conversation_turns: Sequence[Sequence[TurnMetrics]], + baseline_conversation_turns: Sequence[Sequence[TurnMetrics]], + *, + num_samples: int, + seed: int | None = None, +) -> dict[SliceName, tuple[float, float] | None]: + slice_names: tuple[SliceName, ...] = ("follow_up", "standalone", "all") + return { + slice_name: bootstrap_paired_ndcg_at_3_delta_ci( + treatment_conversation_turns, + baseline_conversation_turns, + slice_name, + num_samples=num_samples, + seed=seed, + ) + for slice_name in slice_names + } + + +def _contributing_conversations( + conversation_turns: Sequence[Sequence[TurnMetrics]], + slice_name: SliceName, +) -> tuple[Sequence[TurnMetrics], ...]: + return tuple(turns for turns in conversation_turns if _selected_turns((turns,), slice_name)) + + +def _selected_turns( + conversation_turns: Sequence[Sequence[TurnMetrics]], + slice_name: SliceName, +) -> list[TurnMetrics]: + return [ + turn + for conversation in conversation_turns + for turn in conversation + if slice_name == "all" or turn.slice_name == slice_name + ] + + +def _conversations_by_id( + conversation_turns: Sequence[Sequence[TurnMetrics]], +) -> dict[str, tuple[TurnMetrics, ...]]: + by_id: dict[str, tuple[TurnMetrics, ...]] = {} + for turns in conversation_turns: + if not turns: + raise ValueError("conversation turn metrics must not be empty") + conversation_ids = {turn.conversation_id for turn in turns} + if len(conversation_ids) != 1: + raise ValueError("conversation turn metrics contain multiple conversation IDs") + conversation_id = next(iter(conversation_ids)) + if conversation_id in by_id: + raise ValueError(f"duplicate conversation metrics: {conversation_id}") + by_id[conversation_id] = tuple(turns) + return by_id + + +def _validate_alignment( + treatment_by_id: Mapping[str, Sequence[TurnMetrics]], + baseline_by_id: Mapping[str, Sequence[TurnMetrics]], +) -> None: + if treatment_by_id.keys() != baseline_by_id.keys(): + raise ValueError("treatment and baseline conversation IDs do not match") + for conversation_id, treatment_turns in treatment_by_id.items(): + baseline_turns = baseline_by_id[conversation_id] + treatment_keys = tuple((turn.turn_index, turn.slice_name) for turn in treatment_turns) + baseline_keys = tuple((turn.turn_index, turn.slice_name) for turn in baseline_turns) + if treatment_keys != baseline_keys: + raise ValueError(f"treatment and baseline turns do not align for {conversation_id}") + + +def _percentile_interval( + values: Sequence[float], + *, + lower_percentile: float, + upper_percentile: float, +) -> tuple[float, float]: + sorted_values = sorted(values) + return ( + _percentile(sorted_values, lower_percentile), + _percentile(sorted_values, upper_percentile), + ) + + def _percentile(sorted_values: Sequence[float], percentile: float) -> float: if not sorted_values: return 0.0 diff --git a/src/mapmatched/eval/corpus.py b/src/mapmatched/eval/corpus.py index c547136..08bc8ec 100644 --- a/src/mapmatched/eval/corpus.py +++ b/src/mapmatched/eval/corpus.py @@ -24,6 +24,7 @@ def __init__( self._passage_ids = ids self._passage_embeddings = tuple(tuple(values) for values in passage_embeddings) self._embed_query = embed_query + self._query_embedding_cache: dict[str, tuple[float, ...]] = {} @property def passage_count(self) -> int: @@ -32,7 +33,10 @@ def passage_count(self) -> int: def candidates(self, query: str, limit: int) -> list[ScoredCandidate]: if limit <= 0: raise ValueError("limit must be greater than zero") - query_embedding = tuple(self._embed_query.embed_query(query)) + query_embedding = self._query_embedding_cache.get(query) + if query_embedding is None: + query_embedding = tuple(self._embed_query.embed_query(query)) + self._query_embedding_cache[query] = query_embedding scored = [ ( dot_product(query_embedding, passage_embedding), diff --git a/src/mapmatched/eval/loaders/topiocqa.py b/src/mapmatched/eval/loaders/topiocqa.py index 710b6ec..3e0a77e 100644 --- a/src/mapmatched/eval/loaders/topiocqa.py +++ b/src/mapmatched/eval/loaders/topiocqa.py @@ -20,15 +20,15 @@ def _resolve_data_path(data_path: str | os.PathLike[str] | None) -> Path: HuggingFace `datasets` dropped support for the custom dataset *script* that `McGill-NLP/TopiOCQA` ships, so we read the released JSON directly instead. - Point this at a downloaded split (e.g. ``topiocqa_dev.json``) via the + Point this at the downloaded validation split (``topiocqa_valid.jsonl``) via the ``data_path`` argument or the ``MAPMATCHED_TOPIOCQA_PATH`` environment variable. """ candidate = data_path if data_path is not None else os.environ.get(_PATH_ENV_VAR) if not candidate: raise EvalDependencyUnavailableError( - "TopiOCQA loader needs the dataset JSON. Download a split (e.g. " - "topiocqa_dev.json) from https://github.com/McGill-NLP/topiocqa and " + "TopiOCQA loader needs the dataset JSONL. Download data/topiocqa_valid.jsonl " + "from https://huggingface.co/datasets/McGill-NLP/TopiOCQA and " f"pass data_path=... or set {_PATH_ENV_VAR}." ) path = Path(candidate) diff --git a/src/mapmatched/eval/report.py b/src/mapmatched/eval/report.py index 7d1179d..c6e5b68 100644 --- a/src/mapmatched/eval/report.py +++ b/src/mapmatched/eval/report.py @@ -2,20 +2,41 @@ import json -from .types import EvalReport, MethodMetrics, SliceName +from .types import EvalReport, MethodComparison, MethodMetrics, SliceName def render_markdown_table(report: EvalReport) -> str: + metadata = [ + f"Benchmark: `{report.config.benchmark}`", + f"Tier: `{report.config.tier}`", + f"Embedder: `{report.config.embedder_name}`", + ] + if report.config.profile is not None: + metadata.append(f"Profile: `{report.config.profile}`") lines = [ "## Benchmark results (dev slice)", "", - f"Benchmark: `{report.config.benchmark}` · Tier: `{report.config.tier}` · " - f"Embedder: `{report.config.embedder_name}`", + " · ".join(metadata), "", - "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | nDCG@5 | Recall | Δ vs β=0 |", - "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] + if report.config.dataset_filename is not None: + lines.extend( + [ + f"Data: `{report.config.dataset_filename}` · " + f"SHA-256: `{report.config.dataset_sha256}` · " + f"Conversations: `{len(report.config.conversation_ids)}`", + "", + ] + ) + lines.extend( + [ + "| Benchmark | Slice | Method | β | nDCG@3 | nDCG@3 95% CI | " + "nDCG@5 | Recall | Δ vs β=0 (95% CI) |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) baseline_by_slice = _baseline_ndcg(report.methods) + comparison_by_slice = _comparison_delta_cis(report.comparisons) for method in report.methods: for slice_metrics in method.slices: if slice_metrics.slice_name == "all": @@ -25,7 +46,17 @@ def render_markdown_table(report: EvalReport) -> str: baseline = baseline_by_slice.get( (method.candidate_limit, method.fixed_lag, slice_metrics.slice_name), ) - delta = _format_delta(slice_metrics.ndcg_at_3, baseline) + delta_ci = comparison_by_slice.get( + ( + method.method_name, + method.transition_weight, + method.candidate_limit, + method.fixed_lag, + method.graph_mode, + slice_metrics.slice_name, + ) + ) + delta = _format_delta(slice_metrics.ndcg_at_3, baseline, delta_ci) beta = "—" if method.transition_weight is None else f"{method.transition_weight:.2f}" lines.append( "| " @@ -83,10 +114,42 @@ def _baseline_ndcg( return baseline -def _format_delta(value: float, baseline: float | None) -> str: +def _comparison_delta_cis( + comparisons: tuple[MethodComparison, ...], +) -> dict[ + tuple[str, float | None, int, int | None, str, SliceName], + tuple[float, float] | None, +]: + by_slice: dict[ + tuple[str, float | None, int, int | None, str, SliceName], + tuple[float, float] | None, + ] = {} + for comparison in comparisons: + for slice_metrics in comparison.slices: + by_slice[ + ( + comparison.method_name, + comparison.transition_weight, + comparison.candidate_limit, + comparison.fixed_lag, + comparison.graph_mode, + slice_metrics.slice_name, + ) + ] = slice_metrics.ndcg_at_3_delta_ci + return by_slice + + +def _format_delta( + value: float, + baseline: float | None, + ci: tuple[float, float] | None, +) -> str: if baseline is None: return "—" - return f"{value - baseline:+.3f}" + delta = f"{value - baseline:+.3f}" + if ci is None: + return delta + return f"{delta} [{ci[0]:+.3f}, {ci[1]:+.3f}]" def _format_ci(ci: tuple[float, float] | None) -> str: diff --git a/src/mapmatched/eval/runner.py b/src/mapmatched/eval/runner.py index e982dcd..8e49e75 100644 --- a/src/mapmatched/eval/runner.py +++ b/src/mapmatched/eval/runner.py @@ -4,9 +4,14 @@ from mapmatched import CorpusGraph -from .baselines import MapMatchedMethodConfig, run_mapmatched_conversation +from .baselines import ( + ConversationQueryRewriter, + MapMatchedMethodConfig, + rewrite_conversation_queries, + run_mapmatched_conversation, +) from .baselines.methods import run_maximal_marginal_relevance_conversation -from .bootstrap import bootstrap_slice_cis +from .bootstrap import bootstrap_paired_slice_delta_cis, bootstrap_slice_cis from .corpus import ( BruteForceProvider, build_knn_graph, @@ -21,7 +26,16 @@ compute_entropy_threshold, evaluate_claim, ) -from .types import EvalConfig, EvalConversation, EvalReport, MethodMetrics, Passage, TurnMetrics +from .types import ( + ComparisonSliceMetrics, + EvalConfig, + EvalConversation, + EvalReport, + MethodComparison, + MethodMetrics, + Passage, + TurnMetrics, +) class MethodSpec: @@ -52,6 +66,7 @@ def run_method_on_conversation( method: MethodSpec, config: MapMatchedMethodConfig, ranking_mode: str = "full", + query_rewriter: ConversationQueryRewriter | None = None, ) -> tuple[tuple[tuple[str, ...], ...], tuple[float | None, ...]]: queries = [turn.query for turn in conversation.turns] method_config = MapMatchedMethodConfig( @@ -91,6 +106,13 @@ def run_method_on_conversation( rankings.append(rank_full_corpus(provider, rewritten)) history.append(query) return tuple(rankings), tuple(None for _ in queries) + if method.name == "gemini_rewrite": + if query_rewriter is None: + raise ValueError("gemini_rewrite requires a conversation query rewriter") + rewritten_queries = rewrite_conversation_queries(conversation, query_rewriter) + return tuple( + rank_full_corpus(provider, rewritten_query) for rewritten_query in rewritten_queries + ), tuple(None for _ in queries) if method.name == "resolved_oracle": oracle_queries = tuple( turn.resolved_query if turn.resolved_query is not None else turn.query @@ -143,13 +165,18 @@ def _build_provider_and_graph( embedder: TextEmbedder, *, graph_source: str, + knn_neighbor_count: int, ) -> tuple[BruteForceProvider, CorpusGraph]: passage_ids, passage_embeddings = build_passage_embeddings(passages, embedder) provider = BruteForceProvider(passage_ids, passage_embeddings, embedder) if graph_source == "section": graph = build_section_graph(passages) else: - graph = build_knn_graph(passage_ids, passage_embeddings) + graph = build_knn_graph( + passage_ids, + passage_embeddings, + neighbor_count=knn_neighbor_count, + ) return provider, graph @@ -164,6 +191,7 @@ def evaluate_method( trace_entropies: Sequence[Sequence[float | None]], entropy_threshold: float, graph_mode: str, + query_rewriter: ConversationQueryRewriter | None = None, ) -> MethodMetrics: conversation_turns: list[list[TurnMetrics]] = [] for conversation, trace_entropies_for_conversation in zip( @@ -178,6 +206,7 @@ def evaluate_method( method=method, config=config, ranking_mode=eval_config.ranking_mode, + query_rewriter=query_rewriter, ) per_conversation: list[TurnMetrics] = [] for turn, ranking, trace_entropy in zip( @@ -189,6 +218,7 @@ def evaluate_method( relevances = turn_ranked_relevances(ranking, turn.qrels) per_conversation.append( TurnMetrics( + conversation_id=conversation.conversation_id, turn_index=turn.turn_index, ndcg_at_3=ndcg_at_k(relevances, 3), ndcg_at_5=ndcg_at_k(relevances, 5), @@ -204,6 +234,7 @@ def evaluate_method( classified_conversation_turns.append( [ TurnMetrics( + conversation_id=turn.conversation_id, turn_index=turn.turn_index, ndcg_at_3=turn.ndcg_at_3, ndcg_at_5=turn.ndcg_at_5, @@ -262,12 +293,14 @@ def run_eval( methods: Sequence[MethodSpec], config: MapMatchedMethodConfig, eval_config: EvalConfig, + query_rewriter: ConversationQueryRewriter | None = None, ) -> EvalReport: graph_mode = eval_config.graph_source provider, graph = _build_provider_and_graph( passages, embedder, graph_source=graph_mode, + knn_neighbor_count=eval_config.knn_neighbor_count, ) trace_method = MethodSpec(name="pointwise") trace_entropies = tuple( @@ -298,9 +331,11 @@ def run_eval( trace_entropies=trace_entropies, entropy_threshold=entropy_threshold, graph_mode=graph_mode, + query_rewriter=query_rewriter, ) for method in methods ) + comparisons = _build_method_comparisons(method_metrics, eval_config) mapmatched_methods = [method for method in methods if method.name == "mapmatched"] if not mapmatched_methods: verdict = None @@ -321,4 +356,89 @@ def run_eval( follow_up_min_delta=eval_config.follow_up_min_delta, mapmatched_transition_weight=best_mapmatched.transition_weight, ) - return EvalReport(config=eval_config, methods=method_metrics, verdict=verdict) + return EvalReport( + config=eval_config, + methods=method_metrics, + verdict=verdict, + comparisons=comparisons, + ) + + +def _build_method_comparisons( + methods: Sequence[MethodMetrics], + eval_config: EvalConfig, +) -> tuple[MethodComparison, ...]: + comparisons: list[MethodComparison] = [] + comparable_method_names = { + "gemini_rewrite", + "history_concat", + "mapmatched", + "maximal_marginal_relevance", + } + for method in methods: + if method.method_name not in comparable_method_names: + continue + baseline = _find_pointwise_baseline(methods, method) + if baseline is None: + continue + delta_cis = bootstrap_paired_slice_delta_cis( + _group_turns_by_conversation(method), + _group_turns_by_conversation(baseline), + num_samples=eval_config.bootstrap_samples, + seed=eval_config.bootstrap_seed, + ) + baseline_slices = { + slice_metrics.slice_name: slice_metrics for slice_metrics in baseline.slices + } + comparison_slices = tuple( + ComparisonSliceMetrics( + slice_name=slice_metrics.slice_name, + turn_count=slice_metrics.turn_count, + ndcg_at_3_delta=( + slice_metrics.ndcg_at_3 - baseline_slices[slice_metrics.slice_name].ndcg_at_3 + ), + ndcg_at_3_delta_ci=delta_cis[slice_metrics.slice_name], + ) + for slice_metrics in method.slices + ) + comparisons.append( + MethodComparison( + method_name=method.method_name, + transition_weight=method.transition_weight, + baseline_method_name=baseline.method_name, + baseline_transition_weight=baseline.transition_weight, + candidate_limit=method.candidate_limit, + fixed_lag=method.fixed_lag, + graph_mode=method.graph_mode, + slices=comparison_slices, + ) + ) + return tuple(comparisons) + + +def _find_pointwise_baseline( + methods: Sequence[MethodMetrics], + treatment: MethodMetrics, +) -> MethodMetrics | None: + for method in methods: + if method.method_name != "pointwise": + continue + if method.transition_weight not in (0.0, None): + continue + if method.candidate_limit != treatment.candidate_limit: + continue + if method.fixed_lag != treatment.fixed_lag: + continue + if method.graph_mode != treatment.graph_mode: + continue + return method + return None + + +def _group_turns_by_conversation( + method: MethodMetrics, +) -> tuple[tuple[TurnMetrics, ...], ...]: + grouped: dict[str, list[TurnMetrics]] = {} + for turn in method.turns: + grouped.setdefault(turn.conversation_id, []).append(turn) + return tuple(tuple(turns) for turns in grouped.values()) diff --git a/src/mapmatched/eval/types.py b/src/mapmatched/eval/types.py index 4321cee..a74bed3 100644 --- a/src/mapmatched/eval/types.py +++ b/src/mapmatched/eval/types.py @@ -55,6 +55,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class TurnMetrics: + conversation_id: str turn_index: int ndcg_at_3: float ndcg_at_5: float @@ -84,6 +85,26 @@ class MethodMetrics: turns: tuple[TurnMetrics, ...] +@dataclass(frozen=True, slots=True) +class ComparisonSliceMetrics: + slice_name: SliceName + turn_count: int + ndcg_at_3_delta: float + ndcg_at_3_delta_ci: tuple[float, float] | None = None + + +@dataclass(frozen=True, slots=True) +class MethodComparison: + method_name: str + transition_weight: float | None + baseline_method_name: str + baseline_transition_weight: float | None + candidate_limit: int + fixed_lag: int | None + graph_mode: str + slices: tuple[ComparisonSliceMetrics, ...] + + @dataclass(frozen=True, slots=True) class EvalConfig: benchmark: str @@ -98,8 +119,19 @@ class EvalConfig: ranking_mode: str = "full" # "knn" (embedding fallback) or "section" (structured group_key graph). graph_source: str = "knn" + knn_neighbor_count: int = 10 bootstrap_samples: int = 0 bootstrap_seed: int | None = 42 + profile: str | None = None + dataset_filename: str | None = None + dataset_sha256: str | None = None + conversation_ids: tuple[str, ...] = () + embedding_model: str | None = None + package_version: str | None = None + git_revision: str | None = None + query_rewrite_provider: str | None = None + query_rewrite_model: str | None = None + query_rewrite_prompt_version: str | None = None def to_dict(self) -> dict[str, object]: return { @@ -112,8 +144,19 @@ def to_dict(self) -> dict[str, object]: "follow_up_min_delta": self.follow_up_min_delta, "ranking_mode": self.ranking_mode, "graph_source": self.graph_source, + "knn_neighbor_count": self.knn_neighbor_count, "bootstrap_samples": self.bootstrap_samples, "bootstrap_seed": self.bootstrap_seed, + "profile": self.profile, + "dataset_filename": self.dataset_filename, + "dataset_sha256": self.dataset_sha256, + "conversation_ids": list(self.conversation_ids), + "embedding_model": self.embedding_model, + "package_version": self.package_version, + "git_revision": self.git_revision, + "query_rewrite_provider": self.query_rewrite_provider, + "query_rewrite_model": self.query_rewrite_model, + "query_rewrite_prompt_version": self.query_rewrite_prompt_version, } @@ -131,6 +174,7 @@ class EvalReport: config: EvalConfig methods: tuple[MethodMetrics, ...] verdict: ClaimVerdict | None + comparisons: tuple[MethodComparison, ...] = () def to_dict(self) -> dict[str, object]: return { @@ -157,6 +201,7 @@ def to_dict(self) -> dict[str, object]: ], "turns": [ { + "conversation_id": turn.conversation_id, "turn_index": turn.turn_index, "ndcg_at_3": turn.ndcg_at_3, "ndcg_at_5": turn.ndcg_at_5, @@ -169,6 +214,29 @@ def to_dict(self) -> dict[str, object]: } for method in self.methods ], + "comparisons": [ + { + "method_name": comparison.method_name, + "transition_weight": comparison.transition_weight, + "baseline_method_name": comparison.baseline_method_name, + "baseline_transition_weight": comparison.baseline_transition_weight, + "candidate_limit": comparison.candidate_limit, + "fixed_lag": comparison.fixed_lag, + "graph_mode": comparison.graph_mode, + "slices": [ + { + "slice_name": slice_metrics.slice_name, + "turn_count": slice_metrics.turn_count, + "ndcg_at_3_delta": slice_metrics.ndcg_at_3_delta, + "ndcg_at_3_delta_ci": None + if slice_metrics.ndcg_at_3_delta_ci is None + else list(slice_metrics.ndcg_at_3_delta_ci), + } + for slice_metrics in comparison.slices + ], + } + for comparison in self.comparisons + ], "verdict": None if self.verdict is None else { diff --git a/src/mapmatched/graph.py b/src/mapmatched/graph.py index 805a0f9..b422d8d 100644 --- a/src/mapmatched/graph.py +++ b/src/mapmatched/graph.py @@ -96,9 +96,18 @@ def distance(self, source_chunk_id: str, target_chunk_id: str) -> float: if cached is not None: return cached distances = self._bounded_distances(source_chunk_id, self._maximum_distance) - result = min(distances.get(target_chunk_id, self._maximum_distance), self._maximum_distance) - self._distance_cache[cache_key] = result - return result + for chunk_id in self._adjacency: + distance = min( + distances.get(chunk_id, self._maximum_distance), + self._maximum_distance, + ) + self._distance_cache[self._cache_key(source_chunk_id, chunk_id)] = distance + if cache_key not in self._distance_cache: + self._distance_cache[cache_key] = min( + distances.get(target_chunk_id, self._maximum_distance), + self._maximum_distance, + ) + return self._distance_cache[cache_key] def neighborhood(self, chunk_id: str, radius: float) -> tuple[str, ...]: if not chunk_id: diff --git a/tests/fixtures/topiocqa_micro_sample.jsonl b/tests/fixtures/topiocqa_micro_sample.jsonl new file mode 100644 index 0000000..30017d4 --- /dev/null +++ b/tests/fixtures/topiocqa_micro_sample.jsonl @@ -0,0 +1,4 @@ +{"Conversation_no": 1, "Turn_no": 1, "Question": "What is orbital mechanics?", "Gold_passage": {"id": "orbit-1", "title": "Orbital mechanics", "text": "Orbital mechanics studies spacecraft motion."}, "Additional_answers": [{"Answer": "Gravity determines an orbit.", "Topic": "Gravity"}]} +{"Conversation_no": 2, "Turn_no": 1, "Question": "What is photosynthesis?", "Gold_passage": {"id": "plant-1", "title": "Photosynthesis", "text": "Plants convert light into chemical energy."}, "Additional_answers": []} +{"Conversation_no": 1, "Turn_no": 2, "Question": "How are transfers performed?", "Gold_passage": {"id": "orbit-2", "title": "Orbital mechanics", "text": "A transfer orbit moves between trajectories."}, "Additional_answers": []} +{"Conversation_no": 3, "Turn_no": 1, "Question": "What is plate tectonics?", "Gold_passage": {"id": "earth-1", "title": "Plate tectonics", "text": "Tectonic plates move across Earth."}, "Additional_answers": []} diff --git a/tests/test_eval_baselines.py b/tests/test_eval_baselines.py index 5a2d806..fe02700 100644 --- a/tests/test_eval_baselines.py +++ b/tests/test_eval_baselines.py @@ -9,6 +9,27 @@ from mapmatched.eval.loaders.synthetic import load_synthetic_fixture +def test_brute_force_provider_caches_query_embeddings() -> None: + class CountingEmbedder: + def __init__(self) -> None: + self.query_count = 0 + + def embed_query(self, query: str) -> tuple[float, float]: + del query + self.query_count += 1 + return (1.0, 0.0) + + embedder = CountingEmbedder() + provider = BruteForceProvider( + ("first", "second"), + ((1.0, 0.0), (0.0, 1.0)), + embedder, + ) + provider.candidates("same query", limit=2) + provider.candidates("same query", limit=1) + assert embedder.query_count == 1 + + def test_history_concat_changes_query_sequence() -> None: _, passages = load_synthetic_fixture() embedder = DeterministicHashEmbedder() diff --git a/tests/test_eval_bootstrap.py b/tests/test_eval_bootstrap.py index d271b83..90ee1c0 100644 --- a/tests/test_eval_bootstrap.py +++ b/tests/test_eval_bootstrap.py @@ -1,10 +1,23 @@ -from mapmatched.eval.bootstrap import bootstrap_ndcg_at_3_ci, bootstrap_slice_cis +import pytest + +from mapmatched.eval.bootstrap import ( + bootstrap_ndcg_at_3_ci, + bootstrap_paired_ndcg_at_3_delta_ci, + bootstrap_slice_cis, +) from mapmatched.eval.types import SliceName, TurnMetrics -def _turn(ndcg_at_3: float, slice_name: SliceName) -> TurnMetrics: +def _turn( + ndcg_at_3: float, + slice_name: SliceName, + *, + conversation_id: str = "conversation-1", + turn_index: int = 0, +) -> TurnMetrics: return TurnMetrics( - turn_index=0, + conversation_id=conversation_id, + turn_index=turn_index, ndcg_at_3=ndcg_at_3, ndcg_at_5=ndcg_at_3, recall_at_k=ndcg_at_3, @@ -53,6 +66,94 @@ def test_bootstrap_slice_cis_returns_all_slices() -> None: assert set(cis) == {"follow_up", "standalone", "all"} assert cis["follow_up"] is not None assert cis["follow_up"][0] <= cis["follow_up"][1] + assert cis["all"] is not None + assert cis["all"] != (0.0, 0.0) + + +def test_paired_bootstrap_positive_delta_excludes_zero() -> None: + treatment = ( + (_turn(1.0, "follow_up", conversation_id="conversation-1"),), + (_turn(0.8, "follow_up", conversation_id="conversation-2"),), + ) + baseline = ( + (_turn(0.2, "follow_up", conversation_id="conversation-1"),), + (_turn(0.3, "follow_up", conversation_id="conversation-2"),), + ) + ci = bootstrap_paired_ndcg_at_3_delta_ci( + treatment, + baseline, + "follow_up", + num_samples=200, + seed=7, + ) + assert ci is not None + assert ci[0] > 0.0 + + +def test_paired_bootstrap_zero_delta_is_exact() -> None: + treatment = ( + (_turn(0.2, "standalone", conversation_id="conversation-1"),), + (_turn(0.8, "standalone", conversation_id="conversation-2"),), + ) + ci = bootstrap_paired_ndcg_at_3_delta_ci( + treatment, + treatment, + "standalone", + num_samples=100, + seed=11, + ) + assert ci == (0.0, 0.0) + + +def test_paired_bootstrap_negative_delta_excludes_zero() -> None: + treatment = ( + (_turn(0.1, "follow_up", conversation_id="conversation-1"),), + (_turn(0.2, "follow_up", conversation_id="conversation-2"),), + ) + baseline = ( + (_turn(0.8, "follow_up", conversation_id="conversation-1"),), + (_turn(0.9, "follow_up", conversation_id="conversation-2"),), + ) + ci = bootstrap_paired_ndcg_at_3_delta_ci( + treatment, + baseline, + "follow_up", + num_samples=100, + seed=3, + ) + assert ci is not None + assert ci[1] < 0.0 + + +def test_paired_bootstrap_rejects_misaligned_conversations() -> None: + treatment = ((_turn(1.0, "follow_up", conversation_id="conversation-1"),),) + baseline = ((_turn(1.0, "follow_up", conversation_id="conversation-2"),),) + with pytest.raises(ValueError, match="conversation IDs do not match"): + bootstrap_paired_ndcg_at_3_delta_ci( + treatment, + baseline, + "follow_up", + num_samples=10, + ) + + +def test_paired_bootstrap_ignores_conversations_without_requested_slice() -> None: + treatment = ( + (_turn(0.8, "follow_up", conversation_id="conversation-1"),), + (_turn(0.1, "standalone", conversation_id="conversation-2"),), + ) + baseline = ( + (_turn(0.3, "follow_up", conversation_id="conversation-1"),), + (_turn(0.9, "standalone", conversation_id="conversation-2"),), + ) + ci = bootstrap_paired_ndcg_at_3_delta_ci( + treatment, + baseline, + "follow_up", + num_samples=50, + seed=5, + ) + assert ci == (0.5, 0.5) def test_synthetic_eval_populates_bootstrap_cis() -> None: @@ -92,3 +193,4 @@ def test_synthetic_eval_populates_bootstrap_cis() -> None: ) assert follow_up.ndcg_at_3_ci is not None assert follow_up.ndcg_at_3_ci[0] <= follow_up.ndcg_at_3 <= follow_up.ndcg_at_3_ci[1] + assert report.methods[0].turns[0].conversation_id diff --git a/tests/test_eval_gemini_rewrite.py b/tests/test_eval_gemini_rewrite.py new file mode 100644 index 0000000..9f6cf8c --- /dev/null +++ b/tests/test_eval_gemini_rewrite.py @@ -0,0 +1,128 @@ +import json +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from mapmatched.eval import DeterministicHashEmbedder, EvalConfig, MethodSpec, run_eval +from mapmatched.eval.__main__ import main +from mapmatched.eval.baselines import ( + GeminiQueryRewriter, + GeminiRewriteConfig, + MapMatchedMethodConfig, + create_gemini_query_rewriter, +) +from mapmatched.eval.loaders.synthetic import load_synthetic_fixture + + +class RecordingQueryRewriter: + def __init__(self) -> None: + self.calls: list[tuple[tuple[str, ...], str]] = [] + + def rewrite(self, *, history: Sequence[str], query: str) -> str: + self.calls.append((tuple(history), query)) + return " ".join([*history, query]) + + +def test_gemini_query_rewriter_sends_history_and_returns_text() -> None: + requests: list[dict[str, object]] = [] + + class Response: + text = " standalone query " + + def generate_content(**request: object) -> object: + requests.append(request) + return Response() + + rewriter = GeminiQueryRewriter( + generate_content=generate_content, + config=GeminiRewriteConfig(model="test-model"), + ) + rewritten = rewriter.rewrite(history=("first question",), query="what about it?") + + assert rewritten == "standalone query" + assert requests[0]["model"] == "test-model" + assert requests[0]["config"] == {"thinking_config": {"thinking_level": "minimal"}} + assert '"prior_user_utterances": ["first question"]' in str(requests[0]["contents"]) + + +def test_create_gemini_query_rewriter_requires_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + with pytest.raises(ValueError, match="GEMINI_API_KEY"): + create_gemini_query_rewriter() + + +def test_gemini_rewrite_runs_end_to_end_with_paired_comparison() -> None: + conversations, passages = load_synthetic_fixture() + embedder = DeterministicHashEmbedder() + rewriter = RecordingQueryRewriter() + report = run_eval( + conversations=conversations, + passages=passages, + embedder=embedder, + methods=( + MethodSpec(name="pointwise", transition_weight=0.0), + MethodSpec(name="gemini_rewrite"), + ), + config=MapMatchedMethodConfig(candidate_limit=4), + eval_config=EvalConfig( + benchmark="synthetic", + tier="synthetic", + embedder_name=embedder.name, + recall_k=10, + entropy_threshold=None, + standalone_tolerance=0.02, + follow_up_min_delta=0.0, + bootstrap_samples=20, + bootstrap_seed=42, + ), + query_rewriter=rewriter, + ) + + expected_turn_count = sum(len(conversation.turns) for conversation in conversations) + assert len(rewriter.calls) == expected_turn_count + assert rewriter.calls[0][0] == () + assert rewriter.calls[1][0] == (conversations[0].turns[0].query,) + comparison = next( + comparison + for comparison in report.comparisons + if comparison.method_name == "gemini_rewrite" + ) + assert all(slice_metrics.ndcg_at_3_delta_ci is not None for slice_metrics in comparison.slices) + + +def test_cli_records_gemini_rewrite_metadata( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + rewriter = RecordingQueryRewriter() + monkeypatch.setattr( + "mapmatched.eval.__main__.create_gemini_query_rewriter", + lambda *, model: rewriter, + ) + output_path = tmp_path / "report.json" + + exit_code = main( + [ + "--benchmark", + "synthetic", + "--tier", + "synthetic", + "--include-gemini-rewrite", + "--gemini-model", + "test-model", + "--candidate-limit", + "4", + "--output", + str(output_path), + ] + ) + + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert exit_code == 0 + assert payload["config"]["query_rewrite_provider"] == "gemini" + assert payload["config"]["query_rewrite_model"] == "test-model" + assert payload["config"]["query_rewrite_prompt_version"] == "cast-standalone-v1" + assert any(method["method_name"] == "gemini_rewrite" for method in payload["methods"]) diff --git a/tests/test_eval_report.py b/tests/test_eval_report.py index 23c9bc2..42edbbf 100644 --- a/tests/test_eval_report.py +++ b/tests/test_eval_report.py @@ -1,5 +1,12 @@ from mapmatched.eval.report import render_json, render_markdown_table -from mapmatched.eval.types import EvalConfig, EvalReport, MethodMetrics, SliceMetrics +from mapmatched.eval.types import ( + ComparisonSliceMetrics, + EvalConfig, + EvalReport, + MethodComparison, + MethodMetrics, + SliceMetrics, +) def test_report_json_and_markdown_are_deterministic() -> None: @@ -27,11 +34,42 @@ def test_report_json_and_markdown_are_deterministic() -> None: ), turns=(), ), + MethodMetrics( + method_name="mapmatched", + transition_weight=0.5, + candidate_limit=20, + fixed_lag=None, + graph_mode="knn", + slices=( + SliceMetrics("follow_up", 2, 0.5, 0.5, 0.5), + SliceMetrics("standalone", 1, 0.8, 0.8, 0.8), + SliceMetrics("all", 3, 0.6, 0.6, 0.6), + ), + turns=(), + ), ), verdict=None, + comparisons=( + MethodComparison( + method_name="mapmatched", + transition_weight=0.5, + baseline_method_name="pointwise", + baseline_transition_weight=0.0, + candidate_limit=20, + fixed_lag=None, + graph_mode="knn", + slices=( + ComparisonSliceMetrics("follow_up", 2, 0.1, (0.02, 0.18)), + ComparisonSliceMetrics("standalone", 1, 0.0, (0.0, 0.0)), + ComparisonSliceMetrics("all", 3, 0.067, (0.01, 0.12)), + ), + ), + ), ) json_payload = render_json(report) markdown = render_markdown_table(report) assert '"benchmark": "synthetic"' in json_payload assert "| synthetic | follow_up | pointwise |" in markdown assert "nDCG@3 95% CI" in markdown + assert "+0.100 [+0.020, +0.180]" in markdown + assert '"ndcg_at_3_delta_ci": [' in json_payload diff --git a/tests/test_eval_slices.py b/tests/test_eval_slices.py index b644cce..ccbffcd 100644 --- a/tests/test_eval_slices.py +++ b/tests/test_eval_slices.py @@ -61,8 +61,8 @@ def test_evaluate_claim_checks_follow_up_and_standalone_gates() -> None: def test_aggregate_slice_metrics_skips_other_slices() -> None: turns = ( - TurnMetrics(0, 1.0, 1.0, 1.0, 0.0, "standalone"), - TurnMetrics(1, 0.5, 0.5, 0.5, 1.0, "follow_up"), + TurnMetrics("conversation-1", 0, 1.0, 1.0, 1.0, 0.0, "standalone"), + TurnMetrics("conversation-1", 1, 0.5, 0.5, 0.5, 1.0, "follow_up"), ) follow_up = aggregate_slice_metrics(turns, "follow_up") assert follow_up.turn_count == 1 diff --git a/tests/test_eval_synthetic_runner.py b/tests/test_eval_synthetic_runner.py index 5ff9a7b..631a56e 100644 --- a/tests/test_eval_synthetic_runner.py +++ b/tests/test_eval_synthetic_runner.py @@ -31,9 +31,21 @@ def test_synthetic_eval_runs_end_to_end() -> None: entropy_threshold=None, standalone_tolerance=0.02, follow_up_min_delta=0.0, + bootstrap_samples=50, + bootstrap_seed=42, ), ) assert report.methods markdown = render_markdown_table(report) assert "Benchmark results" in markdown assert "pointwise" in markdown + assert report.comparisons + mapmatched_comparison = next( + comparison for comparison in report.comparisons if comparison.method_name == "mapmatched" + ) + follow_up = next( + slice_metrics + for slice_metrics in mapmatched_comparison.slices + if slice_metrics.slice_name == "follow_up" + ) + assert follow_up.ndcg_at_3_delta_ci is not None diff --git a/tests/test_eval_topiocqa_loader.py b/tests/test_eval_topiocqa_loader.py new file mode 100644 index 0000000..0621998 --- /dev/null +++ b/tests/test_eval_topiocqa_loader.py @@ -0,0 +1,99 @@ +import hashlib +import json +from pathlib import Path + +from mapmatched.eval import DeterministicHashEmbedder, EvalConfig, MethodSpec, run_eval +from mapmatched.eval.__main__ import main +from mapmatched.eval.baselines import MapMatchedMethodConfig +from mapmatched.eval.loaders import load_topiocqa_micro + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "topiocqa_micro_sample.jsonl" + + +def test_topiocqa_loader_limits_conversations_in_file_order() -> None: + conversations, passages = load_topiocqa_micro( + conversation_limit=2, + data_path=FIXTURE_PATH, + ) + assert tuple(conversation.conversation_id for conversation in conversations) == ( + "topiocqa-1", + "topiocqa-2", + ) + assert tuple(len(conversation.turns) for conversation in conversations) == (2, 1) + assert {passage.passage_id for passage in passages} == { + "orbit-1", + "topiocqa-extra-1-0-1", + "orbit-2", + "plant-1", + } + assert conversations[0].turns[0].qrels == { + "orbit-1": 3, + "topiocqa-extra-1-0-1": 2, + } + + +def test_topiocqa_fixture_runs_end_to_end_offline() -> None: + conversations, passages = load_topiocqa_micro( + conversation_limit=2, + data_path=FIXTURE_PATH, + ) + embedder = DeterministicHashEmbedder() + report = run_eval( + conversations=conversations, + passages=passages, + embedder=embedder, + methods=( + MethodSpec(name="pointwise", transition_weight=0.0), + MethodSpec(name="mapmatched", transition_weight=0.5), + ), + config=MapMatchedMethodConfig(candidate_limit=4), + eval_config=EvalConfig( + benchmark="topiocqa", + tier="micro", + embedder_name=embedder.name, + recall_k=4, + entropy_threshold=None, + standalone_tolerance=0.02, + follow_up_min_delta=0.0, + bootstrap_samples=20, + bootstrap_seed=42, + conversation_ids=tuple(conversation.conversation_id for conversation in conversations), + ), + ) + assert report.comparisons + assert report.config.conversation_ids == ("topiocqa-1", "topiocqa-2") + assert all(turn.conversation_id for method in report.methods for turn in method.turns) + + +def test_topiocqa_cli_records_reproduction_metadata(tmp_path: Path) -> None: + output_path = tmp_path / "report.json" + exit_code = main( + [ + "--profile", + "fixture-profile", + "--benchmark", + "topiocqa", + "--data-path", + str(FIXTURE_PATH), + "--conversation-limit", + "2", + "--knn-neighbors", + "2", + "--candidate-limit", + "4", + "--bootstrap-samples", + "10", + "--output", + str(output_path), + ] + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + expected_sha256 = hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest() + assert exit_code == 0 + assert payload["config"]["profile"] == "fixture-profile" + assert payload["config"]["dataset_filename"] == FIXTURE_PATH.name + assert payload["config"]["dataset_sha256"] == expected_sha256 + assert payload["config"]["conversation_ids"] == ["topiocqa-1", "topiocqa-2"] + assert payload["config"]["knn_neighbor_count"] == 2 + assert payload["config"]["package_version"] == "0.1.0" + assert payload["config"]["git_revision"] diff --git a/tests/test_graph_and_scoring.py b/tests/test_graph_and_scoring.py index 07496d1..78a13a3 100644 --- a/tests/test_graph_and_scoring.py +++ b/tests/test_graph_and_scoring.py @@ -45,6 +45,28 @@ def test_directed_graph_and_shortest_path_cutoff() -> None: assert graph.distance("a", "c") == 1.5 +def test_distance_reuses_single_source_shortest_paths() -> None: + class CountingGraph(InMemoryCorpusGraph): + def __init__(self) -> None: + super().__init__( + [ + GraphEdge("a", "b", 1.0), + GraphEdge("b", "c", 1.0), + ] + ) + self.search_count = 0 + + def _bounded_distances(self, source: str, cutoff: float) -> dict[str, float]: + self.search_count += 1 + return super()._bounded_distances(source, cutoff) + + graph = CountingGraph() + assert graph.distance("a", "b") == 1.0 + assert graph.distance("a", "c") == 2.0 + assert graph.distance("c", "a") == 2.0 + assert graph.search_count == 1 + + @pytest.mark.parametrize( ("method", "expected"), [