diff --git a/README.md b/README.md index cf8ce91a..bae804b1 100644 --- a/README.md +++ b/README.md @@ -560,7 +560,7 @@ markdown frontmatter filters. They compose freely. | You type | What it does | | ------------------------------- | ----------------------------------------------------------------------------------------- | -| `invoice` | One term. Searches body, title, headings, and filename. Stemmed (`invoice` = `invoices`). | +| `invoice` | One term in body, title, headings and filename. Stemmed; `invoice` outweighs `invoices`. | | `holiday budget plan` | Several terms, ranked. Docs matching more terms rank higher; all-term docs reach the top. | | `holiday AND budget` | Require both terms. | | `"follow up email"` | Exact phrase, in order. Also matches `follow-up email`. | diff --git a/fnd/analysis.py b/fnd/analysis.py index 7d043da3..0ccf0b60 100644 --- a/fnd/analysis.py +++ b/fnd/analysis.py @@ -79,11 +79,16 @@ def index_token(word: str) -> str: return "".join(tokens) if tokens else word.lower() -@lru_cache(maxsize=65536) -def word_token(word: str) -> str: - """One word as the index's word field stores it: folded and lowercased, unstemmed.""" +def word_tokens(text: str) -> list[str]: + """``text``'s tokens as the index's word field stores them: folded, lowercased, unstemmed.""" analyser = getattr(_LOCAL, "words", None) if analyser is None: analyser = _LOCAL.words = _build(stem=False) - tokens = analyser.analyze(word) + return analyser.analyze(text) + + +@lru_cache(maxsize=65536) +def word_token(word: str) -> str: + """One word as the index's word field stores it: folded and lowercased, unstemmed.""" + tokens = word_tokens(word) return "".join(tokens) if tokens else word.lower() diff --git a/fnd/query.py b/fnd/query.py index 4062036b..745d66df 100644 --- a/fnd/query.py +++ b/fnd/query.py @@ -743,6 +743,7 @@ def _build_query( from fnd.query_dsl import preprocess from fnd.query_filters import extract_filters from fnd.schema import ( + EXACT_FORM_SHARE, F_BODY, F_HEADING_PATH, F_PATH_TOKENS, @@ -790,7 +791,11 @@ def _build_query( node = parse_query_ast(content, alternatives=alternatives) if has_content else None if has_content: body_required = compile_query( - node, searcher=self, schema=schema, parse_kwargs=body_parse_kwargs + node, + searcher=self, + schema=schema, + parse_kwargs=body_parse_kwargs, + exact_form_share=EXACT_FORM_SHARE, ) elif filters: body_required = tantivy.Query.all_query() diff --git a/fnd/query_compile.py b/fnd/query_compile.py index 701d3282..7ebc801c 100644 --- a/fnd/query_compile.py +++ b/fnd/query_compile.py @@ -19,7 +19,7 @@ import tantivy from tantivy import Occur, Query -from fnd.analysis import fold, index_token +from fnd.analysis import fold, index_token, word_tokens from fnd.query_ast import ( Alternatives, And, @@ -66,22 +66,29 @@ def compile_query( searcher: Searcher, schema: tantivy.Schema, parse_kwargs: dict[str, object], + exact_form_share: float = 0.0, ) -> Query: """Compile ``node`` against ``searcher``'s index. ``parse_kwargs`` carries the body ``default_field_names`` (and any cascade ``fuzzy_fields``) used for plain - term/phrase leaves so the auto-fuzzy pass still reaches them.""" + term/phrase leaves so the auto-fuzzy pass still reaches them. A plain-text leaf + reads ``exact_form_share`` of its weight from its words as typed (F_WORDS).""" if node is None: return Query.empty_query() - return _Compiler(searcher, schema, parse_kwargs).compile(node) + return _Compiler(searcher, schema, parse_kwargs, exact_form_share).compile(node) class _Compiler: def __init__( - self, searcher: Searcher, schema: tantivy.Schema, parse_kwargs: dict[str, object] + self, + searcher: Searcher, + schema: tantivy.Schema, + parse_kwargs: dict[str, object], + exact_form_share: float, ) -> None: self._s = searcher self._schema = schema self._pk = parse_kwargs + self._share = exact_form_share def compile(self, n: Node) -> Query: if isinstance(n, Term): @@ -157,14 +164,30 @@ def _term(self, text: str) -> Query: return compiled # Whitespace inside an atom comes from a quoted value (``TODO:"a b"``). escaped = literal_phrase(text) if any(ch.isspace() for ch in text) else literal(text) - return _parse_query(self._s._index, escaped, **self._pk) + return self._exact_form(_parse_query(self._s._index, escaped, **self._pk), text) def _phrase(self, n: Phrase) -> Query: words = n.text.split() if any(is_phrase_pattern(w) for w in words): # parse_query drops ``*`` and ``~``, so this is a positional regex phrase on F_BODY. return self._pattern_phrase(words, n.slop) - return _parse_query(self._s._index, literal_phrase(n.text, slop=n.slop), **self._pk) + parsed = _parse_query(self._s._index, literal_phrase(n.text, slop=n.slop), **self._pk) + return self._exact_form(parsed, n.text) + + def _exact_form(self, stems: Query, text: str) -> Query: + """``stems`` matching as before, with the share of its weight moved to chunks + holding ``text`` as typed. One word only: F_WORDS has no positions to read a + phrase by, and its words apart would outscore the phrase itself.""" + tokens = word_tokens(text) + if not self._share or len(tokens) != 1: + return stems + typed = Query.term_query(self._schema, F_WORDS, tokens[0]) + return Query.boolean_query( + [ + (Occur.Must, Query.boost_query(stems, 1.0 - self._share)), + (Occur.Should, Query.boost_query(typed, self._share)), + ] + ) def _pattern_phrase(self, words: list[str], slop: int) -> Query: if len(words) == 1: diff --git a/fnd/schema.py b/fnd/schema.py index a6e687db..691f9d83 100644 --- a/fnd/schema.py +++ b/fnd/schema.py @@ -123,6 +123,10 @@ F_PATH_TOKENS: 0.25, F_BODY: 1.0, } +# A plain body word's weight: this share from its form as typed (F_WORDS), the rest its stem. +# NQ page nDCG@10 +0.004, +0.004, +0.007 (full, 3-word, 2-word questions); full questions' +# deeper-section MRR -0.006 (p 0.03). BEIR level; NFCorpus strong-signal 16 to 12 of 323. +EXACT_FORM_SHARE: Final = 0.5 # Within a file, a query word's heading BM25 times this counts where it beats the body's # BM25 for the word. NQ right section first, full questions: 1 gives 0.380, 2 0.444, # 3 0.436, 5 0.425. diff --git a/tests/test_exact_form_outranks_stem_variants.py b/tests/test_exact_form_outranks_stem_variants.py new file mode 100644 index 00000000..60205903 --- /dev/null +++ b/tests/test_exact_form_outranks_stem_variants.py @@ -0,0 +1,105 @@ +"""A chunk holding the word as typed outranks one holding only a stem variant of it.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from fnd.index import build_index +from fnd.layered import search_layered +from fnd.query import Searcher + +_PAD = "The quick brown fox jumps over the lazy dog near the river bank today. " * 3 + + +def _searcher( + tmp_path: Path, index_dir: Path, files: dict[str, str], titles: dict[str, str] | None = None +) -> Searcher: + """One single-chunk file per entry, equal padding round each body.""" + root = tmp_path / "notes" + root.mkdir() + for name, body in files.items(): + title = (titles or {}).get(name, "Notes") + (root / f"{name}.md").write_text( + f"---\ntitle: {title}\n---\n# Notes\n\n{_PAD}{body}. {_PAD}\n" + ) + build_index(roots=[root], index_dir=index_dir, collection="c") + return Searcher(index_dir=index_dir) + + +def _rank(searcher: Searcher, query: str) -> list[str]: + return [Path(g.path).stem for g in search_layered(searcher, query=query, limit=10)] + + +def _scores(searcher: Searcher, query: str) -> dict[str, float]: + """Each file's raw score for ``query``.""" + pool = searcher._candidates(query, window=50, collection=None) # pyright: ignore[reportPrivateUsage] + return {Path(h.path).stem: h.score for h in pool.hits} + + +# Each variant file holds its form twice to the literal's once: stemming alone ranks them first. +_FILES = { + "a_variant": "collections and collections", + "b_literal": "collect()", + "c_variant": "collected and collected", +} + + +@pytest.mark.parametrize("query", ["collect", "collect()", '"collect()"', "COLLECT"]) +def test_the_word_as_typed_ranks_above_its_stem_variants( + tmp_path: Path, tmp_index_dir: Path, query: str +) -> None: + assert _rank(_searcher(tmp_path, tmp_index_dir, _FILES), query)[0] == "b_literal" + + +def test_a_typed_variant_ranks_above_the_stem_it_shares( + tmp_path: Path, tmp_index_dir: Path +) -> None: + files = {"a_typed": "collections", "c_variant": "collected and collected"} + assert _rank(_searcher(tmp_path, tmp_index_dir, files), "collections")[0] == "a_typed" + + +def test_stem_variants_still_match(tmp_path: Path, tmp_index_dir: Path) -> None: + assert set(_rank(_searcher(tmp_path, tmp_index_dir, _FILES), "collect")) == set(_FILES) + + +def test_a_pattern_beside_a_plain_word_keeps_its_whole_weight( + tmp_path: Path, tmp_index_dir: Path +) -> None: + s = _searcher(tmp_path, tmp_index_dir, {"w": "cryptography cryptography", "p": "invoice"}) + for pattern in ("crypt*", "cryptografy~"): + assert _scores(s, f"{pattern} invoice")["w"] == pytest.approx(_scores(s, pattern)["w"]) + + +def test_a_boost_scales_the_word_as_typed_too(tmp_path: Path, tmp_index_dir: Path) -> None: + s = _searcher(tmp_path, tmp_index_dir, {"a": "collect", "b": "collections collections"}) + for name in ("a", "b"): + assert _scores(s, "collect^3")[name] == pytest.approx(3 * _scores(s, "collect")[name]) + + +def test_a_word_scoped_to_another_field_earns_no_body_credit( + tmp_path: Path, tmp_index_dir: Path +) -> None: + files = {"x": "baz", "y": "baz bar bar bar"} + s = _searcher(tmp_path, tmp_index_dir, files, titles={"x": "foo bar"}) + assert _scores(s, 'title:"foo bar" OR baz')["y"] == pytest.approx(_scores(s, "baz")["y"]) + + +def test_a_colon_that_names_no_field_reads_as_text(tmp_path: Path, tmp_index_dir: Path) -> None: + s = _searcher(tmp_path, tmp_index_dir, {"d": "TODO: fix it", "e": "fix it later"}) + assert _scores(s, 'TODO:"fix it"') == pytest.approx(_scores(s, '"TODO fix it"')) + + +def test_a_negated_phrase_rewards_none_of_its_words(tmp_path: Path, tmp_index_dir: Path) -> None: + """The kept chunk holds the phrase's words apart, so only the phrase is excluded.""" + s = _searcher(tmp_path, tmp_index_dir, {"g": "gather collect the data", "h": "gather"}) + assert _scores(s, 'gather NOT "collect data"') == pytest.approx(_scores(s, "gather")) + + +def test_a_phrase_ranks_by_the_phrase_not_its_words_apart( + tmp_path: Path, tmp_index_dir: Path +) -> None: + files = {"a": "follow up email", "b": "follow up email. " + "email " * 15} + scores = _scores(_searcher(tmp_path, tmp_index_dir, files), '"follow up email"') + assert scores["a"] > scores["b"]