fix(search): rank the word as typed above its stem variants

README.md

@@ -560,7 +560,7 @@ markdown frontmatter filters. They compose freely.
 
 | You type                        | What it does                                                                              |
 | ------------------------------- | ----------------------------------------------------------------------------------------- |
-| `invoice`                       | One term. Searches body, title, headings, and filename. Stemmed (`invoice` = `invoices`). |
+| `invoice`                       | One term in body, title, headings and filename. Stemmed; `invoice` outweighs `invoices`.  |
 | `holiday budget plan`           | Several terms, ranked. Docs matching more terms rank higher; all-term docs reach the top. |
 | `holiday AND budget`            | Require both terms.                                                                       |
 | `"follow up email"`             | Exact phrase, in order. Also matches `follow-up email`.                                   |

fnd/analysis.py

@@ -79,11 +79,16 @@ def index_token(word: str) -> str:
     return "".join(tokens) if tokens else word.lower()
 
 
-@lru_cache(maxsize=65536)
-def word_token(word: str) -> str:
-    """One word as the index's word field stores it: folded and lowercased, unstemmed."""
+def word_tokens(text: str) -> list[str]:
+    """``text``'s tokens as the index's word field stores them: folded, lowercased, unstemmed."""
     analyser = getattr(_LOCAL, "words", None)
     if analyser is None:
         analyser = _LOCAL.words = _build(stem=False)
-    tokens = analyser.analyze(word)
+    return analyser.analyze(text)
+
+
+@lru_cache(maxsize=65536)
+def word_token(word: str) -> str:
+    """One word as the index's word field stores it: folded and lowercased, unstemmed."""
+    tokens = word_tokens(word)
     return "".join(tokens) if tokens else word.lower()

fnd/query.py

@@ -743,6 +743,7 @@ def _build_query(
         from fnd.query_dsl import preprocess
         from fnd.query_filters import extract_filters
         from fnd.schema import (
+            EXACT_FORM_SHARE,
             F_BODY,
             F_HEADING_PATH,
             F_PATH_TOKENS,
@@ -790,7 +791,11 @@ def _build_query(
             node = parse_query_ast(content, alternatives=alternatives) if has_content else None
             if has_content:
                 body_required = compile_query(
-                    node, searcher=self, schema=schema, parse_kwargs=body_parse_kwargs
+                    node,
+                    searcher=self,
+                    schema=schema,
+                    parse_kwargs=body_parse_kwargs,
+                    exact_form_share=EXACT_FORM_SHARE,
                 )
             elif filters:
                 body_required = tantivy.Query.all_query()

fnd/query_compile.py

@@ -19,7 +19,7 @@
 import tantivy
 from tantivy import Occur, Query
 
-from fnd.analysis import fold, index_token
+from fnd.analysis import fold, index_token, word_tokens
 from fnd.query_ast import (
     Alternatives,
     And,
@@ -66,22 +66,29 @@ def compile_query(
     searcher: Searcher,
     schema: tantivy.Schema,
     parse_kwargs: dict[str, object],
+    exact_form_share: float = 0.0,
 ) -> Query:
     """Compile ``node`` against ``searcher``'s index. ``parse_kwargs`` carries the
     body ``default_field_names`` (and any cascade ``fuzzy_fields``) used for plain
-    term/phrase leaves so the auto-fuzzy pass still reaches them."""
+    term/phrase leaves so the auto-fuzzy pass still reaches them. A plain-text leaf
+    reads ``exact_form_share`` of its weight from its words as typed (F_WORDS)."""
     if node is None:
         return Query.empty_query()
-    return _Compiler(searcher, schema, parse_kwargs).compile(node)
+    return _Compiler(searcher, schema, parse_kwargs, exact_form_share).compile(node)
 
 
 class _Compiler:
     def __init__(
-        self, searcher: Searcher, schema: tantivy.Schema, parse_kwargs: dict[str, object]
+        self,
+        searcher: Searcher,
+        schema: tantivy.Schema,
+        parse_kwargs: dict[str, object],
+        exact_form_share: float,
     ) -> None:
         self._s = searcher
         self._schema = schema
         self._pk = parse_kwargs
+        self._share = exact_form_share
 
     def compile(self, n: Node) -> Query:
         if isinstance(n, Term):
@@ -157,14 +164,30 @@ def _term(self, text: str) -> Query:
             return compiled
         # Whitespace inside an atom comes from a quoted value (``TODO:"a b"``).
         escaped = literal_phrase(text) if any(ch.isspace() for ch in text) else literal(text)
-        return _parse_query(self._s._index, escaped, **self._pk)
+        return self._exact_form(_parse_query(self._s._index, escaped, **self._pk), text)
 
     def _phrase(self, n: Phrase) -> Query:
         words = n.text.split()
         if any(is_phrase_pattern(w) for w in words):
             # parse_query drops ``*`` and ``~``, so this is a positional regex phrase on F_BODY.
             return self._pattern_phrase(words, n.slop)
-        return _parse_query(self._s._index, literal_phrase(n.text, slop=n.slop), **self._pk)
+        parsed = _parse_query(self._s._index, literal_phrase(n.text, slop=n.slop), **self._pk)
+        return self._exact_form(parsed, n.text)
+
+    def _exact_form(self, stems: Query, text: str) -> Query:
+        """``stems`` matching as before, with the share of its weight moved to chunks
+        holding ``text`` as typed. One word only: F_WORDS has no positions to read a
+        phrase by, and its words apart would outscore the phrase itself."""
+        tokens = word_tokens(text)
+        if not self._share or len(tokens) != 1:
+            return stems
+        typed = Query.term_query(self._schema, F_WORDS, tokens[0])
+        return Query.boolean_query(
+            [
+                (Occur.Must, Query.boost_query(stems, 1.0 - self._share)),
+                (Occur.Should, Query.boost_query(typed, self._share)),
+            ]
+        )
 
     def _pattern_phrase(self, words: list[str], slop: int) -> Query:
         if len(words) == 1:

fnd/schema.py

@@ -123,6 +123,10 @@
     F_PATH_TOKENS: 0.25,
     F_BODY: 1.0,
 }
+# A plain body word's weight: this share from its form as typed (F_WORDS), the rest its stem.
+# NQ page nDCG@10 +0.004, +0.004, +0.007 (full, 3-word, 2-word questions); full questions'
+# deeper-section MRR -0.006 (p 0.03). BEIR level; NFCorpus strong-signal 16 to 12 of 323.
+EXACT_FORM_SHARE: Final = 0.5
 # Within a file, a query word's heading BM25 times this counts where it beats the body's
 # BM25 for the word. NQ right section first, full questions: 1 gives 0.380, 2 0.444,
 # 3 0.436, 5 0.425.

tests/test_exact_form_outranks_stem_variants.py

@@ -0,0 +1,105 @@
+"""A chunk holding the word as typed outranks one holding only a stem variant of it."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from fnd.index import build_index
+from fnd.layered import search_layered
+from fnd.query import Searcher
+
+_PAD = "The quick brown fox jumps over the lazy dog near the river bank today. " * 3
+
+
+def _searcher(
+    tmp_path: Path, index_dir: Path, files: dict[str, str], titles: dict[str, str] | None = None
+) -> Searcher:
+    """One single-chunk file per entry, equal padding round each body."""
+    root = tmp_path / "notes"
+    root.mkdir()
+    for name, body in files.items():
+        title = (titles or {}).get(name, "Notes")
+        (root / f"{name}.md").write_text(
+            f"---\ntitle: {title}\n---\n# Notes\n\n{_PAD}{body}. {_PAD}\n"
+        )
+    build_index(roots=[root], index_dir=index_dir, collection="c")
+    return Searcher(index_dir=index_dir)
+
+
+def _rank(searcher: Searcher, query: str) -> list[str]:
+    return [Path(g.path).stem for g in search_layered(searcher, query=query, limit=10)]
+
+
+def _scores(searcher: Searcher, query: str) -> dict[str, float]:
+    """Each file's raw score for ``query``."""
+    pool = searcher._candidates(query, window=50, collection=None)  # pyright: ignore[reportPrivateUsage]
+    return {Path(h.path).stem: h.score for h in pool.hits}
+
+
+# Each variant file holds its form twice to the literal's once: stemming alone ranks them first.
+_FILES = {
+    "a_variant": "collections and collections",
+    "b_literal": "collect()",
+    "c_variant": "collected and collected",
+}
+
+
+@pytest.mark.parametrize("query", ["collect", "collect()", '"collect()"', "COLLECT"])
+def test_the_word_as_typed_ranks_above_its_stem_variants(
+    tmp_path: Path, tmp_index_dir: Path, query: str
+) -> None:
+    assert _rank(_searcher(tmp_path, tmp_index_dir, _FILES), query)[0] == "b_literal"
+
+
+def test_a_typed_variant_ranks_above_the_stem_it_shares(
+    tmp_path: Path, tmp_index_dir: Path
+) -> None:
+    files = {"a_typed": "collections", "c_variant": "collected and collected"}
+    assert _rank(_searcher(tmp_path, tmp_index_dir, files), "collections")[0] == "a_typed"
+
+
+def test_stem_variants_still_match(tmp_path: Path, tmp_index_dir: Path) -> None:
+    assert set(_rank(_searcher(tmp_path, tmp_index_dir, _FILES), "collect")) == set(_FILES)
+
+
+def test_a_pattern_beside_a_plain_word_keeps_its_whole_weight(
+    tmp_path: Path, tmp_index_dir: Path
+) -> None:
+    s = _searcher(tmp_path, tmp_index_dir, {"w": "cryptography cryptography", "p": "invoice"})
+    for pattern in ("crypt*", "cryptografy~"):
+        assert _scores(s, f"{pattern} invoice")["w"] == pytest.approx(_scores(s, pattern)["w"])
+
+
+def test_a_boost_scales_the_word_as_typed_too(tmp_path: Path, tmp_index_dir: Path) -> None:
+    s = _searcher(tmp_path, tmp_index_dir, {"a": "collect", "b": "collections collections"})
+    for name in ("a", "b"):
+        assert _scores(s, "collect^3")[name] == pytest.approx(3 * _scores(s, "collect")[name])
+
+
+def test_a_word_scoped_to_another_field_earns_no_body_credit(
+    tmp_path: Path, tmp_index_dir: Path
+) -> None:
+    files = {"x": "baz", "y": "baz bar bar bar"}
+    s = _searcher(tmp_path, tmp_index_dir, files, titles={"x": "foo bar"})
+    assert _scores(s, 'title:"foo bar" OR baz')["y"] == pytest.approx(_scores(s, "baz")["y"])
+
+
+def test_a_colon_that_names_no_field_reads_as_text(tmp_path: Path, tmp_index_dir: Path) -> None:
+    s = _searcher(tmp_path, tmp_index_dir, {"d": "TODO: fix it", "e": "fix it later"})
+    assert _scores(s, 'TODO:"fix it"') == pytest.approx(_scores(s, '"TODO fix it"'))
+
+
+def test_a_negated_phrase_rewards_none_of_its_words(tmp_path: Path, tmp_index_dir: Path) -> None:
+    """The kept chunk holds the phrase's words apart, so only the phrase is excluded."""
+    s = _searcher(tmp_path, tmp_index_dir, {"g": "gather collect the data", "h": "gather"})
+    assert _scores(s, 'gather NOT "collect data"') == pytest.approx(_scores(s, "gather"))
+
+
+def test_a_phrase_ranks_by_the_phrase_not_its_words_apart(
+    tmp_path: Path, tmp_index_dir: Path
+) -> None:
+    files = {"a": "follow up email", "b": "follow up email. " + "email " * 15}
+    scores = _scores(_searcher(tmp_path, tmp_index_dir, files), '"follow up email"')
+    assert scores["a"] > scores["b"]