From e6683f790b05619347f5e9b8a807669b8e8e442e Mon Sep 17 00:00:00 2001 From: Kris Wong Date: Fri, 25 Sep 2026 14:59:23 -0500 Subject: [PATCH 1/2] fix(code-index): measure autosync against the baseline the index recorded; keep a worktree's Zoekt snapshot in its own git dir Co-Authored-By: lemoncrow <302591943+lemoncrow-agent[bot]@users.noreply.github.com> LemonCrow-Session: s1 LemonCrow-Model: claude-opus-5-5 --- .../infra/code_intel/zoekt/indexer.py | 25 ++--- .../pro/capabilities/code_context/engine.py | 86 ++++++++++++++--- tests/core/test_code_context.py | 95 ++++++++++++++++++- .../code_intel/test_zoekt_indexer_worktree.py | 53 +++++++++++ 4 files changed, 231 insertions(+), 28 deletions(-) create mode 100644 tests/infra/code_intel/test_zoekt_indexer_worktree.py diff --git a/src/lemoncrow/infra/code_intel/zoekt/indexer.py b/src/lemoncrow/infra/code_intel/zoekt/indexer.py index 544d89107..1b69517f1 100644 --- a/src/lemoncrow/infra/code_intel/zoekt/indexer.py +++ b/src/lemoncrow/infra/code_intel/zoekt/indexer.py @@ -9,6 +9,8 @@ from time import time from typing import Any +from .server import _git_dirs, _read_git_head + _TEXT_SUFFIXES = { ".py", ".ts", @@ -91,22 +93,19 @@ def index_age_seconds(self) -> int: snapshot = self.ensure_snapshot() return int(max(0, time() - snapshot.indexed_at)) - def _snapshot_cache_path(self) -> Path: - return self.repo_root / ".git" / "lemoncrow" / "zoekt_snapshot.json" + def _snapshot_cache_path(self) -> Path | None: + # The checkout's own git admin directory: a linked worktree's `.git` is a + # file, and its admin directory goes away with `git worktree remove`. + dirs = _git_dirs(self.repo_root) + return None if dirs is None else dirs[0] / "lemoncrow" / "zoekt_snapshot.json" def _current_head(self) -> str | None: - head_file = self.repo_root / ".git" / "HEAD" - try: - ref = head_file.read_text(encoding="utf-8").strip() - if ref.startswith("ref: "): - ref_path = self.repo_root / ".git" / ref[5:] - return ref_path.read_text(encoding="utf-8").strip() - return ref - except OSError: - return None + return _read_git_head(self.repo_root) def _load_snapshot_from_disk(self) -> ZoektIndexSnapshot | None: cache_path = self._snapshot_cache_path() + if cache_path is None: + return None try: data = json.loads(cache_path.read_text(encoding="utf-8")) stored_head = data.get("head") @@ -114,11 +113,13 @@ def _load_snapshot_from_disk(self) -> ZoektIndexSnapshot | None: if current_head is None or stored_head != current_head: return None return ZoektIndexSnapshot.from_dict(data["snapshot"]) - except (FileNotFoundError, json.JSONDecodeError, KeyError, TypeError, ValueError): + except (OSError, KeyError, TypeError, ValueError): return None def _save_snapshot_to_disk(self, snapshot: ZoektIndexSnapshot) -> None: cache_path = self._snapshot_cache_path() + if cache_path is None: + return current_head = self._current_head() payload = json.dumps({"head": current_head, "snapshot": snapshot.to_dict()}) tmp = cache_path.with_suffix(".tmp") diff --git a/src/lemoncrow/pro/capabilities/code_context/engine.py b/src/lemoncrow/pro/capabilities/code_context/engine.py index e10b5dc59..d9e1731c5 100644 --- a/src/lemoncrow/pro/capabilities/code_context/engine.py +++ b/src/lemoncrow/pro/capabilities/code_context/engine.py @@ -2673,6 +2673,14 @@ def _resolve_index_max_workers() -> int: _AUTOSYNC_GIT_HEAD_TIMEOUT_S = 3.0 +@dataclass(frozen=True) +class _IndexedBaseline: + """The tree signature and git HEAD an index run started from; None where unrecorded.""" + + signature: str | None = None + head: str | None = None + + def _resolve_autosync_index_max_workers() -> int: """Worker count for background autosync indexing. @@ -4357,6 +4365,9 @@ def _index_repo_unsafe( ) -> IndexStats: """Unlocked inner — callers must hold ``self._autosync_lock``.""" self.db_path.parent.mkdir(parents=True, exist_ok=True) + # Read before the scan, so a change landing mid-scan differs from the recorded + # baseline and the next autosync check reindexes it rather than taking it as done. + baseline = _IndexedBaseline(self._source_tree_signature(), self._autosync_git_head()) all_files = [ path for path in iter_source_files( @@ -4608,6 +4619,7 @@ def _index_repo_unsafe( with self._connect() as conn: self._init_schema(conn) self._stamp_indexer_semantics_version(conn) + self._record_indexed_baseline(conn, baseline) # Compute+persist the centrality map for the new index_version NOW so the # O(edges) power iteration is charged to indexing, never to the first @@ -4687,6 +4699,44 @@ def _stamp_indexer_semantics_version(self, conn: sqlite3.Connection) -> None: (str(_CODE_INDEXER_SEMANTICS_VERSION),), ) + def _indexed_baseline_key(self) -> str: + # Per repo: a shared db_path holds several repos' indexes. + return f"autosync_baseline:{self.repo_id}" + + def _record_indexed_baseline(self, conn: sqlite3.Connection, baseline: _IndexedBaseline) -> None: + conn.execute( + """ + INSERT INTO engine_state(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (self._indexed_baseline_key(), json.dumps({"signature": baseline.signature, "head": baseline.head})), + ) + + def _indexed_baseline(self) -> _IndexedBaseline: + """What the last completed index run of this repo started from. + + The autosync loop measures change against this, not against its own first + look at the tree: an engine is replaced whenever the index version moves, and + a replacement that took the tree as it found it would treat every change made + since the last index run as already indexed. + """ + try: + with self._connect() as conn: + self._init_schema(conn) + row = conn.execute( + "SELECT value FROM engine_state WHERE key = ?", (self._indexed_baseline_key(),) + ).fetchone() + data = json.loads(str(row["value"])) if row is not None else {} + except (sqlite3.Error, ValueError): + return _IndexedBaseline() + if not isinstance(data, dict): + return _IndexedBaseline() + signature, head = data.get("signature"), data.get("head") + return _IndexedBaseline( + signature if isinstance(signature, str) and signature else None, + head if isinstance(head, str) and head else None, + ) + def _delete_files_index(self, conn: sqlite3.Connection, rels: list[str]) -> None: """Remove every indexed row for the files *rels*, in one pass per table. @@ -14929,7 +14979,7 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> self._autosync_state = "idle" self._record_autosync_event(event="reindex", reason=known_change, reindexed=False) return False - self._autosync_signature = self._source_tree_signature() + self._autosync_signature = self._indexed_baseline().signature or self._source_tree_signature() self._autosync_last_sync_ms = int(time.time() * 1000) self._autosync_pending_events = 0 self._autosync_state = "idle" @@ -14938,13 +14988,20 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> return True current_signature = self._source_tree_signature() + reason = "source_signature_changed" if self._autosync_signature is None: - self._autosync_signature = current_signature - self._autosync_last_sync_ms = int(time.time() * 1000) - self._autosync_state = "idle" - self._record_autosync_event(event="bootstrap", reason="seed_signature", reindexed=False) - return False - if current_signature == self._autosync_signature: + recorded = self._indexed_baseline().signature + if recorded is None: + # Indexed before baselines were recorded: nothing says what it covers. + reason = "no_indexed_baseline" + else: + self._autosync_signature = recorded + if current_signature == recorded: + self._autosync_last_sync_ms = int(time.time() * 1000) + self._autosync_state = "idle" + self._record_autosync_event(event="bootstrap", reason="indexed_baseline", reindexed=False) + return False + elif current_signature == self._autosync_signature: self._autosync_state = "idle" self._autosync_pending_events = 0 self._record_autosync_event(event="full_check", reason="unchanged", reindexed=False) @@ -14961,14 +15018,14 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> # Reindex failed; leave the signature/pending state stale so the next # poll retries instead of recording a failed sync as complete. self._autosync_state = "idle" - self._record_autosync_event(event="reindex", reason="source_signature_changed", reindexed=False) + self._record_autosync_event(event="reindex", reason=reason, reindexed=False) return False - self._autosync_signature = self._source_tree_signature() + self._autosync_signature = self._indexed_baseline().signature or self._source_tree_signature() self._autosync_last_sync_ms = int(time.time() * 1000) self._autosync_pending_events = 0 self._autosync_state = "idle" self._autosync_reindex_count += 1 - self._record_autosync_event(event="reindex", reason="source_signature_changed", reindexed=True) + self._record_autosync_event(event="reindex", reason=reason, reindexed=True) return True def _maybe_refresh_zoekt_index(self) -> None: @@ -15205,15 +15262,16 @@ def _autosync_tick(self, now_ms: int) -> None: def _autosync_poll_locked(self, now_ms: int) -> None: head = self._autosync_git_head() - if head is not None and self._autosync_head is not None and head != self._autosync_head: + if head is not None and self._autosync_head is None: + # Like the tree signature: start from the HEAD the index was built at. + self._autosync_head = self._indexed_baseline().head or head + if head is not None and head != self._autosync_head: # On failure keep the old HEAD, so the next tick retries. if self._maybe_autosync_reindex_locked(known_change="head_moved"): - self._autosync_head = head + self._autosync_head = self._indexed_baseline().head or head # The reindex reseeded the tree signature: that is a full check. self._autosync_last_full_check_ms = now_ms return - if head is not None: - self._autosync_head = head last = self._autosync_last_full_check_ms if last is None or now_ms - last >= max(self._autosync_poll_ms, _AUTOSYNC_MIN_POLL_MS): self._autosync_last_full_check_ms = now_ms diff --git a/tests/core/test_code_context.py b/tests/core/test_code_context.py index 46af4677a..949d31f58 100644 --- a/tests/core/test_code_context.py +++ b/tests/core/test_code_context.py @@ -19,7 +19,7 @@ CallGraphNode, traverse_call_graph, ) -from lemoncrow.pro.capabilities.code_context.engine import _CODE_INDEXER_SEMANTICS_VERSION +from lemoncrow.pro.capabilities.code_context.engine import _CODE_INDEXER_SEMANTICS_VERSION, _IndexedBaseline from lemoncrow.pro.capabilities.code_context.models import SymbolRecord, TextMatch from lemoncrow.pro.capabilities.code_context.output_policy import TRUNCATION_MARKER from lemoncrow.pro.code_intel.cross_lang.runner import CrossLangRunner @@ -2166,10 +2166,20 @@ def signature() -> str: self.tree_walks += 1 return real_signature() + def record_baseline() -> None: + # What a completed index run leaves behind; the walk is the subprocess's, not the loop's. + with engine._connect() as conn: + engine._init_schema(conn) + engine._record_indexed_baseline(conn, _IndexedBaseline(real_signature(), engine._autosync_git_head())) + def run_index_subprocess(*, force: bool = False) -> bool: self.reindexes.append(self.tree_walks) - return reindex_results.pop(0) if reindex_results else True + ok = reindex_results.pop(0) if reindex_results else True + if ok: + record_baseline() + return ok + record_baseline() monkeypatch.setattr(engine, "_source_tree_signature", signature) monkeypatch.setattr(engine, "_run_index_subprocess", run_index_subprocess) monkeypatch.setattr(engine, "index_ready", lambda: True) @@ -2324,6 +2334,87 @@ def is_alive(self) -> bool: assert probe.reindexes == [] +def _indexed_git_fixture(tmp_path: Path) -> CodeContextEngine: + repo = tmp_path / "repo" + _init_git_fixture_repo(repo) + _write_fixture_repo(repo) + _commit_all(repo, "initial") + engine = CodeContextEngine(repo, db_path=tmp_path / "code.sqlite", autosync_enabled=False) + engine.index_repo() + return engine + + +def _replacement_engine(engine: CodeContextEngine) -> CodeContextEngine: + """What the engine cache builds once the index version moves: a new engine on the same index.""" + return CodeContextEngine(engine.repo_root, db_path=engine.db_path, autosync_enabled=False) + + +def _finds(engine: CodeContextEngine, name: str) -> bool: + return any(s.symbol_name == name for s in engine.search_symbols(name, mode="lexical", limit=5, auto_index=False)) + + +def test_a_replacement_engine_indexes_a_commit_made_since_the_last_index_run(tmp_path: Path) -> None: + engine = _indexed_git_fixture(tmp_path) + (engine.repo_root / "src" / "orders.py").write_text("class CommittedBeforeItLooked:\n pass\n", encoding="utf-8") + _commit_all(engine.repo_root, "move HEAD") + + replacement = _replacement_engine(engine) + replacement._autosync_tick(0) + + assert _finds(replacement, "CommittedBeforeItLooked") + assert replacement._autosync_history[-1]["reason"] == "head_moved" + + +def test_a_replacement_engine_indexes_a_working_tree_change_made_since_the_last_index_run(tmp_path: Path) -> None: + engine = _indexed_git_fixture(tmp_path) + (engine.repo_root / "src" / "orders.py").write_text("class EditedBeforeItLooked:\n pass\n", encoding="utf-8") + + replacement = _replacement_engine(engine) + replacement._autosync_tick(0) + + assert _finds(replacement, "EditedBeforeItLooked") + assert replacement._autosync_history[-1]["reason"] == "source_signature_changed" + + +def test_a_change_made_during_an_index_run_is_reindexed_by_the_next_check( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo = tmp_path / "repo" + _init_git_fixture_repo(repo) + _write_fixture_repo(repo) + _commit_all(repo, "initial") + engine = CodeContextEngine(repo, db_path=tmp_path / "code.sqlite", autosync_enabled=False) + real_extract = engine._parallel_extract + + def extract_while_the_tree_changes(*args: object, **kwargs: object) -> object: + results = real_extract(*args, **kwargs) # type: ignore[arg-type] + (repo / "src" / "orders.py").write_text("class WrittenMidScan:\n pass\n", encoding="utf-8") + return results + + monkeypatch.setattr(engine, "_parallel_extract", extract_while_the_tree_changes) + engine.index_repo() + monkeypatch.undo() + assert not _finds(engine, "WrittenMidScan") + + replacement = _replacement_engine(engine) + replacement._autosync_tick(0) + + assert _finds(replacement, "WrittenMidScan") + + +def test_an_index_without_a_recorded_baseline_is_reindexed_on_the_first_check( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + engine, probe = _autosync_probe_engine(tmp_path, monkeypatch) + with engine._connect() as conn: + conn.execute("DELETE FROM engine_state WHERE key = ?", (engine._indexed_baseline_key(),)) + + engine._autosync_tick(0) + + assert len(probe.reindexes) == 1 + assert engine._autosync_history[-1]["reason"] == "no_indexed_baseline" + + def test_incremental_index_noop_does_not_bump_version(tmp_path: Path) -> None: _write_fixture_repo(tmp_path) engine = CodeContextEngine(tmp_path, db_path=tmp_path / "code.sqlite") diff --git a/tests/infra/code_intel/test_zoekt_indexer_worktree.py b/tests/infra/code_intel/test_zoekt_indexer_worktree.py new file mode 100644 index 000000000..388acd9e9 --- /dev/null +++ b/tests/infra/code_intel/test_zoekt_indexer_worktree.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from lemoncrow.infra.code_intel.zoekt.indexer import ZoektIndexer + + +def _git(cwd: Path, *args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=cwd, check=True, capture_output=True, text=True + ).stdout.strip() + + +@pytest.fixture +def linked_worktree(tmp_path: Path) -> Path: + main = tmp_path / "main" + main.mkdir() + _git(main, "init", "-q", "-b", "main") + (main / "app.py").write_text("a = 1\nb = 2\n", encoding="utf-8") + _git(main, "add", "-A") + _git(main, "commit", "-qm", "init") + worktree = tmp_path / "wt" + _git(main, "worktree", "add", "-q", "-b", "wt", str(worktree)) + return worktree + + +def test_a_linked_worktree_keeps_its_snapshot_in_its_own_git_admin_directory( + linked_worktree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A linked worktree's `.git` is a file; reading a snapshot under it raised + # NotADirectoryError and killed the snapshot thread on every engine start. + first = ZoektIndexer(linked_worktree) + assert first.ensure_snapshot().total_lines == 2 + + admin_dir = Path(_git(linked_worktree, "rev-parse", "--absolute-git-dir")) + assert (admin_dir / "lemoncrow" / "zoekt_snapshot.json").is_file() + + def no_rebuild(self: ZoektIndexer) -> None: + raise AssertionError("the saved snapshot for this HEAD should be read back, not rebuilt") + + monkeypatch.setattr(ZoektIndexer, "_build_snapshot", no_rebuild) + assert ZoektIndexer(linked_worktree).ensure_snapshot().total_lines == 2 + + +def test_a_linked_worktree_snapshot_is_rebuilt_once_its_head_moves(linked_worktree: Path) -> None: + ZoektIndexer(linked_worktree).ensure_snapshot() + (linked_worktree / "app.py").write_text("a = 1\nb = 2\nc = 3\n", encoding="utf-8") + _git(linked_worktree, "commit", "-qam", "grow") + + assert ZoektIndexer(linked_worktree).ensure_snapshot().total_lines == 3 From 4e3b9d1b30c847594685f918a9d5b811102ff7ea Mon Sep 17 00:00:00 2001 From: Kris Wong Date: Fri, 25 Sep 2026 15:17:44 -0500 Subject: [PATCH 2/2] fix(code-index): a reindex that records no baseline measures from the tree as it is, not a stale baseline Addresses cr-70588 bha_p0_f0: a seeded worktree index the run declines to rebuild kept main's baseline, so autosync re-ran the index subprocess every tick. Co-Authored-By: Claude Opus 5.5 (1M context) Co-Authored-By: lemoncrow <302591943+lemoncrow-agent[bot]@users.noreply.github.com> LemonCrow-Session: 90d6a214-e1bd-4377-b621-531411b973b6 LemonCrow-Model: claude-opus-5-5 --- .../pro/capabilities/code_context/engine.py | 24 +++++++++++++++---- tests/core/test_code_context.py | 17 +++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/lemoncrow/pro/capabilities/code_context/engine.py b/src/lemoncrow/pro/capabilities/code_context/engine.py index d9e1731c5..de200ba49 100644 --- a/src/lemoncrow/pro/capabilities/code_context/engine.py +++ b/src/lemoncrow/pro/capabilities/code_context/engine.py @@ -14973,13 +14973,12 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> """ if known_change is not None: self._autosync_state = "syncing" - if not self._run_index_subprocess(): + if not self._reindex_and_adopt_baseline(): # Reindex failed; leave the signature/pending state stale so the # next poll retries instead of recording a failed sync as done. self._autosync_state = "idle" self._record_autosync_event(event="reindex", reason=known_change, reindexed=False) return False - self._autosync_signature = self._indexed_baseline().signature or self._source_tree_signature() self._autosync_last_sync_ms = int(time.time() * 1000) self._autosync_pending_events = 0 self._autosync_state = "idle" @@ -15014,13 +15013,12 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> self._record_autosync_event(event="change_detected", reason="within_debounce_window", reindexed=False) return False self._autosync_state = "syncing" - if not self._run_index_subprocess(): + if not self._reindex_and_adopt_baseline(): # Reindex failed; leave the signature/pending state stale so the next # poll retries instead of recording a failed sync as complete. self._autosync_state = "idle" self._record_autosync_event(event="reindex", reason=reason, reindexed=False) return False - self._autosync_signature = self._indexed_baseline().signature or self._source_tree_signature() self._autosync_last_sync_ms = int(time.time() * 1000) self._autosync_pending_events = 0 self._autosync_state = "idle" @@ -15028,6 +15026,23 @@ def _maybe_autosync_reindex_locked(self, *, known_change: str | None = None) -> self._record_autosync_event(event="reindex", reason=reason, reindexed=True) return True + def _reindex_and_adopt_baseline(self) -> bool: + """Run the index subprocess; on success, measure from the baseline it recorded. + + A run can succeed without recording one -- a seeded worktree index it declines + to rebuild still carries the main checkout's baseline. Adopting that would + re-detect the same change and reindex again on every tick, so a run that left + the baseline untouched falls back to the tree and HEAD as they are now. + """ + before = self._indexed_baseline() + if not self._run_index_subprocess(): + return False + after = self._indexed_baseline() + recorded = after if after != before else _IndexedBaseline() + self._autosync_signature = recorded.signature or self._source_tree_signature() + self._autosync_head = recorded.head or self._autosync_git_head() or self._autosync_head + return True + def _maybe_refresh_zoekt_index(self) -> None: """Keep the git-repo Zoekt shard fresh at commit granularity. @@ -15268,7 +15283,6 @@ def _autosync_poll_locked(self, now_ms: int) -> None: if head is not None and head != self._autosync_head: # On failure keep the old HEAD, so the next tick retries. if self._maybe_autosync_reindex_locked(known_change="head_moved"): - self._autosync_head = self._indexed_baseline().head or head # The reindex reseeded the tree signature: that is a full check. self._autosync_last_full_check_ms = now_ms return diff --git a/tests/core/test_code_context.py b/tests/core/test_code_context.py index 949d31f58..85a6a2340 100644 --- a/tests/core/test_code_context.py +++ b/tests/core/test_code_context.py @@ -2402,6 +2402,23 @@ def extract_while_the_tree_changes(*args: object, **kwargs: object) -> object: assert _finds(replacement, "WrittenMidScan") +def test_a_reindex_that_records_no_baseline_is_not_rerun_on_every_tick( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + engine, _ = _autosync_probe_engine(tmp_path, monkeypatch, git=True) + # A seeded worktree index carries the main checkout's baseline, and an index run that + # declines to rebuild it (stale semantics, main not yet rebuilt) succeeds but records nothing. + with engine._connect() as conn: + engine._record_indexed_baseline(conn, _IndexedBaseline("main-signature", "0" * 40)) + runs: list[int] = [] + monkeypatch.setattr(engine, "_run_index_subprocess", lambda *, force=False: runs.append(1) or True) + + for minute in range(3): + engine._autosync_tick(minute * _AUTOSYNC_MINUTE_MS) + + assert len(runs) == 1 + + def test_an_index_without_a_recorded_baseline_is_reindexed_on_the_first_check( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: