From 3d1f664c2ac3c72085a867c5e9da4c30e44c8b86 Mon Sep 17 00:00:00 2001 From: Pringled Date: Fri, 18 Sep 2026 10:04:14 +0200 Subject: [PATCH 1/4] feat: Add idle TTL for MCP in-memory index cache Set SEMBLE_MCP_CACHE_TTL (seconds) to drop indexes unused for that long from memory; they reload from the disk cache on the next search. Closes #269 --- docs/installation.md | 2 ++ src/semble/mcp.py | 35 ++++++++++++++++++++++++++++++----- tests/test_mcp.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index fd4d74ed..d3b88110 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -329,6 +329,8 @@ The MCP server indexes each requested content selection on first use and caches claude mcp add semble -s user -- uvx --from "semble[mcp]" semble --content all ``` +Indexes stay in memory for the lifetime of the server. To free memory while idle, set `SEMBLE_MCP_CACHE_TTL` to a number of seconds; indexes unused for that long are dropped from memory and reloaded from the disk cache on the next search. + ### Instructions (AGENTS.md / CLAUDE.md) Add the snippet below to your `AGENTS.md` or `CLAUDE.md` so your agent knows when and how to call the semble CLI: diff --git a/src/semble/mcp.py b/src/semble/mcp.py index f03f7435..b9356091 100644 --- a/src/semble/mcp.py +++ b/src/semble/mcp.py @@ -3,6 +3,7 @@ import asyncio import json import logging +import os import time from collections import OrderedDict from collections.abc import Sequence @@ -64,6 +65,16 @@ def _resolve_content_selection( return (ContentType(content),) +def _idle_ttl() -> float: + """Read the idle TTL in seconds for in-memory indexes from SEMBLE_MCP_CACHE_TTL; 0 disables it.""" + value = os.getenv("SEMBLE_MCP_CACHE_TTL", "0") + try: + return max(float(value), 0.0) + except ValueError: + logger.warning("Ignoring invalid SEMBLE_MCP_CACHE_TTL: %r", value) + return 0.0 + + def create_server(cache: _IndexCache, default_content: Sequence[ContentType] = (ContentType.CODE,)) -> FastMCP: """Build and return a configured FastMCP server backed by the given cache.""" server = FastMCP( @@ -171,7 +182,7 @@ async def serve( content: Sequence[ContentType] = (ContentType.CODE,), ) -> None: """Start an MCP stdio server.""" - cache = _IndexCache() + cache = _IndexCache(idle_ttl=_idle_ttl()) async def _load_and_prewarm() -> None: """Pre-load the embedding model in parallel with starting the server.""" @@ -196,8 +207,10 @@ async def _load_and_prewarm() -> None: class _IndexCache: """Cache of indexed repos and local paths for the lifetime of the MCP server process.""" - def __init__(self) -> None: - """Initialise an empty cache.""" + def __init__(self, idle_ttl: float = 0.0) -> None: + """Initialise an empty cache; entries unused for `idle_ttl` seconds are dropped from memory (0 = never).""" + self._idle_ttl = idle_ttl + self._idle_timers: dict[_CacheKey, asyncio.TimerHandle] = {} self._model_path: str | None = None self._model_error: BaseException | None = None self._model_ready = asyncio.Event() @@ -263,6 +276,14 @@ def evict(self, cache_key: _CacheKey) -> None: """Evict one exact index variant from memory.""" self._tasks.pop(cache_key, None) self._revalidate_after.pop(cache_key, None) + if (timer := self._idle_timers.pop(cache_key, None)) is not None: + timer.cancel() + + def _evict_idle(self, cache_key: _CacheKey) -> None: + """Drop an entry that has not been accessed within the idle TTL; the disk cache is kept.""" + logger.info("Evicting idle index %r from memory", cache_key) + self.evict(cache_key) + self._merged = None # may hold a reference to the evicted index async def _evict_if_stale(self, cache_key: _CacheKey) -> None: """Evict a cached local-path entry whose on-disk cache no longer matches its files. @@ -305,10 +326,14 @@ async def get( # Re-check after the await: another caller may have populated the entry. if cache_key not in self._tasks: if len(self._tasks) >= _CACHE_MAX_SIZE: - evicted_key, _ = self._tasks.popitem(last=False) - self._revalidate_after.pop(evicted_key, None) + self.evict(next(iter(self._tasks))) self._tasks[cache_key] = asyncio.create_task(self._build_tracked(source, ref, model_path, cache_key)) self._tasks.move_to_end(cache_key) + if self._idle_ttl > 0: + if (timer := self._idle_timers.get(cache_key)) is not None: + timer.cancel() + loop = asyncio.get_running_loop() + self._idle_timers[cache_key] = loop.call_later(self._idle_ttl, self._evict_idle, cache_key) task = self._tasks[cache_key] try: return await asyncio.shield(task) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 22acf230..e0592b3b 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -9,7 +9,7 @@ import pytest from model2vec import StaticModel -from semble.mcp import _CACHE_MAX_SIZE, _IndexCache, create_server, serve +from semble.mcp import _CACHE_MAX_SIZE, _idle_ttl, _IndexCache, create_server, serve from semble.types import Chunk, ContentType, SearchResult from semble.utils import format_results, is_git_url, resolve_chunk from tests.conftest import make_chunk @@ -523,6 +523,34 @@ async def test_index_cache_lru_eviction(cache: _IndexCache, tmp_path: Path) -> N assert len(cache._tasks) == _CACHE_MAX_SIZE +@pytest.mark.anyio +async def test_index_cache_idle_ttl_eviction(cache: _IndexCache, tmp_path: Path) -> None: + """Entries are dropped from memory once unused for the idle TTL, and each access resets the timer.""" + cache._idle_ttl = 0.05 + key = cache._compute_cache_key(str(tmp_path)) + with patch("semble.mcp.SembleIndex.from_path", return_value=MagicMock()): + await cache.get(str(tmp_path)) + cache._merged = ([], MagicMock()) + await asyncio.sleep(0.03) + await cache.get(str(tmp_path)) + await asyncio.sleep(0.03) + assert key in cache._tasks + await asyncio.sleep(0.05) + assert key not in cache._tasks + assert key not in cache._idle_timers + assert cache._merged is None + + +@pytest.mark.parametrize(("value", "expected"), [(None, 0.0), ("300", 300.0), ("-1", 0.0), ("abc", 0.0)]) +def test_idle_ttl_from_env(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: float) -> None: + """SEMBLE_MCP_CACHE_TTL is parsed as seconds, with unset, negative, or invalid values disabling it.""" + if value is None: + monkeypatch.delenv("SEMBLE_MCP_CACHE_TTL", raising=False) + else: + monkeypatch.setenv("SEMBLE_MCP_CACHE_TTL", value) + assert _idle_ttl() == expected + + def test_cache_evict(cache: _IndexCache, tmp_path: Path) -> None: """evict() removes an existing exact cache entry.""" key = cache._compute_cache_key(str(tmp_path)) From 27b8895f5cb9795028c30c6fcac4dec9d54ec3a3 Mon Sep 17 00:00:00 2001 From: Pringled Date: Fri, 18 Sep 2026 10:07:19 +0200 Subject: [PATCH 2/4] refactor: Read SEMBLE_MCP_CACHE_TTL as a module constant and fix TTL test --- src/semble/mcp.py | 13 ++----------- tests/test_mcp.py | 17 +++++------------ 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/semble/mcp.py b/src/semble/mcp.py index b9356091..d0f13252 100644 --- a/src/semble/mcp.py +++ b/src/semble/mcp.py @@ -28,6 +28,7 @@ ) _CACHE_MAX_SIZE = 10 # Max number of cached indexes to keep in memory +_CACHE_IDLE_TTL = float(os.environ.get("SEMBLE_MCP_CACHE_TTL", 0)) # Idle seconds before dropping an index (0 = never) _MIN_REVALIDATE_FACTOR = 3 # Don't recheck staleness sooner than this many times the last build's duration ContentSelection = Literal["code", "docs", "config", "all"] _CacheKey = tuple[str, tuple[ContentType, ...]] @@ -65,16 +66,6 @@ def _resolve_content_selection( return (ContentType(content),) -def _idle_ttl() -> float: - """Read the idle TTL in seconds for in-memory indexes from SEMBLE_MCP_CACHE_TTL; 0 disables it.""" - value = os.getenv("SEMBLE_MCP_CACHE_TTL", "0") - try: - return max(float(value), 0.0) - except ValueError: - logger.warning("Ignoring invalid SEMBLE_MCP_CACHE_TTL: %r", value) - return 0.0 - - def create_server(cache: _IndexCache, default_content: Sequence[ContentType] = (ContentType.CODE,)) -> FastMCP: """Build and return a configured FastMCP server backed by the given cache.""" server = FastMCP( @@ -182,7 +173,7 @@ async def serve( content: Sequence[ContentType] = (ContentType.CODE,), ) -> None: """Start an MCP stdio server.""" - cache = _IndexCache(idle_ttl=_idle_ttl()) + cache = _IndexCache(idle_ttl=_CACHE_IDLE_TTL) async def _load_and_prewarm() -> None: """Pre-load the embedding model in parallel with starting the server.""" diff --git a/tests/test_mcp.py b/tests/test_mcp.py index e0592b3b..c831d296 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -9,7 +9,7 @@ import pytest from model2vec import StaticModel -from semble.mcp import _CACHE_MAX_SIZE, _idle_ttl, _IndexCache, create_server, serve +from semble.mcp import _CACHE_MAX_SIZE, _IndexCache, create_server, serve from semble.types import Chunk, ContentType, SearchResult from semble.utils import format_results, is_git_url, resolve_chunk from tests.conftest import make_chunk @@ -528,7 +528,10 @@ async def test_index_cache_idle_ttl_eviction(cache: _IndexCache, tmp_path: Path) """Entries are dropped from memory once unused for the idle TTL, and each access resets the timer.""" cache._idle_ttl = 0.05 key = cache._compute_cache_key(str(tmp_path)) - with patch("semble.mcp.SembleIndex.from_path", return_value=MagicMock()): + with ( + patch("semble.mcp.SembleIndex.from_path", return_value=MagicMock()), + patch("semble.mcp.get_validated_cache", return_value=MagicMock()), + ): await cache.get(str(tmp_path)) cache._merged = ([], MagicMock()) await asyncio.sleep(0.03) @@ -541,16 +544,6 @@ async def test_index_cache_idle_ttl_eviction(cache: _IndexCache, tmp_path: Path) assert cache._merged is None -@pytest.mark.parametrize(("value", "expected"), [(None, 0.0), ("300", 300.0), ("-1", 0.0), ("abc", 0.0)]) -def test_idle_ttl_from_env(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: float) -> None: - """SEMBLE_MCP_CACHE_TTL is parsed as seconds, with unset, negative, or invalid values disabling it.""" - if value is None: - monkeypatch.delenv("SEMBLE_MCP_CACHE_TTL", raising=False) - else: - monkeypatch.setenv("SEMBLE_MCP_CACHE_TTL", value) - assert _idle_ttl() == expected - - def test_cache_evict(cache: _IndexCache, tmp_path: Path) -> None: """evict() removes an existing exact cache entry.""" key = cache._compute_cache_key(str(tmp_path)) From ac4d0f8acbd22e81c3e0c17930f4c53a4102d93a Mon Sep 17 00:00:00 2001 From: Pringled Date: Fri, 18 Sep 2026 10:08:00 +0200 Subject: [PATCH 3/4] refactor: Simplify idle TTL wiring --- src/semble/mcp.py | 17 ++++++++--------- tests/test_mcp.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/semble/mcp.py b/src/semble/mcp.py index d0f13252..3bc79019 100644 --- a/src/semble/mcp.py +++ b/src/semble/mcp.py @@ -173,7 +173,7 @@ async def serve( content: Sequence[ContentType] = (ContentType.CODE,), ) -> None: """Start an MCP stdio server.""" - cache = _IndexCache(idle_ttl=_CACHE_IDLE_TTL) + cache = _IndexCache() async def _load_and_prewarm() -> None: """Pre-load the embedding model in parallel with starting the server.""" @@ -198,9 +198,8 @@ async def _load_and_prewarm() -> None: class _IndexCache: """Cache of indexed repos and local paths for the lifetime of the MCP server process.""" - def __init__(self, idle_ttl: float = 0.0) -> None: - """Initialise an empty cache; entries unused for `idle_ttl` seconds are dropped from memory (0 = never).""" - self._idle_ttl = idle_ttl + def __init__(self) -> None: + """Initialise an empty cache.""" self._idle_timers: dict[_CacheKey, asyncio.TimerHandle] = {} self._model_path: str | None = None self._model_error: BaseException | None = None @@ -272,7 +271,6 @@ def evict(self, cache_key: _CacheKey) -> None: def _evict_idle(self, cache_key: _CacheKey) -> None: """Drop an entry that has not been accessed within the idle TTL; the disk cache is kept.""" - logger.info("Evicting idle index %r from memory", cache_key) self.evict(cache_key) self._merged = None # may hold a reference to the evicted index @@ -320,11 +318,12 @@ async def get( self.evict(next(iter(self._tasks))) self._tasks[cache_key] = asyncio.create_task(self._build_tracked(source, ref, model_path, cache_key)) self._tasks.move_to_end(cache_key) - if self._idle_ttl > 0: - if (timer := self._idle_timers.get(cache_key)) is not None: + if _CACHE_IDLE_TTL > 0: + if timer := self._idle_timers.get(cache_key): timer.cancel() - loop = asyncio.get_running_loop() - self._idle_timers[cache_key] = loop.call_later(self._idle_ttl, self._evict_idle, cache_key) + self._idle_timers[cache_key] = asyncio.get_running_loop().call_later( + _CACHE_IDLE_TTL, self._evict_idle, cache_key + ) task = self._tasks[cache_key] try: return await asyncio.shield(task) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index c831d296..f41d3914 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -526,9 +526,9 @@ async def test_index_cache_lru_eviction(cache: _IndexCache, tmp_path: Path) -> N @pytest.mark.anyio async def test_index_cache_idle_ttl_eviction(cache: _IndexCache, tmp_path: Path) -> None: """Entries are dropped from memory once unused for the idle TTL, and each access resets the timer.""" - cache._idle_ttl = 0.05 key = cache._compute_cache_key(str(tmp_path)) with ( + patch("semble.mcp._CACHE_IDLE_TTL", 0.05), patch("semble.mcp.SembleIndex.from_path", return_value=MagicMock()), patch("semble.mcp.get_validated_cache", return_value=MagicMock()), ): From 30fae6827407e09314a098e98e8d74bf66e7ece0 Mon Sep 17 00:00:00 2001 From: Pringled Date: Fri, 18 Sep 2026 10:15:17 +0200 Subject: [PATCH 4/4] fix: Start idle timer after build and rename to SEMBLE_MCP_IDLE_TIMEOUT --- docs/installation.md | 2 +- src/semble/mcp.py | 22 +++++++++++++--------- tests/test_mcp.py | 20 +++++++++++++------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index d3b88110..07f05535 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -329,7 +329,7 @@ The MCP server indexes each requested content selection on first use and caches claude mcp add semble -s user -- uvx --from "semble[mcp]" semble --content all ``` -Indexes stay in memory for the lifetime of the server. To free memory while idle, set `SEMBLE_MCP_CACHE_TTL` to a number of seconds; indexes unused for that long are dropped from memory and reloaded from the disk cache on the next search. +Indexes stay in memory for the lifetime of the server. To free memory while idle, set `SEMBLE_MCP_IDLE_TIMEOUT` to a number of seconds; indexes unused for that long are dropped from memory and reloaded from the disk cache on the next search. ### Instructions (AGENTS.md / CLAUDE.md) diff --git a/src/semble/mcp.py b/src/semble/mcp.py index 3bc79019..13e35704 100644 --- a/src/semble/mcp.py +++ b/src/semble/mcp.py @@ -28,7 +28,9 @@ ) _CACHE_MAX_SIZE = 10 # Max number of cached indexes to keep in memory -_CACHE_IDLE_TTL = float(os.environ.get("SEMBLE_MCP_CACHE_TTL", 0)) # Idle seconds before dropping an index (0 = never) +_CACHE_IDLE_TIMEOUT = float( + os.environ.get("SEMBLE_MCP_IDLE_TIMEOUT", 0) +) # Idle seconds before dropping an index (0 = never) _MIN_REVALIDATE_FACTOR = 3 # Don't recheck staleness sooner than this many times the last build's duration ContentSelection = Literal["code", "docs", "config", "all"] _CacheKey = tuple[str, tuple[ContentType, ...]] @@ -270,7 +272,7 @@ def evict(self, cache_key: _CacheKey) -> None: timer.cancel() def _evict_idle(self, cache_key: _CacheKey) -> None: - """Drop an entry that has not been accessed within the idle TTL; the disk cache is kept.""" + """Drop an entry that has not been accessed within the idle timeout; the disk cache is kept.""" self.evict(cache_key) self._merged = None # may hold a reference to the evicted index @@ -318,15 +320,9 @@ async def get( self.evict(next(iter(self._tasks))) self._tasks[cache_key] = asyncio.create_task(self._build_tracked(source, ref, model_path, cache_key)) self._tasks.move_to_end(cache_key) - if _CACHE_IDLE_TTL > 0: - if timer := self._idle_timers.get(cache_key): - timer.cancel() - self._idle_timers[cache_key] = asyncio.get_running_loop().call_later( - _CACHE_IDLE_TTL, self._evict_idle, cache_key - ) task = self._tasks[cache_key] try: - return await asyncio.shield(task) + index = await asyncio.shield(task) except asyncio.CancelledError: # pragma: no cover if task.done(): self.evict(cache_key) @@ -336,3 +332,11 @@ async def get( if self._tasks.get(cache_key) is task: self.evict(cache_key) raise + # Start the idle timer only once the index is ready, so slow builds are not evicted mid-build. + if _CACHE_IDLE_TIMEOUT > 0 and self._tasks.get(cache_key) is task: + if timer := self._idle_timers.get(cache_key): + timer.cancel() + self._idle_timers[cache_key] = asyncio.get_running_loop().call_later( + _CACHE_IDLE_TIMEOUT, self._evict_idle, cache_key + ) + return index diff --git a/tests/test_mcp.py b/tests/test_mcp.py index f41d3914..faa81f7d 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -524,21 +524,27 @@ async def test_index_cache_lru_eviction(cache: _IndexCache, tmp_path: Path) -> N @pytest.mark.anyio -async def test_index_cache_idle_ttl_eviction(cache: _IndexCache, tmp_path: Path) -> None: - """Entries are dropped from memory once unused for the idle TTL, and each access resets the timer.""" +async def test_index_cache_idle_timeout_eviction(cache: _IndexCache, tmp_path: Path) -> None: + """Entries are dropped once unused for the idle timeout, which starts after the build and resets on access.""" + + def slow_build(*args: Any, **kwargs: Any) -> MagicMock: + time.sleep(0.15) # longer than the timeout + return MagicMock() + key = cache._compute_cache_key(str(tmp_path)) with ( - patch("semble.mcp._CACHE_IDLE_TTL", 0.05), - patch("semble.mcp.SembleIndex.from_path", return_value=MagicMock()), + patch("semble.mcp._CACHE_IDLE_TIMEOUT", 0.1), + patch("semble.mcp.SembleIndex.from_path", side_effect=slow_build), patch("semble.mcp.get_validated_cache", return_value=MagicMock()), ): await cache.get(str(tmp_path)) + assert key in cache._tasks + first_timer = cache._idle_timers[key] cache._merged = ([], MagicMock()) - await asyncio.sleep(0.03) await cache.get(str(tmp_path)) - await asyncio.sleep(0.03) + assert first_timer.cancelled() assert key in cache._tasks - await asyncio.sleep(0.05) + await asyncio.sleep(0.3) assert key not in cache._tasks assert key not in cache._idle_timers assert cache._merged is None