From 350d4caf8b1b062e7f563b7a2a5e3e0bb7b9d7f8 Mon Sep 17 00:00:00 2001 From: Oranje AI <293843428+oranjeai@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:01:06 +1200 Subject: [PATCH] memory: count only resolvable items in memU category totals ## Symptom `memory_expand_category` advertises more items than it can return, e.g. "Category 'procedures' - showing 20 of 68069 items" when only 65655 are reachable; paging past the reachable end yields nothing. The web memory sidebar badge is inflated the same way. Measured on a live 146030-item store: 6455 of 175114 membership rows are unreturnable, a 3.69% overstatement present in all 8 categories (3.29% to 4.29%). ## Root cause `memu_category_items` is a link table with no foreign key to `memu_memory_items`, so delete-side writers strand membership rows whose item is gone. Two read paths counted those rows directly: - `MemUBridge.expand_category` computed `total` with an unjoined `count(*)`, while the item listing three lines below already joined `memu_memory_items`. So `total` and `items` were computed over two different row sets in the same function and connection. - `_read_memu_snapshot_sync` exported unjoined `(category_id, item_id)` pairs as `category_items`, which the web sidebar counts via `ids.length`. The stranded rows are permanent, not in-flight: they were created between 2026-06-25 and 2026-08-02. ## The fix Add the listing's own join predicate at both sites, so the count and the listing agree by construction rather than by coincidence. Chosen over an equivalent `EXISTS` because it is textually the same join the function already performs, leaving no room for the two to drift. Both queries stay fully index-covered (`SCAN ci USING COVERING INDEX idx_sqlite_category_items_unique` + `SEARCH i USING COVERING INDEX ...`); measured cost is 72.2 -> 102.8 ms for the once-per-page-load snapshot (already in a worker thread) and 9.4 -> 12.9 ms for a per-category total. Reported counts decrease for existing users. That is the point: the new numbers are what the store can actually return. Two pre-existing and distinct mismatches are deliberately NOT addressed here: `total` ignores the `query` filter, and the sidebar badge counts all memory types while the Facts tab renders five of them. --- nerve/gateway/routes/memory.py | 8 +++- nerve/memory/memu_bridge.py | 6 ++- tests/test_recall_breadcrumbs.py | 68 +++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/nerve/gateway/routes/memory.py b/nerve/gateway/routes/memory.py index d38a3665..181a1632 100644 --- a/nerve/gateway/routes/memory.py +++ b/nerve/gateway/routes/memory.py @@ -167,9 +167,13 @@ def _read_memu_snapshot_sync(db_path: str) -> str: "created_at": row["created_at"], }) - # Category-item links + # Category-item links. The link table has no foreign key, so join the + # item table to skip relations whose item is gone. cat_items: dict[str, list[str]] = {} - for row in db.execute("SELECT category_id, item_id FROM memu_category_items"): + for row in db.execute( + "SELECT ci.category_id, ci.item_id FROM memu_category_items ci " + "JOIN memu_memory_items i ON i.id = ci.item_id" + ): cat_items.setdefault(row["category_id"], []).append(row["item_id"]) finally: db.close() diff --git a/nerve/memory/memu_bridge.py b/nerve/memory/memu_bridge.py index 9642175b..f8c51f91 100644 --- a/nerve/memory/memu_bridge.py +++ b/nerve/memory/memu_bridge.py @@ -2887,8 +2887,12 @@ def _run() -> dict[str, Any]: ).fetchone() if not crow: return {"name": None, "total": 0, "items": []} + # Same join as the listing below: the link table has no foreign + # key, so relations to deleted items must not be counted. total = db.execute( - "SELECT count(*) FROM memu_category_items WHERE category_id = ?", + "SELECT count(*) FROM memu_category_items ci " + "JOIN memu_memory_items i ON i.id = ci.item_id " + "WHERE ci.category_id = ?", (cat_id,), ).fetchone()[0] sql = ( diff --git a/tests/test_recall_breadcrumbs.py b/tests/test_recall_breadcrumbs.py index ff7a6504..25f0aba5 100644 --- a/tests/test_recall_breadcrumbs.py +++ b/tests/test_recall_breadcrumbs.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import sqlite3 from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -21,6 +22,7 @@ ) from nerve.agent.tools.registry import ToolContext from nerve.config import MemoryConfig, NerveConfig +from nerve.gateway.routes.memory import _read_memu_snapshot_sync from nerve.memory.memu_bridge import MemUBridge, _category_breadcrumb @@ -157,12 +159,15 @@ async def test_recall_caps_items_and_categories(tmp_path) -> None: def _create_category_schema(db_path: str) -> None: + # Covers both consumers: the bridge reads items/categories/links, the gateway + # snapshot additionally reads resource_id, happened_at and memu_resources. db = sqlite3.connect(db_path) db.executescript( """ CREATE TABLE memu_memory_items ( id TEXT PRIMARY KEY, memory_type TEXT, summary TEXT, - created_at TEXT, updated_at TEXT + created_at TEXT, updated_at TEXT, + resource_id TEXT, happened_at TEXT ); CREATE TABLE memu_memory_categories ( id TEXT PRIMARY KEY, name TEXT, description TEXT, summary TEXT @@ -170,6 +175,10 @@ def _create_category_schema(db_path: str) -> None: CREATE TABLE memu_category_items ( id TEXT PRIMARY KEY, item_id TEXT, category_id TEXT ); + CREATE TABLE memu_resources ( + id TEXT PRIMARY KEY, url TEXT, modality TEXT, caption TEXT, + created_at TEXT + ); """ ) db.commit() @@ -201,6 +210,24 @@ def _seed_category(db_path: str) -> None: db.close() +def _seed_dangling_relations(db_path: str) -> list[str]: + """Add membership rows whose item_id has no memu_memory_items row. + + The link table has no foreign key, so delete-side writers strand rows here. + Returns the dead item ids. + """ + dead = ["it-gone-1", "it-gone-2"] + db = sqlite3.connect(db_path) + for iid in dead: + db.execute( + "INSERT INTO memu_category_items (id, item_id, category_id) VALUES (?,?,?)", + (f"link-{iid}", iid, "cat-pref"), + ) + db.commit() + db.close() + return dead + + @pytest.mark.asyncio async def test_expand_category_returns_recent_items(tmp_path) -> None: config = _make_config(tmp_path) @@ -230,6 +257,45 @@ async def test_expand_category_keyword_filter(tmp_path) -> None: assert [i["id"] for i in result["items"]] == ["it-3"] +@pytest.mark.asyncio +async def test_expand_category_total_excludes_dangling_relations(tmp_path) -> None: + """total must equal what the store can actually return, not raw link rows.""" + config = _make_config(tmp_path) + db_path = config.memory.sqlite_dsn.replace("sqlite:///", "") + _create_category_schema(db_path) + _seed_category(db_path) + _seed_dangling_relations(db_path) + + bridge = _stub_bridge(config) + result = await bridge.expand_category("cat:cat-pref", limit=10) + + # 5 link rows exist, only 3 resolve to a live item. + assert result["total"] == 3 + # The invariant: unpaged, the advertised total is exactly what is listed. + assert result["total"] == len(result["items"]) + + +@pytest.mark.asyncio +async def test_memu_snapshot_category_items_excludes_dangling(tmp_path) -> None: + """The gateway snapshot must not export links to deleted items.""" + config = _make_config(tmp_path) + db_path = config.memory.sqlite_dsn.replace("sqlite:///", "") + _create_category_schema(db_path) + _seed_category(db_path) + dead = _seed_dangling_relations(db_path) + + payload = json.loads(_read_memu_snapshot_sync(db_path)) + ids = payload["category_items"]["cat-pref"] + + assert len(ids) == 3 + for dead_id in dead: + assert dead_id not in ids + # Cross-site agreement: the web badge and the agent's total must match. + bridge = _stub_bridge(config) + result = await bridge.expand_category("cat-pref", limit=10) + assert len(ids) == result["total"] + + @pytest.mark.asyncio async def test_expand_category_unknown_id(tmp_path) -> None: config = _make_config(tmp_path)