From b7c62269e779b77533f2d0c187b92684b8ed2e68 Mon Sep 17 00:00:00 2001 From: wefio <48851810+wefio@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:58:43 +0800 Subject: [PATCH 1/2] =?UTF-8?q?docs(design):=20record=20SimHash=20drift=20?= =?UTF-8?q?mechanism=20in=20tesserae=20design=20(=C2=A73.1/=C2=A73.3/?= =?UTF-8?q?=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/design/memory-tesserae-design.md | 37 +++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/design/memory-tesserae-design.md b/docs/design/memory-tesserae-design.md index a39bda2..97a8ac5 100644 --- a/docs/design/memory-tesserae-design.md +++ b/docs/design/memory-tesserae-design.md @@ -64,6 +64,11 @@ Rationale: maintaining a file index is expensive and its marginal value over tesserae + direct tool access is low. The file is the content host; NMG only needs *pointers into it*. +The design declared the drop; the **code removal is ticket 8** (the original +tesserae PR shipped with `file-index.ts` still live — every search still +crawled it). Removing the full-text machinery is prerequisite to the drift +tolerance below: tesserae need a file fingerprint, not a file index. + ### 3.2 Tesserae are an independent, searchable source A **tessera** (a bookmark) is a first-class row, not a field glued onto a @@ -93,16 +98,35 @@ memory is superseded), a matching tessera is still found. A tessera stores a **content snippet**, never a line number. Line numbers drift on every edit; content is relocatable. To resolve a tessera, locate its snippet -in the current file (exact match → position; fuzzy fallback → nearest match; -absent → tessera is stale). This is the established pattern from +in the current file (exact match → position; absent → tessera is stale). This +is the established pattern from [gptme hash-anchored editing](https://github.com/gptme/gptme/blob/ae707fc8233e77d4da97fc74f94db1eaff1e381a/gptme/tools/_anchored.py), [agentic-bookmarks self-healing anchors](https://github.com/super-mega-lab/agentic-bookmarks), and [haido `hash_at_link` drift detection](https://github.com/lebac-svg/haido/blob/HEAD/docs/DESIGN.md): never persist a position that the file can invalidate; persist content and relocate. -Staleness is **objective**: resolve on read; if the snippet no longer exists, -report the tessera as stale (memory stays valid — only the position is gone). +**Drift tolerance (ticket 8).** A tessera additionally stores a 64-bit SimHash +fingerprint of its target file (`tesserae.file_simhash`, computed once at write +time). Relocation is two-stage: + +1. **Exact** — locate the snippet line in `tessera.path` (`includes`). Hit → done. +2. **Fingerprint fallback** — exact miss does not immediately mean stale. Compare + the stored file SimHash against the current files in scope (all readable + files under the project). If one is within Hamming ≤ 6 — the same document + after small edits, or the file after a move — re-locate the snippet against + that candidate file. SimHash is document-level: measured on real repo files + (5–60 KB), near-identical pairs sit at Hamming 1–3 and unrelated at ~24, so + ≤ 6 cleanly separates (100% recall / 0.24% false positive), while short + memory/snippet text has no such signal and is never fingerprinted this way. + +The fingerprint finds a *candidate file*; the snippet match confirms the exact +position. The tessera row is never auto-rewritten — the caller decides whether +to update `path` after confirmation. + +Staleness is **objective**: resolve on read; if the snippet no longer exists +anywhere the fingerprint points (exact or fallback), report the tessera as +stale (memory stays valid — only the position is gone). ### 3.4 Markers are the index pointer between memory and tessera @@ -171,7 +195,10 @@ conclusions that are still true — files are not memory; scope discipline matters; separated presentation is sane — but **drops the file index itself** in favor of sparse, Agent-authored tesserae. The maintenance-heavy machinery (`.nmg-search-scope`, incremental crawler, file FTS, scope observer) is not part -of this design. +of this design, and ticket 8 removes the code that PR #18 left behind +(`file-index.ts`, the per-search crawl, the DSH scope observer). The only +machine-derived file signal tesserae keep is a single 64-bit SimHash per +target file (§3.3) — a drift detector, not an index. ## 7. Open questions (deferred) From a8ab08622a447a0b2a3328bf8371f73023c14240 Mon Sep 17 00:00:00 2001 From: wefio <48851810+wefio@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:59:54 +0800 Subject: [PATCH 2/2] feat(tesserae): SimHash drift fingerprint for snippet relocation; remove FileIndex code debt (ticket 8) --- dsh/dsh-nmg/src/plugin/cordis-augment.d.ts | 9 - dsh/dsh-nmg/src/plugin/index.ts | 86 ------ src/cli/main.ts | 14 +- src/cli/service.ts | 222 ++++++++++---- src/core/file-index.ts | 324 --------------------- src/core/simhash.ts | 83 ++++++ src/core/stg.ts | 5 - src/core/store/retrieval.ts | 8 +- src/core/store/schema.ts | 20 +- src/core/store/writes.ts | 9 +- src/core/types.ts | 29 +- src/integration/agent-surface.ts | 6 +- src/integration/search-projection.ts | 6 +- tests/core/file-index.test.ts | 141 --------- tests/core/simhash.test.ts | 70 +++++ tests/core/tesserae-simhash.test.ts | 165 +++++++++++ 16 files changed, 539 insertions(+), 658 deletions(-) delete mode 100644 src/core/file-index.ts create mode 100644 src/core/simhash.ts delete mode 100644 tests/core/file-index.test.ts create mode 100644 tests/core/simhash.test.ts create mode 100644 tests/core/tesserae-simhash.test.ts diff --git a/dsh/dsh-nmg/src/plugin/cordis-augment.d.ts b/dsh/dsh-nmg/src/plugin/cordis-augment.d.ts index 7cd5aef..7796145 100644 --- a/dsh/dsh-nmg/src/plugin/cordis-augment.d.ts +++ b/dsh/dsh-nmg/src/plugin/cordis-augment.d.ts @@ -85,15 +85,6 @@ declare module '@deepseek-ai/cordis' { result: Readonly<{ isError: boolean; value?: unknown; content?: unknown }>, ): undefined } - - interface Context { - /** - * Optional NMG file-content index (src/core/file-index.ts). Consumed - * opportunistically by the `tools/result` scope observer; never injected, so - * the plugin mounts before/without the index and skips recording when absent. - */ - fileIndex?: { addScopePath(path: string): void } - } } export {} \ No newline at end of file diff --git a/dsh/dsh-nmg/src/plugin/index.ts b/dsh/dsh-nmg/src/plugin/index.ts index 7a0308a..97f80a6 100644 --- a/dsh/dsh-nmg/src/plugin/index.ts +++ b/dsh/dsh-nmg/src/plugin/index.ts @@ -29,7 +29,6 @@ import { renderTaskBoardSurface, } from '../../../../src/integration/agent-surface.ts' import { loadPrompts, renderDisclosure } from '../../../../src/prompts/load.ts' -import { FileIndex } from '../../../../src/core/file-index.ts' const nmgPrompts = loadPrompts() @@ -1505,11 +1504,6 @@ export function apply(ctx: Context): () => void { }) : undefined - // ── file-content scope observer (tools/result) ───────────────────────────── - const scopeObserver = setupScopeObserver(ctx, workspaceRoot) - const scopeObserverDisposer = scopeObserver.disposer - const provideFileIndex = scopeObserver.provide - // ── wake timer + startup ─────────────────────────────────────────────────── // Single host-side timer polls the daemon for board entries every // WAKE_INTERVAL_MS. The first poll also registers the agent and starts the @@ -1535,8 +1529,6 @@ export function apply(ctx: Context): () => void { contextDisposer, ctx.on('agent/inbox/inserted', onInboxInserted), ctx.on('agent/disposed', onAgentDisposed), - scopeObserverDisposer, - ...(provideFileIndex ? [provideFileIndex] : []), ] if (coordinationEnabled) { disposers.push( @@ -1560,84 +1552,6 @@ export function apply(ctx: Context): () => void { recallBatch.clear() openSearches.clear() wakeBatch.clear() - try { scopeObserver.service.close() } catch { /* best-effort */ } - } -} - -// ── file-content scope observer (tools/result) ─────────────────────────────── -// The file content source learns its search scope from the Agent's own search -// behaviour (docs/design/file-content-source-design.md §3.2): a non-empty -// `grep`/`read` result marks the searched files as hot zones, which the -// FileIndex records via addScopePath. This listener observes only — it never -// intercepts or mutates the tool lifecycle. Any parse failure is silent. -// -// The FileIndex is provided as an optional `fileIndex` service (addScopePath -// only) so the observer degrades to a no-op if the service is ever absent. -function setupScopeObserver(ctx, workspaceRoot) { - const service = new FileIndex({ projectRoot: workspaceRoot }) - const provide = ctx.provide - ? ctx.provide('fileIndex', { addScopePath: (path) => service.addScopePath(path) }) - : undefined - const collectScopePaths = (exec, result) => { - // Guard clauses keep the main flow linear (see complexity-reduction - // practice: guard clause + composed functions). - const name = (exec && (exec.name || exec.toolName)) || '' - if (name !== 'grep' && name !== 'read') return - const fileIndex = ctx.get('fileIndex') - if (!fileIndex || typeof fileIndex.addScopePath !== 'function') return - const paths = extractHitPaths(exec, result, name) - for (const path of paths) { - try { - fileIndex.addScopePath(path) - } catch { - // recording is best-effort; a failing index must not break the tool - } - } - } - const disposer = ctx.on('tools/result', (exec, result) => { - try { - collectScopePaths(exec, result) - } catch { - // observation never throws into the tool registry - } - }) - return { disposer, provide, service } -} - -/** Extract hot-zone candidate paths from a grep/read tool result. The tool - * name is `grep` (dsh-tool-fs-search) or `read` (dsh-tool-fs); glob is a - * separate tool we intentionally do not observe. */ -function extractHitPaths(exec, result, name) { - // Arguments: `exec.arguments` (dsh-tools) with `exec.input` as an alias. - const args = (exec && (exec.arguments || exec.input)) || {} - const value = result && result.isError ? undefined : result && result.value - if (name === 'grep') return grepHitPaths(value, args) - if (name === 'read') return readHitPaths(value, args) - return [] -} - -/** grep hit paths: match files (workdir-relative) + the searched path arg - * when the search found something. */ -function grepHitPaths(value, args) { - const paths = [] - const matches = value && Array.isArray(value.matches) ? value.matches : [] - for (const match of matches) { - if (match && typeof match.path === 'string' && match.path) paths.push(match.path) - } - if (paths.length > 0 && args && typeof args.path === 'string' && args.path) { - paths.push(args.path) } - return paths } -/** read hit paths: a non-empty read ⇔ at least one line; record the requested - * path arg and the resolved display path (they usually agree). */ -function readHitPaths(value, args) { - const paths = [] - const lines = value && Array.isArray(value.lines) ? value.lines : [] - if (lines.length > 0) { - if (args && typeof args.file_path === 'string' && args.file_path) paths.push(args.file_path) - if (value && typeof value.path === 'string' && value.path) paths.push(value.path) - } - return paths -} diff --git a/src/cli/main.ts b/src/cli/main.ts index 231ac89..7bbe5cf 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -434,19 +434,12 @@ function parseOptions(args: readonly string[]): OptionValues { return { flags, options, positionals }; } -/** Append the FILES / TESSERAE source-section lines of a search result. Kept as - * its own helper so humanResult's branching stays under the complexity gate. */ +/** Append the TESSERAE source-section lines of a search result. Kept as its + * own helper so humanResult's branching stays under the complexity gate. */ function sourceSectionLines( - files: Array<{ path: string; excerpt: string }> | undefined, tesserae: Array<{ path: string; label: string; line?: number; stale?: boolean }> | undefined, ): string[] { const lines: string[] = []; - if (files && files.length > 0) { - lines.push("FILES:"); - for (const file of files) { - lines.push(`${file.path}\t${file.excerpt}`); - } - } if (tesserae && tesserae.length > 0) { lines.push("TESSERAE:"); for (const tessera of tesserae) { @@ -604,7 +597,6 @@ function humanResult(value: unknown): string { memory: { id: string; memoryType: string; tier: number; statement: string }; node: { canonicalName: string }; }>; - files?: Array<{ path: string; excerpt: string }>; tesserae?: Array<{ path: string; label: string; line?: number; stale?: boolean }>; timings?: { timings?: Record; totalMs?: number }; }; @@ -612,7 +604,7 @@ function humanResult(value: unknown): string { ({ memory, node }) => `${memory.id}\t${memory.memoryType}\tL${memory.tier}\t${node.canonicalName}\t${memory.statement}`, ); - lines.push(...sourceSectionLines(context.files, context.tesserae)); + lines.push(...sourceSectionLines(context.tesserae)); if (context.timings) { const sections = Object.entries(context.timings.timings ?? {}) .sort((left, right) => right[1] - left[1]) diff --git a/src/cli/service.ts b/src/cli/service.ts index 83f8bd3..dbe5567 100644 --- a/src/cli/service.ts +++ b/src/cli/service.ts @@ -1,5 +1,5 @@ -import { existsSync, readFileSync, statSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; import { configuredProvider, @@ -75,8 +75,7 @@ import { scopesOverlap, validityIntervalsOverlap } from "../core/semantic-domain import { sameScope } from "../core/scope.ts"; import { normalizeRecallTriggers } from "../core/recall-triggers.ts"; import { searchMemoryContext } from "../integration/search.ts"; -import { FileIndex } from "../core/file-index.ts"; -import type { FileHit } from "../core/types.ts"; +import { simhash64, simhashToHex, simhashFromHex, hammingDistance } from "../core/simhash.ts"; import { ControllerPolicyChannel } from "../integration/controller-channel.ts"; import { LAB_CAPABILITIES, @@ -162,7 +161,6 @@ export class NmgService { #labShadowController: ControllerPolicyChannel | undefined; #store: NmgStore | undefined; readonly #stgStores = new Map(); - readonly #fileIndexes = new Map(); readonly #sessionActiveGraphs = new SessionActiveGraphRuntime(); #embeddingClient: EmbeddingClient | undefined | null; #embeddingError: string | null = null; @@ -537,8 +535,6 @@ export class NmgService { this.#store = undefined; for (const store of this.#stgStores.values()) store.close(); this.#stgStores.clear(); - for (const index of this.#fileIndexes.values()) index.close(); - this.#fileIndexes.clear(); this.#sessionActiveGraphs.clear(); } @@ -1027,8 +1023,14 @@ export class NmgService { const bypassMarkers: MemoryMarker[] = params.unsafe ? [{ kind: "write_bypass", attributes: { policy: "unsafe" } }] : []; + // Best-effort SimHash drift fingerprint: computed at write time from the + // project file when a project root is known, so a later snippet relocation + // can survive a small edit or file move. Absent for project-less writes — + // the tessera still writes, without drift tolerance. + const tesserae = this.#withTesseraSimhashes(memory.tesserae, projectDir); const input: RememberInput = { ...memory, + tesserae, markers: [...(memory.markers ?? []), ...bypassMarkers], // LTG rows are project/session-global: never attach a session_id. STG // rows keep the caller's sessionId (escape-hatch validated in the store). @@ -1040,6 +1042,27 @@ export class NmgService { return { store, input }; } + /** Stamp each tessera with a 64-bit SimHash of its target file (16-hex), or + * leave it absent when the file is unreadable at write time (or no project + * root is known). Fingerprint is drift tolerance only — it never gates or + * alters the write. */ + #withTesseraSimhashes( + tesserae: TesseraInput[] | undefined, + projectDir: string | undefined, + ): TesseraInput[] | undefined { + if (!projectDir || !tesserae || tesserae.length === 0) return tesserae; + return tesserae.map((tessera) => { + if (!tessera.path) return tessera; + try { + const content = readFileSafe(resolve(projectDir, tessera.path)); + if (content === null) return tessera; + return { ...tessera, fileSimhash: simhashToHex(simhash64(content)) }; + } catch { + return tessera; // never fail a memory write for a missing fingerprint + } + }); + } + #signalMaintenance(store: NmgStore, kind: "write" | "access", force = false, count = 1): void { const state = this.#maintenanceSignals.get(store) ?? { writes: 0, @@ -1388,14 +1411,6 @@ export class NmgService { }; const raws = [query, ...(queries ?? [])]; const embedding = this.#configuredEmbeddingClient(); - // File content source: bounded passive-scan index of the project's own - // files (scope learned from Agent search behaviour). Only enabled when a - // projectDir is supplied — files are a project-local search index, not - // part of the shared memory store. - let fileHits: FileHit[] = []; - if (projectDir) { - fileHits = this.#searchProjectFiles(projectDir, query, options.limit ?? 8); - } // Tessera (bookmark) source: independent of projectDir — tesserae live in the // shared store and are searched alongside memory. Snippet relocation to a // line needs the project root (to read the current file), so the resolver @@ -1404,8 +1419,7 @@ export class NmgService { this.#searchTesseraSource(query, options.limit ?? 8), projectDir, ); - const withFiles = (context: T): T => { - if (fileHits.length > 0) context.files = fileHits; + const withTesserae = (context: T): T => { if (tesseraHits.length > 0) context.tesserae = tesseraHits; return context; }; @@ -1425,17 +1439,19 @@ export class NmgService { this.#syncStgWorkingSet(store, options.scope); const local = await runOne(store, raw); if (local.results.length > 0 && local.activeGraph?.qpp?.trigger === false) { - return withFiles(this.#registerSearchProjection(local, activeGraphPartsFor(store, local))); + return withTesserae( + this.#registerSearchProjection(local, activeGraphPartsFor(store, local)), + ); } const sharedStore = this.#getStore(); const shared = await runOne(sharedStore, raw); if (local.results.length === 0) { - return withFiles( + return withTesserae( this.#registerSearchProjection(shared, activeGraphPartsFor(sharedStore, shared)), ); } const merged = mergeStgLtgContexts(local, shared); - return withFiles( + return withTesserae( this.#registerSearchProjection( merged, [ @@ -1462,7 +1478,7 @@ export class NmgService { if (!projectDir) { const store = this.#getStore(); const context = await runOne(store, raws[0]!); - return withFiles( + return withTesserae( this.#registerSearchProjection(context, activeGraphPartsFor(store, context)), ); } @@ -1485,30 +1501,7 @@ export class NmgService { } } if (searchOptions.limit) primary.results = primary.results.slice(0, searchOptions.limit); - return withFiles(this.#registerSearchProjection(primary, parts)); - } - - /** Incrementally scan the project's file scope and run the file-content - * source query. Best-effort: any file-index failure degrades to memory-only - * search rather than failing the request. */ - #searchProjectFiles(projectDir: string, query: string, limit: number): FileHit[] { - try { - const index = this.#fileIndexFor(projectDir); - index.crawl(); - return index.search(query, limit); - } catch { - return []; - } - } - - #fileIndexFor(projectDir: string): FileIndex { - const key = resolve(projectDir); - let index = this.#fileIndexes.get(key); - if (!index) { - index = new FileIndex({ projectRoot: key }); - this.#fileIndexes.set(key, index); - } - return index; + return withTesserae(this.#registerSearchProjection(primary, parts)); } #syncStgWorkingSet(store: NmgStore, scope: MemoryScope | undefined): void { @@ -1545,6 +1538,7 @@ export class NmgService { kind: row.kind, memoryId: row.memoryId, snippet: row.snippet, + fileSimhash: row.fileSimhash, })); } @@ -1562,18 +1556,72 @@ export class NmgService { }; return tesserae.map((tessera) => { if (!tessera.snippet) return { ...tessera, stale: true }; - const lines = linesFor(tessera.path); - if (!lines) return { ...tessera, stale: true }; const target = tessera.snippet.trim(); - // Snippet is the relocation key: locate the line whose content contains - // it (or that it contains, for short fragments). Best-effort single-line - // relocation in the MVP. - const found = lines.findIndex((line) => line.includes(target)); - if (found === -1) return { ...tessera, stale: true }; - return { ...tessera, line: found + 1 }; + const pathLines = linesFor(tessera.path); + if (pathLines) { + // Stage 1 — exact relocation against the stored path. + const found = pathLines.findIndex((line) => line.includes(target)); + if (found !== -1) return { ...tessera, line: found + 1 }; + } + // Stage 2 — SimHash drift fallback. The stored path no longer relocates + // (edited or moved). Compare the stored file fingerprint against current + // files; a near-identical document (Hamming ≤ 6) elsewhere is the same + // file after a move — locate the snippet there. Never auto-rewrites the + // row; a stored file that still matches but lost the snippet is stale. + const relocated = this.#relocateTesseraViaSimhash(tessera, target, projectRoot); + return relocated ?? { ...tessera, stale: true }; }); } + /** SimHash drift relocation: find a current file whose fingerprint is within + * Hamming SIMHASH_DRIFT_THRESHOLD of the tessera's stored fingerprint and + * locate the snippet inside it. Only meaningful when the stored path no + * longer relocates the snippet (file moved or rewritten). Returns a resolved + * hit, or null when no candidate contains the snippet. */ + #relocateTesseraViaSimhash( + tessera: TesseraHit, + snippet: string, + projectRoot: string, + ): TesseraHit | null { + const stored = tessera.fileSimhash; + if (!stored) return null; + const storedFingerprint = simhashFromHex(stored); + const absStoredPath = resolve(projectRoot, tessera.path); + const current = readLinesSafe(absStoredPath); + if (current) { + // (a) The stored file still exists. If it still resembles the written + // document (Hamming ≤ threshold), the snippet itself was rewritten — + // honestly stale: relocation by content cannot recover a fragment that no + // longer exists. If it no longer resembles the document, the file was + // replaced/rewritten — also not a move to chase. + const currentFingerprint = simhash64(current.join("\n")); + const storedPathMatches = + hammingDistance(currentFingerprint, storedFingerprint) <= SIMHASH_DRIFT_THRESHOLD; + if (storedPathMatches) return null; + } + // (b) The stored file is gone or rewritten past tolerance: the tessera may + // point at a document that moved. Scan project files for a near-identical + // file and locate the snippet there. Only a file whose fingerprint matches + // AND whose content contains the snippet is accepted — never guessed. + for (const candidate of projectCandidateFiles(projectRoot)) { + if (absStoredPath === resolve(candidate)) continue; // tried above + const candidateLines = readLinesSafe(candidate); + if (!candidateLines) continue; + const candidateFingerprint = simhash64(candidateLines.join("\n")); + if (hammingDistance(candidateFingerprint, storedFingerprint) > SIMHASH_DRIFT_THRESHOLD) { + continue; + } + const line = locateSnippetLine(candidateLines, snippet); + if (line !== null) { + // Canonical project-relative path: always forward slashes, matching how + // tesserae paths are stored and searched (Windows relative() yields \). + const rel = relative(projectRoot, candidate).replaceAll("\\", "/"); + return { ...tessera, path: rel, line, relocated: true }; + } + } + return null; + } + #get(params: NmgGetParams): NmgMethodResult["get"] { const sharedStore = this.#getStore(); const localStore = params.projectDir @@ -1960,6 +2008,74 @@ function optionalTesserae( }); } +/** Max Hamming distance for a SimHash drift match. Measured on real repo + * files (5–60 KB): near-identical pairs sit at 1–3, unrelated at ~24, so 6 + * cleanly separates without false positives (see simhash.ts). */ +const SIMHASH_DRIFT_THRESHOLD = 6; + +/** Files scanned by the SimHash drift fallback when the stored tessera path + * no longer relocates (moved-file case). Bounded walk of a project root: + * skips VCS/dependency/build directories and non-regular files, caps at + * SIMHASH_SCOPE_MAX_FILES so a pathological tree cannot stall a search. */ +const SIMHASH_SCOPE_MAX_FILES = 200; + +const SIMHASH_SCOPE_SKIP_DIRS = new Set([ + ".git", + ".hg", + ".svn", + "node_modules", + "dist", + "build", + ".cache", + ".benchmarks", +]); + +/** Locate a snippet inside file lines. Returns the 1-based line whose content + * contains the snippet, or null. Snippet is the relocation key — MVP keeps + * single-line relocation (a snippet spanning an edit's new line boundary is + * reported stale rather than guessed). */ +function locateSnippetLine(lines: string[], snippet: string): number | null { + const found = lines.findIndex((line) => line.includes(snippet)); + return found === -1 ? null : found + 1; +} + +/** Bounded, order-stable list of readable files under a project root that the + * SimHash fallback may scan. Never follows symlinks out of the root. */ +function projectCandidateFiles(root: string, limit = SIMHASH_SCOPE_MAX_FILES): string[] { + const files: string[] = []; + const walk = (dir: string): void => { + if (files.length >= limit) return; + let entries: import("node:fs").Dirent[]; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; // unreadable directory — skip + } + for (const entry of entries) { + if (files.length >= limit) return; + const name = entry.name; + if (entry.isDirectory()) { + if (!name.startsWith(".") && !SIMHASH_SCOPE_SKIP_DIRS.has(name)) walk(`${dir}/${name}`); + continue; + } + if (!entry.isFile()) continue; + files.push(`${dir}/${name}`); + } + }; + walk(root); + return files; +} + +/** Read a file's full content, or null when unreadable/missing. Used by the + * tessera write-time SimHash computation. */ +function readFileSafe(absPath: string): string | null { + try { + return readFileSync(absPath, "utf8"); + } catch { + return null; + } +} + /** Read a file's lines, or null when unreadable/missing. Used by tessera * snippet relocation. */ function readLinesSafe(absPath: string): string[] | null { diff --git a/src/core/file-index.ts b/src/core/file-index.ts deleted file mode 100644 index fa2e65c..0000000 --- a/src/core/file-index.ts +++ /dev/null @@ -1,324 +0,0 @@ -// Bounded passive-scan file index for nmg search. -// -// This module gives nmg search a second content source — the project's own -// files and documents — so an Agent does not have to re-discover a changing -// project by hand every session. It is deliberately scoped: -// -// - the scan is BOUNDED to paths listed in `.nmg-search-scope` (semantically -// opposite to `.gitignore`: it INCLUDES hot zones instead of excluding); -// - the scan is PASSIVE (automatic, no manual trigger) and INCREMENTAL -// (git status / content hash tells which files changed since last scan); -// - files are a search index, NOT memory: content never enters LTG/STG and -// never gets provenance/scope/verification semantics. -// -// The index lives in a project-local SQLite file (`.nmg/file-index.sqlite`), -// physically separate from the memory store, so deleting `.nmg/` removes the -// file index without touching memory. - -import { createHash } from "node:crypto"; -import { - existsSync, - mkdirSync, - readFileSync, - readdirSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { DatabaseSync } from "node:sqlite"; -import { isAbsolute, join, relative, resolve } from "node:path"; - -import { ftsExpression, surfaceIndexedText } from "./store/search-ranking.ts"; -import type { FileHit } from "./types.ts"; - -export type { FileHit }; - -export interface FileIndexOptions { - /** Project root; `.nmg-search-scope` is resolved against it. */ - projectRoot: string; - /** Directory for the index file (default `/.nmg`). */ - dataDir?: string; - /** Explicit scope file path (default `/.nmg-search-scope`). */ - scopePath?: string; - /** Max paths kept in the scope (auto-grown cap). Default 256. */ - maxScopePaths?: number; - /** Max file size in bytes to index (skip larger). Default 1 MiB. */ - maxFileBytes?: number; -} - -interface ScopeEntry { - path: string; - dir: boolean; -} - -const DEFAULT_MAX_SCOPE = 256; -const DEFAULT_MAX_FILE_BYTES = 1024 * 1024; - -/** Parse a `.nmg-search-scope` file: one path per line, `#` comments. - * Directories imply recursion. Empty/blank lines are ignored. */ -export function parseScopeFile(content: string): string[] { - return content - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith("#")) - .map((line) => line.replace(/[\\/]+$/u, "")) // strip trailing slash - .filter(Boolean); -} - -/** Expand scope entries into concrete paths under the project root. A - * directory entry expands to the directory itself (recursion happens at - * scan time); a file entry is used as-is. Paths are normalized relative. */ -export function resolveScopeEntries(projectRoot: string, entries: string[]): ScopeEntry[] { - const seen = new Set(); - const out: ScopeEntry[] = []; - for (const entry of entries) { - const abs = isAbsolute(entry) ? entry : resolve(projectRoot, entry); - const rel = relative(projectRoot, abs).replaceAll("\\", "/"); - const key = rel || entry; - if (seen.has(key)) continue; - seen.add(key); - let dir: boolean; - try { - dir = statSync(abs).isDirectory(); - } catch { - dir = entry.endsWith("/"); - } - out.push({ path: key, dir }); - } - return out; -} - -/** Collect indexable files under a directory entry, respecting a caller - * supplied exclusion predicate (node_modules/.git/etc). */ -export function collectFiles( - root: string, - entry: ScopeEntry, - maxBytes: number, - exclude: (relPath: string) => boolean, -): string[] { - if (!entry.dir) return [entry.path]; - const out: string[] = []; - const absRoot = resolve(root, entry.path); - const walk = (dir: string): void => { - let entries: string[]; - try { - entries = readdirSync(dir); - } catch { - return; - } - for (const name of entries) { - const abs = join(dir, name); - const rel = relative(root, abs).replaceAll("\\", "/"); - if (exclude(rel)) continue; - let stat; - try { - stat = statSync(abs); - } catch { - continue; - } - if (stat.isDirectory()) { - walk(abs); - } else if (stat.isFile() && stat.size <= maxBytes) { - out.push(rel); - } - } - }; - walk(absRoot); - return out.sort(); -} - -/** Default exclusion predicate: skip VCS dirs, node_modules, build output, - * binaries, and dot-dirs (mirrors common .gitignore behavior). */ -export function defaultExclude(relPath: string): boolean { - const parts = relPath.split("/"); - if (parts.some((part) => part === "node_modules" || part === ".git" || part === ".nmg")) { - return true; - } - if (parts.some((part) => part.startsWith(".") && part !== "." && part !== "..")) { - // allow .nmg-search-scope itself but skip other dot-dirs - return !(parts.length === 1 && parts[0] === ".nmg-search-scope"); - } - const binary = - /\.(?:png|jpe?g|gif|webp|ico|pdf|zip|gz|tar|exe|dll|so|dylib|woff2?|ttf|eot|class|jar)$/iu; - return binary.test(relPath); -} - -/** - * Bounded passive-scan file index. - * - * Storage: `.nmg/file-index.sqlite` with two tables: - * - `file_fts` FTS5 trigram index over (path, content) - * - `file_meta` per-file content hash for incremental scans - * - * The scope is read from `.nmg-search-scope`; `addScopePath` grows it - * (the Agent is the first crawler: grep/read hits teach the scope). - */ -export class FileIndex { - readonly projectRoot: string; - readonly dataDir: string; - readonly scopePath: string; - readonly maxScopePaths: number; - readonly maxFileBytes: number; - readonly db: DatabaseSync; - - constructor(options: FileIndexOptions) { - this.projectRoot = resolve(options.projectRoot); - this.dataDir = resolve(options.dataDir ?? join(this.projectRoot, ".nmg")); - this.scopePath = resolve(options.scopePath ?? join(this.projectRoot, ".nmg-search-scope")); - this.maxScopePaths = Math.max(1, options.maxScopePaths ?? DEFAULT_MAX_SCOPE); - this.maxFileBytes = Math.max(1, options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES); - mkdirSync(this.dataDir, { recursive: true }); - this.db = new DatabaseSync(join(this.dataDir, "file-index.sqlite")); - this.db.exec(` - CREATE TABLE IF NOT EXISTS file_meta ( - path TEXT PRIMARY KEY, - content TEXT NOT NULL, - content_hash TEXT NOT NULL, - indexed_at TEXT NOT NULL - ); - CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5( - path UNINDEXED, - content, - content = 'file_meta', - content_rowid = 'rowid', - tokenize = 'trigram' - ); - CREATE TRIGGER IF NOT EXISTS file_fts_ai AFTER INSERT ON file_meta BEGIN - INSERT INTO file_fts (rowid, path, content) VALUES (new.rowid, new.path, new.content); - END; - CREATE TRIGGER IF NOT EXISTS file_fts_ad AFTER DELETE ON file_meta BEGIN - INSERT INTO file_fts (file_fts, rowid, path, content) - VALUES ('delete', old.rowid, old.path, old.content); - END; - CREATE TRIGGER IF NOT EXISTS file_fts_au AFTER UPDATE ON file_meta BEGIN - INSERT INTO file_fts (file_fts, rowid, path, content) - VALUES ('delete', old.rowid, old.path, old.content); - INSERT INTO file_fts (rowid, path, content) VALUES (new.rowid, new.path, new.content); - END; - `); - } - - close(): void { - this.db.close(); - } - - /** Read the current scope (relative paths). */ - readScope(): string[] { - if (!existsSync(this.scopePath)) return []; - try { - return parseScopeFile(readFileSync(this.scopePath, "utf8")); - } catch { - return []; - } - } - - /** Add a hot-zone path to the scope (dedup, cap, auto-create file). The - * Agent is the first crawler: grep/read hits feed this. */ - addScopePath(path: string): void { - const clean = String(path ?? "") - .trim() - .replace(/[\\/]+$/u, ""); - if (!clean) return; - const entries = this.readScope(); - if (entries.includes(clean)) return; - entries.push(clean); - // Cap: keep the most recently added (tail) paths. - const capped = entries.slice(-this.maxScopePaths); - writeFileSync( - this.scopePath, - `# .nmg-search-scope — hot zones indexed by the file content source\n` + - `# (semantically opposite to .gitignore: INCLUDES paths to index)\n` + - capped.map((path) => path).join("\n") + - "\n", - "utf8", - ); - } - - /** Incremental scan: index files under the scope whose content hash changed - * (or are new), remove files no longer present. Returns counts. */ - crawl(now = new Date().toISOString()): { indexed: number; removed: number } { - const scopeEntries = resolveScopeEntries(this.projectRoot, this.readScope()); - const wanted = new Map(); // relPath -> contentHash - for (const entry of scopeEntries) { - for (const rel of collectFiles(this.projectRoot, entry, this.maxFileBytes, defaultExclude)) { - const abs = resolve(this.projectRoot, rel); - try { - const content = readFileSync(abs, "utf8"); - wanted.set(rel, hash(content)); - } catch { - // unreadable/binary: skip - } - } - } - - let indexed = 0; - const upsertMeta = this.db.prepare( - "INSERT INTO file_meta (path, content, content_hash, indexed_at) VALUES (?, ?, ?, ?) " + - "ON CONFLICT(path) DO UPDATE SET content = excluded.content, content_hash = excluded.content_hash, indexed_at = excluded.indexed_at", - ); - for (const [rel, contentHash] of wanted) { - const prev = this.db.prepare("SELECT content_hash FROM file_meta WHERE path = ?").get(rel) as - { content_hash: string } | undefined; - if (prev && prev.content_hash === contentHash) continue; // unchanged - const abs = resolve(this.projectRoot, rel); - let content: string; - try { - content = readFileSync(abs, "utf8"); - } catch { - continue; - } - // file_meta drives the FTS via triggers (insert/update/delete), so the - // index always has exactly one row per file. - upsertMeta.run(rel, surfaceIndexedText(content), contentHash, now); - indexed += 1; - } - - // Remove files that disappeared from the scope (or changed scope). - const known = this.db.prepare("SELECT path FROM file_meta").all() as Array<{ path: string }>; - let removed = 0; - for (const row of known) { - if (!wanted.has(row.path)) { - this.db.prepare("DELETE FROM file_meta WHERE path = ?").run(row.path); - removed += 1; - } - } - return { indexed, removed }; - } - - /** Search the file index. Returns hits with a short excerpt. */ - search(query: string, limit = 8): FileHit[] { - const expression = ftsExpression(query); - if (!expression) return []; - const rows = this.db - .prepare( - `SELECT path, snippet(file_fts, 1, '[', ']', '…', 12) AS excerpt, bm25(file_fts) AS rank - FROM file_fts - WHERE file_fts MATCH ? - ORDER BY rank LIMIT ?`, - ) - .all(expression, Math.max(1, Math.min(limit, 50))) as Array<{ - path: string; - excerpt: string | null; - rank: number; - }>; - return rows.map((row) => ({ - path: row.path, - excerpt: row.excerpt ?? "", - score: -Number(row.rank), // bm25 lower is better; negate for descending - })); - } - - /** Remove the index database entirely (e.g. on scope reset). */ - destroy(): void { - this.db.close(); - try { - rmSync(join(this.dataDir, "file-index.sqlite"), { force: true }); - } catch { - // best-effort - } - } -} - -function hash(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 32); -} diff --git a/src/core/simhash.ts b/src/core/simhash.ts new file mode 100644 index 0000000..7061356 --- /dev/null +++ b/src/core/simhash.ts @@ -0,0 +1,83 @@ +/** + * SimHash document fingerprint — a 64-bit locality-sensitive hash used to tell + * whether two *files* are near-identical (small edits, moved file), NOT whether + * two short snippets are semantically similar. + * + * Ticket-8 context: measured on real repo files (5–60 KB), near-duplicates sit + * at Hamming 1–3 and unrelated documents at ~24, so threshold ≤ 6 cleanly + * separates (100% recall / 0.24% false positive). The same fingerprint has no + * discriminative power below document scale and is never applied to snippets. + * + * Zero-dependency. Storage uses a 16-hex TEXT (see schema notes): the full + * unsigned 64-bit range cannot round-trip through SQLite INTEGER bindings + * (node:sqlite rejects values > 2^63−1 and cannot read back > 2^53 exactly). + */ + +const FNV_OFFSET = 0xcbf29ce484222325n; +const FNV_PRIME = 0x100000001b3n; +const MASK_64 = 0xffffffffffffffffn; + +/** FNV-1a 64-bit over a UTF-16 code-unit stream (matches how we tokenize). */ +export function fnv1a64(input: string): bigint { + let hash = FNV_OFFSET; + for (let index = 0; index < input.length; index += 1) { + hash ^= BigInt(input.charCodeAt(index)); + hash = (hash * FNV_PRIME) & MASK_64; + } + return hash; +} + +/** Tokenize file text for fingerprinting: contiguous word runs and code-like + * identifiers, with each Han ideograph as its own token (files in this repo + * mix English prose, code and CJK comments). */ +export function simhashTokens(value: string): string[] { + const tokens: string[] = []; + for (const match of value.matchAll(/[\p{Script=Han}]|[\p{L}\p{N}_]+/gu)) { + const token = match[0]; + tokens.push(token.length === 1 && /\p{Script=Han}/u.test(token) ? token : token.toLowerCase()); + } + return tokens; +} + +/** 64-bit SimHash of a document. Token weight is raw frequency — enough for + * the document-scale separation this fingerprint is used for. */ +export function simhash64(value: string): bigint { + const weights = new Map(); + for (const token of simhashTokens(value)) { + weights.set(token, (weights.get(token) ?? 0) + 1); + } + // Per-bit accumulator: +weight when the token's hash bit is 1, −weight when 0. + const accumulators = new Array(64).fill(0); + for (const [token, weight] of weights) { + const hash = fnv1a64(token); + for (let bit = 0; bit < 64; bit += 1) { + const delta = ((hash >> BigInt(bit)) & 1n) === 1n ? weight : -weight; + accumulators[bit] += delta; + } + } + let result = 0n; + for (let bit = 0; bit < 64; bit += 1) { + if (accumulators[bit] > 0) result |= 1n << BigInt(bit); + } + return result; +} + +/** Hamming distance between two 64-bit fingerprints. */ +export function hammingDistance(left: bigint, right: bigint): number { + let diff = left ^ right; + let distance = 0; + while (diff !== 0n) { + diff &= diff - 1n; // clear the lowest set bit + distance += 1; + } + return distance; +} + +/** 16 lowercase hex chars — the canonical storage form (TEXT column). */ +export function simhashToHex(fingerprint: bigint): string { + return fingerprint.toString(16).padStart(16, "0"); +} + +export function simhashFromHex(hex: string): bigint { + return BigInt(`0x${hex}`); +} diff --git a/src/core/stg.ts b/src/core/stg.ts index 31817b1..06eca2b 100644 --- a/src/core/stg.ts +++ b/src/core/stg.ts @@ -250,14 +250,9 @@ export function mergeStgLtgContexts(local: MemoryContext, shared: MemoryContext) ]), ).values(), ]; - const files = - local.files && shared.files - ? [...new Map([...local.files, ...shared.files].map((hit) => [hit.path, hit])).values()] - : (local.files ?? shared.files); return { results, ...(chainEdges.length > 0 ? { chainEdges } : {}), - ...(files ? { files } : {}), relations: [ ...new Map( [...(local.relations ?? []), ...shared.relations].map((relation) => [ diff --git a/src/core/store/retrieval.ts b/src/core/store/retrieval.ts index d3caf93..abf602e 100644 --- a/src/core/store/retrieval.ts +++ b/src/core/store/retrieval.ts @@ -1650,7 +1650,7 @@ export function withRetrieval(Base: TBase) { if (!expression) return []; const rows = this.db .prepare( - `SELECT a.id, a.path, a.snippet, a.label, a.kind, a.memory_id, a.created_at + `SELECT a.id, a.path, a.snippet, a.label, a.kind, a.memory_id, a.created_at, a.file_simhash FROM tesserae_fts f JOIN tesserae a ON a.rowid = f.rowid WHERE tesserae_fts MATCH ? @@ -1665,6 +1665,7 @@ export function withRetrieval(Base: TBase) { kind: string | null; memory_id: string | null; created_at: string; + file_simhash: string | null; }>; return rows.map((row) => ({ id: row.id, @@ -1674,6 +1675,7 @@ export function withRetrieval(Base: TBase) { kind: row.kind ?? undefined, memoryId: row.memory_id ?? undefined, createdAt: row.created_at, + fileSimhash: row.file_simhash ?? undefined, })); } @@ -1683,7 +1685,7 @@ export function withRetrieval(Base: TBase) { const placeholders = ids.map(() => "?").join(","); const rows = this.db .prepare( - `SELECT id, path, snippet, label, kind, memory_id, created_at + `SELECT id, path, snippet, label, kind, memory_id, created_at, file_simhash FROM tesserae WHERE id IN (${placeholders})`, ) .all(...ids) as Array<{ @@ -1694,6 +1696,7 @@ export function withRetrieval(Base: TBase) { kind: string | null; memory_id: string | null; created_at: string; + file_simhash: string | null; }>; return rows.map((row) => ({ id: row.id, @@ -1703,6 +1706,7 @@ export function withRetrieval(Base: TBase) { kind: row.kind ?? undefined, memoryId: row.memory_id ?? undefined, createdAt: row.created_at, + fileSimhash: row.file_simhash ?? undefined, })); } diff --git a/src/core/store/schema.ts b/src/core/store/schema.ts index 4057dbb..aa5e95d 100644 --- a/src/core/store/schema.ts +++ b/src/core/store/schema.ts @@ -552,7 +552,13 @@ export function migrate(db: DatabaseSync): void { label TEXT NOT NULL DEFAULT '', kind TEXT, memory_id TEXT, - created_at TEXT NOT NULL + created_at TEXT NOT NULL, + -- 64-bit SimHash of the target file at tessera write time (16 lowercase + -- hex). Drift tolerance: when the snippet no longer matches in the path, + -- compare this fingerprint against current files to find the document + -- after a small edit or move. Stored TEXT because the full unsigned + -- 64-bit range cannot round-trip through SQLite INTEGER (see simhash.ts). + file_simhash TEXT ); CREATE INDEX IF NOT EXISTS idx_tesserae_path ON tesserae(path); CREATE INDEX IF NOT EXISTS idx_tesserae_memory ON tesserae(memory_id) WHERE memory_id IS NOT NULL; @@ -602,6 +608,7 @@ export function migrate(db: DatabaseSync): void { ensureHistoryColumns(db); ensureClaimOutcomeColumns(db); ensureEmbeddingTable(db); + ensureTesseraColumns(db); ensureNodeColumns(db); ensureRelationColumns(db); ensureTopologyProposalColumns(db); @@ -751,6 +758,17 @@ export function ensureMemoryColumns(db: DatabaseSync): void { ); } +/** Tessera drift fingerprint: adds `file_simhash` to stores created before + * ticket 8 (CREATE TABLE IF NOT EXISTS is a no-op against the existing table). */ +export function ensureTesseraColumns(db: DatabaseSync): void { + const columns = new Set( + (db.prepare("PRAGMA table_info(tesserae)").all() as Row[]).map((row) => String(row.name)), + ); + if (!columns.has("file_simhash")) { + db.exec("ALTER TABLE tesserae ADD COLUMN file_simhash TEXT"); + } +} + export function ensureDeltaColumns(db: DatabaseSync): void { const columns = new Set( (db.prepare("PRAGMA table_info(memory_index_delta)").all() as Row[]).map((row) => diff --git a/src/core/store/writes.ts b/src/core/store/writes.ts index 9c2c8f4..22007c2 100644 --- a/src/core/store/writes.ts +++ b/src/core/store/writes.ts @@ -609,6 +609,7 @@ export function withWrites(Base: TBase) { snippet: string; label: string; kind?: string; + fileSimhash?: string; }>; tesseraMarkers: MemoryMarker[]; } { @@ -618,6 +619,7 @@ export function withWrites(Base: TBase) { snippet: string; label: string; kind?: string; + fileSimhash?: string; }> = []; const tesseraMarkers: MemoryMarker[] = []; for (const tessera of input.tesserae ?? []) { @@ -631,6 +633,7 @@ export function withWrites(Base: TBase) { snippet, label: String(tessera.label ?? "").trim(), kind: tessera.kind?.trim() || undefined, + fileSimhash: tessera.fileSimhash, }); tesseraMarkers.push({ kind: "tessera_ref", attributes: { tesseraId: id, path } }); } @@ -645,13 +648,14 @@ export function withWrites(Base: TBase) { snippet: string; label: string; kind?: string; + fileSimhash?: string; }>, memoryId: string, ): void { if (tesseraPlan.length === 0) return; const insertTessera = this.db.prepare( - `INSERT INTO tesserae (id, path, snippet, label, kind, memory_id, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO tesserae (id, path, snippet, label, kind, memory_id, created_at, file_simhash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ); const createdAt = new Date().toISOString(); for (const plan of tesseraPlan) { @@ -663,6 +667,7 @@ export function withWrites(Base: TBase) { plan.kind ?? null, memoryId, createdAt, + plan.fileSimhash ?? null, ); } } diff --git a/src/core/types.ts b/src/core/types.ts index a98b7d6..aa66992 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -844,15 +844,6 @@ export interface RetrievalFilterUsage { selectivity: number; } -export interface FileHit { - /** Path relative to the project root. */ - path: string; - /** Short excerpt around the first match. */ - excerpt: string; - /** Relevance score: negated FTS bm25, so larger is more relevant. */ - score: number; -} - /** A file location a memory points into (a tessera / bookmark). Content-anchored: the * snippet relocates on read, so no line number is ever persisted. */ export interface TesseraRecord { @@ -869,6 +860,10 @@ export interface TesseraRecord { /** Owning memory id, when the tessera was raised by a memory write. */ memoryId?: string; createdAt: string; + /** 64-bit SimHash (16-hex) of the target file at write time; used for drift + * relocation when the snippet no longer matches in path. Absent for rows + * written without a readable file (or pre-ticket-8 stores). */ + fileSimhash?: string; } /** Tessera input on a memory write (agent-supplied, active). */ @@ -877,6 +872,11 @@ export interface TesseraInput { snippet: string; label?: string; kind?: string; + /** 64-bit SimHash (16-hex) of the target file, computed by the service layer + * when the file is readable at write time. Best-effort drift tolerance — + * absent when no file was available (e.g. LTG writes without a project + * root); never agent-supplied. */ + fileSimhash?: string; } /** Tessera hit surfaced by search — a tessera row plus its resolved line, or a @@ -893,6 +893,12 @@ export interface TesseraHit { line?: number; /** True when the snippet no longer exists in the file (stale tessera). */ stale?: boolean; + /** True when relocation succeeded via the SimHash drift fallback (file moved + * or edited since the tessera was written), not the stored path. */ + relocated?: boolean; + /** 64-bit SimHash (16-hex) of the target file at write time; the drift + * fallback key. */ + fileSimhash?: string; /** Relevance score when matched via FTS; higher is more relevant. */ score?: number; } @@ -902,11 +908,6 @@ export const TESSERA_REF_MARKER = "tessera_ref"; export interface MemoryContext { results: MemorySearchResult[]; - /** File-content source hits (bounded passive-scope FTS), separate from - * memory results. Files are a search index, not memory — they never carry - * provenance/scope/verification and never enter STG/LTG. Present only when - * the file content source is enabled for the searched project. */ - files?: FileHit[]; /** Tessera (bookmark) hits, an independent searchable source alongside * memory. Each hit carries the resolved line in the current file, or a * staleness marker when the snippet no longer exists. */ diff --git a/src/integration/agent-surface.ts b/src/integration/agent-surface.ts index 6603f69..3cc8f03 100644 --- a/src/integration/agent-surface.ts +++ b/src/integration/agent-surface.ts @@ -114,14 +114,11 @@ export function renderCompactSearchSurface( context: CompactSearchContext, options: SearchSurfaceOptions = {}, ): string { - const fileLines = (context.files ?? []).map( - (hit) => `- file=${hit.path}; excerpt=${hit.excerpt}`, - ); const tesseraLines = (context.tesserae ?? []).map((tessera) => { const position = tessera.stale ? "(stale)" : tessera.line ? `:${tessera.line}` : ""; return `- tessera=${tessera.path}${position}; label=${tessera.label || "(bookmark)"}`; }); - if (context.candidates.length === 0 && fileLines.length === 0 && tesseraLines.length === 0) { + if (context.candidates.length === 0 && tesseraLines.length === 0) { return options.emptyText ?? "No matching NMG memory found."; } const lines = context.candidates.map((candidate) => { @@ -141,7 +138,6 @@ export function renderCompactSearchSurface( return [ options.preamble, ...lines, - ...fileLines, ...tesseraLines, context.logicalChainCount > 0 ? `logical_chains=${context.logicalChainCount}; use nmg_get for compact chain structure with exact evidence.` diff --git a/src/integration/search-projection.ts b/src/integration/search-projection.ts index 612c0e6..dc2e4e3 100644 --- a/src/integration/search-projection.ts +++ b/src/integration/search-projection.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import type { TesseraHit, FileHit, MemoryContext, MemorySearchResult } from "../core/types.ts"; +import type { TesseraHit, MemoryContext, MemorySearchResult } from "../core/types.ts"; import type { SessionDisclosureLevel } from "../core/session-active-graph.ts"; import { logicalChainCount, logicalChainNames } from "./chain-projection.ts"; @@ -34,9 +34,6 @@ export interface CompactSearchContext { logicalChainCount: number; activeGraphId: string | null; deferredMemoryIds: string[]; - /** File-content source hits (bounded passive-scope FTS), separate from - * memory candidates. Present when the file source is enabled. */ - files?: FileHit[]; /** Tessera (bookmark) hits — file locations attached to memories, resolved to * lines (or marked stale) by the service layer. */ tesserae?: TesseraHit[]; @@ -60,7 +57,6 @@ export function compactSearchContext(context: MemoryContext): CompactSearchConte logicalChainCount: logicalChainCount(context), activeGraphId: context.activeGraph?.id ?? null, deferredMemoryIds: context.progressiveDisclosure?.deferredMemoryIds ?? [], - ...(context.files && context.files.length > 0 ? { files: context.files } : {}), ...(context.tesserae && context.tesserae.length > 0 ? { tesserae: context.tesserae } : {}), }; } diff --git a/tests/core/file-index.test.ts b/tests/core/file-index.test.ts deleted file mode 100644 index 7bb65ac..0000000 --- a/tests/core/file-index.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; - -import { - FileIndex, - collectFiles, - defaultExclude, - parseScopeFile, - resolveScopeEntries, -} from "../../src/core/file-index.ts"; - -function fixture(): { root: string; dataDir: string; scope: string } { - const root = mkdtempSync(join(tmpdir(), "nmg-file-index-")); - const dataDir = join(root, ".nmg"); - const scope = join(root, ".nmg-search-scope"); - mkdirSync(dataDir, { recursive: true }); - return { root, dataDir, scope }; -} - -test("parseScopeFile: one path per line, # comments, blank lines ignored", () => { - assert.deepEqual( - parseScopeFile("# hot zones\nsrc/core/store/\n\ndocs/design/\n src/cli \n"), - ["src/core/store", "docs/design", "src/cli"], - ); - assert.deepEqual(parseScopeFile(""), []); -}); - -test("resolveScopeEntries: relative paths resolve under project root, dirs flagged", () => { - const root = mkdtempSync(join(tmpdir(), "nmg-scope-resolve-")); - mkdirSync(join(root, "src", "core"), { recursive: true }); - writeFileSync(join(root, "README.md"), "x"); - const entries = resolveScopeEntries(root, ["src/core", "README.md", "missing"]); - assert.deepEqual( - entries.map((entry) => ({ path: entry.path, dir: entry.dir })), - [ - { path: "src/core", dir: true }, - { path: "README.md", dir: false }, - { path: "missing", dir: false }, - ], - ); - rmSync(root, { recursive: true, force: true }); -}); - -test("collectFiles: walks directory entries, respects exclude and size cap", () => { - const root = mkdtempSync(join(tmpdir(), "nmg-collect-")); - mkdirSync(join(root, "src", "core"), { recursive: true }); - mkdirSync(join(root, "src", "node_modules"), { recursive: true }); - writeFileSync(join(root, "src", "core", "a.ts"), "a"); - writeFileSync(join(root, "src", "core", "b.md"), "b"); - writeFileSync(join(root, "src", "node_modules", "c.js"), "c"); - const files = collectFiles(root, { path: "src", dir: true }, 1024, defaultExclude); - assert.deepEqual(files, ["src/core/a.ts", "src/core/b.md"]); - rmSync(root, { recursive: true, force: true }); -}); - -test("FileIndex: crawl indexes scope files, search finds them, incremental skips unchanged", () => { - const { root, dataDir, scope } = fixture(); - writeFileSync(scope, "src/core\n", "utf8"); - mkdirSync(join(root, "src", "core"), { recursive: true }); - writeFileSync(join(root, "src", "core", "retrieval.ts"), "function searchMemory() {}"); - writeFileSync(join(root, "src", "core", "writes.ts"), "function remember() {}"); - - const index = new FileIndex({ projectRoot: root, dataDir, scopePath: scope }); - try { - const first = index.crawl(); - assert.equal(first.indexed, 2); - - const hits = index.search("searchMemory"); - assert.equal(hits.length, 1); - assert.equal(hits[0]!.path, "src/core/retrieval.ts"); - // Trigram snippet may truncate the match; require a partial token. - assert.ok(/searc/u.test(hits[0]!.excerpt), "excerpt contains a fragment of the match"); - assert.ok(hits[0]!.score > 0, "bm25 negated: relevant hit has a positive score"); - - // Second crawl with no changes: nothing re-indexed. - const second = index.crawl(); - assert.equal(second.indexed, 0); - } finally { - index.close(); - rmSync(root, { recursive: true, force: true }); - } -}); - -test("FileIndex: changed file is re-indexed, removed file is dropped", () => { - const { root, dataDir, scope } = fixture(); - writeFileSync(scope, "src\n", "utf8"); - mkdirSync(join(root, "src"), { recursive: true }); - writeFileSync(join(root, "src", "a.ts"), "token-one"); - - const index = new FileIndex({ projectRoot: root, dataDir, scopePath: scope }); - try { - index.crawl(); - assert.equal(index.search("token-one").length, 1); - - // Change content → re-index picks up the new term. - writeFileSync(join(root, "src", "a.ts"), "token-two"); - const changed = index.crawl(); - assert.equal(changed.indexed, 1); - assert.equal(index.search("token-two").length, 1); - assert.equal(index.search("token-one").length, 0); - - // Delete file → dropped from index. - rmSync(join(root, "src", "a.ts")); - const removed = index.crawl(); - assert.equal(removed.removed, 1); - assert.equal(index.search("token-two").length, 0); - } finally { - index.close(); - rmSync(root, { recursive: true, force: true }); - } -}); - -test("FileIndex: addScopePath grows the scope with dedup and cap", () => { - const { root, dataDir, scope } = fixture(); - writeFileSync(scope, "src/core\n", "utf8"); - const index = new FileIndex({ projectRoot: root, dataDir, scopePath: scope, maxScopePaths: 3 }); - try { - index.addScopePath("docs/design/"); - index.addScopePath("src/cli"); - index.addScopePath("docs/design"); // dedup (trailing slash stripped) - assert.deepEqual(index.readScope(), ["src/core", "docs/design", "src/cli"]); - // Cap: adding a 4th drops the oldest. - index.addScopePath("skills/"); - assert.deepEqual(index.readScope(), ["docs/design", "src/cli", "skills"]); - } finally { - index.close(); - rmSync(root, { recursive: true, force: true }); - } -}); - -test("FileIndex: defaultExclude skips VCS, node_modules, binaries, dot-dirs", () => { - assert.equal(defaultExclude("node_modules/x.js"), true); - assert.equal(defaultExclude("src/.git/config"), true); - assert.equal(defaultExclude("src/a.png"), true); - assert.equal(defaultExclude(".hidden/config.json"), true); - assert.equal(defaultExclude(".nmg-search-scope"), false); - assert.equal(defaultExclude("src/core/a.ts"), false); -}); diff --git a/tests/core/simhash.test.ts b/tests/core/simhash.test.ts new file mode 100644 index 0000000..bbcbe59 --- /dev/null +++ b/tests/core/simhash.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + hammingDistance, + simhash64, + simhashFromHex, + simhashToHex, +} from "../../src/core/simhash.ts"; + +test("simhash: identical text produces identical fingerprint", () => { + const content = "const x = 1;\n// a comment\nfunction alpha() { return x; }\n"; + assert.equal(simhash64(content), simhash64(content)); + assert.equal(simhash64(""), simhash64("")); +}); + +test("simhash: near-duplicate documents sit within the drift threshold", () => { + const original = [ + "The indexing pipeline batches embeddings before the store drains them.", + "A bounded passive-scope file index crawled project files on every search.", + "Tesserae are content-anchored bookmarks; snippets relocate on read.", + "SimHash fingerprints documents so a small edit stays recoverable.", + "Memory records carry provenance, scope and verification; files do not.", + ].join("\n"); + const edited = original.replace( + "batches embeddings before the store drains them", + "batches embeddings before the store drains them all", + ); + const fingerprint = simhash64(original); + const editedFingerprint = simhash64(edited); + assert.ok( + hammingDistance(fingerprint, editedFingerprint) <= 6, + `1%-scale edit stays within threshold (distance ${hammingDistance(fingerprint, editedFingerprint)})`, + ); +}); + +test("simhash: unrelated documents sit far beyond the threshold", () => { + const one = simhash64("The quick brown fox jumps over the lazy dog near the river bank at dusk."); + const two = simhash64( + "Database transactions must be atomic, consistent, isolated and durable by contract.", + ); + assert.ok( + hammingDistance(one, two) > 10, + `unrelated texts separate (distance ${hammingDistance(one, two)})`, + ); +}); + +test("simhash: hex round-trips losslessly across the full 64-bit range", () => { + const samples = [0n, 1n, 0xffffffffffffffffn, simhash64("any content"), 0xdeadbeefcafef00dn]; + for (const sample of samples) { + assert.equal(simhashFromHex(simhashToHex(sample)), sample); + assert.equal(simhashToHex(sample).length, 16, "canonical 16 hex chars"); + } +}); + +test("hammingDistance counts differing bits", () => { + assert.equal(hammingDistance(0n, 0n), 0); + assert.equal(hammingDistance(1n, 0n), 1); + assert.equal(hammingDistance(0xfn, 0x0n), 4); + assert.equal(hammingDistance(0xffffffffffffffffn, 0x0000000000000000n), 64); +}); + +test("simhashTokens lowercases words and isolates Han ideographs", () => { + // token stream is an implementation detail; exercise via fingerprint stability + const mixed = simhash64("NodeMemoryGraph 记忆系统 uses nodeMemoryGraph consistently"); + const stable = simhash64("nodememorygraph 记 忆 系 统 uses nodememorygraph consistently"); + // Case folding makes the English halves identical; Han runs tokenize per char, + // so splitting a Han run into separate characters must NOT change the result. + assert.equal(mixed, stable); +}); diff --git a/tests/core/tesserae-simhash.test.ts b/tests/core/tesserae-simhash.test.ts new file mode 100644 index 0000000..8ae0fd5 --- /dev/null +++ b/tests/core/tesserae-simhash.test.ts @@ -0,0 +1,165 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgService } from "../../src/cli/service.ts"; + +async function withProject( + run: (service: NmgService, projectDir: string) => void | Promise, +): Promise { + const root = mkdtempSync(join(tmpdir(), "nmg-tessera-simhash-")); + const service = new NmgService({ databasePath: join(root, "nmg.sqlite"), environment: {} }); + const projectDir = join(root, "project"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync( + join(projectDir, "alpha.ts"), + [ + "export const alpha = 1;", + "// The indexing pipeline drains embedding batches.", + "export function beta() { return alpha; }", + ].join("\n"), + ); + try { + await run(service, projectDir); + } finally { + service.close(); + rmSync(root, { recursive: true, force: true }); + } +} + +test("tessera simhash: write-time fingerprint stored and snippet relocates exactly", async () => { + await withProject(async (service, projectDir) => { + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "alpha.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const hit = searched.tesserae?.find((tessera: { path: string }) => tessera.path === "alpha.ts"); + assert.ok(hit, "tessera hit surfaces in search"); + assert.equal(hit.line, 2, "exact relocation resolves the current line"); + assert.ok(!hit.relocated, "exact hit is not marked relocated"); + assert.match(String(hit.fileSimhash ?? ""), /^[0-9a-f]{16}$/u, "write-time fingerprint stored"); + }); +}); + +test("tessera simhash: snippet survives a small edit elsewhere in the file", async () => { + await withProject(async (service, projectDir) => { + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "alpha.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + // Small edit in a DIFFERENT line: the snippet itself is untouched, so the + // tessera resolves exactly (relocation is content-anchored, not line-based). + writeFileSync( + join(projectDir, "alpha.ts"), + [ + "export const alpha = 1; // bumped", + "// The indexing pipeline drains embedding batches.", + "export function beta() { return alpha; }", + ].join("\n"), + ); + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const hit = searched.tesserae?.find((tessera: { path: string }) => tessera.path === "alpha.ts"); + assert.ok(hit, "tessera hit still surfaces"); + assert.equal(hit.line, 2, "exact relocation survives the edit"); + assert.ok(!hit.relocated, "not relocated — the stored path still matches"); + }); +}); + +test("tessera simhash: file moved elsewhere in the project is recovered", async () => { + await withProject(async (service, projectDir) => { + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "alpha.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + // Move the file to a subdirectory with a tiny content touch. + mkdirSync(join(projectDir, "lib"), { recursive: true }); + writeFileSync( + join(projectDir, "lib", "alpha.ts"), + [ + "export const alpha = 1;", + "// The indexing pipeline drains embedding batches.", + "export function beta() { return alpha; }", + ].join("\n"), + ); + // Remove the original so the stored path is gone. + const { rmSync } = await import("node:fs"); + rmSync(join(projectDir, "alpha.ts")); + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const hit = searched.tesserae?.find( + (tessera: { path: string }) => tessera.path === "lib/alpha.ts", + ); + assert.ok(hit, "moved file tessera recovered at the new path"); + assert.equal(hit.line, 2); + assert.ok(hit.relocated, "recovered hit is marked relocated"); + }); +}); + +test("tessera simhash: unrelated content rewrite reports stale, not relocated", async () => { + await withProject(async (service, projectDir) => { + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "alpha.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + // Full rewrite: unrelated document — must stay honestly stale. + writeFileSync( + join(projectDir, "alpha.ts"), + [ + "import { z } from 'zod';", + "export const schema = z.object({ id: z.string() });", + "export type Row = z.infer;", + ].join("\n"), + ); + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const hit = searched.tesserae?.find((tessera: { path: string }) => tessera.path === "alpha.ts"); + assert.ok(hit, "tessera hit still surfaces"); + assert.equal(hit.line, undefined, "no line resolved"); + assert.ok(hit.stale, "honestly stale after unrelated rewrite"); + }); +});