feat(tesserae): SimHash drift relocation + remove FileIndex code debt (ticket 8)

docs/design/memory-tesserae-design.md

@@ -64,6 +64,11 @@ Rationale: maintaining a file index is expensive and its marginal value over
 tesserae + direct tool access is low. The file is the content host; NMG only
 needs *pointers into it*.
 
+The design declared the drop; the **code removal is ticket 8** (the original
+tesserae PR shipped with `file-index.ts` still live — every search still
+crawled it). Removing the full-text machinery is prerequisite to the drift
+tolerance below: tesserae need a file fingerprint, not a file index.
+
 ### 3.2 Tesserae are an independent, searchable source
 
 A **tessera** (a bookmark) is a first-class row, not a field glued onto a
@@ -93,16 +98,35 @@ memory is superseded), a matching tessera is still found.
 
 A tessera stores a **content snippet**, never a line number. Line numbers drift
 on every edit; content is relocatable. To resolve a tessera, locate its snippet
-in the current file (exact match → position; fuzzy fallback → nearest match;
-absent → tessera is stale). This is the established pattern from
+in the current file (exact match → position; absent → tessera is stale). This
+is the established pattern from
 [gptme hash-anchored editing](https://github.com/gptme/gptme/blob/ae707fc8233e77d4da97fc74f94db1eaff1e381a/gptme/tools/_anchored.py),
 [agentic-bookmarks self-healing anchors](https://github.com/super-mega-lab/agentic-bookmarks),
 and [haido `hash_at_link` drift detection](https://github.com/lebac-svg/haido/blob/HEAD/docs/DESIGN.md):
 never persist a position that the file can invalidate; persist content and
 relocate.
 
-Staleness is **objective**: resolve on read; if the snippet no longer exists,
-report the tessera as stale (memory stays valid — only the position is gone).
+**Drift tolerance (ticket 8).** A tessera additionally stores a 64-bit SimHash
+fingerprint of its target file (`tesserae.file_simhash`, computed once at write
+time). Relocation is two-stage:
+
+1. **Exact** — locate the snippet line in `tessera.path` (`includes`). Hit → done.
+2. **Fingerprint fallback** — exact miss does not immediately mean stale. Compare
+   the stored file SimHash against the current files in scope (all readable
+   files under the project). If one is within Hamming ≤ 6 — the same document
+   after small edits, or the file after a move — re-locate the snippet against
+   that candidate file. SimHash is document-level: measured on real repo files
+   (5–60 KB), near-identical pairs sit at Hamming 1–3 and unrelated at ~24, so
+   ≤ 6 cleanly separates (100% recall / 0.24% false positive), while short
+   memory/snippet text has no such signal and is never fingerprinted this way.
+
+The fingerprint finds a *candidate file*; the snippet match confirms the exact
+position. The tessera row is never auto-rewritten — the caller decides whether
+to update `path` after confirmation.
+
+Staleness is **objective**: resolve on read; if the snippet no longer exists
+anywhere the fingerprint points (exact or fallback), report the tessera as
+stale (memory stays valid — only the position is gone).
 
 ### 3.4 Markers are the index pointer between memory and tessera
 
@@ -171,7 +195,10 @@ conclusions that are still true — files are not memory; scope discipline
 matters; separated presentation is sane — but **drops the file index itself** in
 favor of sparse, Agent-authored tesserae. The maintenance-heavy machinery
 (`.nmg-search-scope`, incremental crawler, file FTS, scope observer) is not part
-of this design.
+of this design, and ticket 8 removes the code that PR #18 left behind
+(`file-index.ts`, the per-search crawl, the DSH scope observer). The only
+machine-derived file signal tesserae keep is a single 64-bit SimHash per
+target file (§3.3) — a drift detector, not an index.
 
 ## 7. Open questions (deferred)
 

dsh/dsh-nmg/src/plugin/cordis-augment.d.ts

@@ -85,15 +85,6 @@ declare module '@deepseek-ai/cordis' {
       result: Readonly<{ isError: boolean; value?: unknown; content?: unknown }>,
     ): undefined
   }
-
-  interface Context {
-    /**
-     * Optional NMG file-content index (src/core/file-index.ts). Consumed
-     * opportunistically by the `tools/result` scope observer; never injected, so
-     * the plugin mounts before/without the index and skips recording when absent.
-     */
-    fileIndex?: { addScopePath(path: string): void }
-  }
 }
 
 export {}

dsh/dsh-nmg/src/plugin/index.ts

@@ -29,7 +29,6 @@ import {
   renderTaskBoardSurface,
 } from '../../../../src/integration/agent-surface.ts'
 import { loadPrompts, renderDisclosure } from '../../../../src/prompts/load.ts'
-import { FileIndex } from '../../../../src/core/file-index.ts'
 
 const nmgPrompts = loadPrompts()
 
@@ -1505,11 +1504,6 @@ export function apply(ctx: Context): () => void {
       })
     : undefined
 
-  // ── file-content scope observer (tools/result) ─────────────────────────────
-  const scopeObserver = setupScopeObserver(ctx, workspaceRoot)
-  const scopeObserverDisposer = scopeObserver.disposer
-  const provideFileIndex = scopeObserver.provide
-
   // ── wake timer + startup ───────────────────────────────────────────────────
   // Single host-side timer polls the daemon for board entries every
   // WAKE_INTERVAL_MS. The first poll also registers the agent and starts the
@@ -1535,8 +1529,6 @@ export function apply(ctx: Context): () => void {
     contextDisposer,
     ctx.on('agent/inbox/inserted', onInboxInserted),
     ctx.on('agent/disposed', onAgentDisposed),
-    scopeObserverDisposer,
-    ...(provideFileIndex ? [provideFileIndex] : []),
   ]
   if (coordinationEnabled) {
     disposers.push(
@@ -1560,84 +1552,6 @@ export function apply(ctx: Context): () => void {
     recallBatch.clear()
     openSearches.clear()
     wakeBatch.clear()
-    try { scopeObserver.service.close() } catch { /* best-effort */ }
-  }
-}
-
-// ── file-content scope observer (tools/result) ───────────────────────────────
-// The file content source learns its search scope from the Agent's own search
-// behaviour (docs/design/file-content-source-design.md §3.2): a non-empty
-// `grep`/`read` result marks the searched files as hot zones, which the
-// FileIndex records via addScopePath. This listener observes only — it never
-// intercepts or mutates the tool lifecycle. Any parse failure is silent.
-//
-// The FileIndex is provided as an optional `fileIndex` service (addScopePath
-// only) so the observer degrades to a no-op if the service is ever absent.
-function setupScopeObserver(ctx, workspaceRoot) {
-  const service = new FileIndex({ projectRoot: workspaceRoot })
-  const provide = ctx.provide
-    ? ctx.provide('fileIndex', { addScopePath: (path) => service.addScopePath(path) })
-    : undefined
-  const collectScopePaths = (exec, result) => {
-    // Guard clauses keep the main flow linear (see complexity-reduction
-    // practice: guard clause + composed functions).
-    const name = (exec && (exec.name || exec.toolName)) || ''
-    if (name !== 'grep' && name !== 'read') return
-    const fileIndex = ctx.get('fileIndex')
-    if (!fileIndex || typeof fileIndex.addScopePath !== 'function') return
-    const paths = extractHitPaths(exec, result, name)
-    for (const path of paths) {
-      try {
-        fileIndex.addScopePath(path)
-      } catch {
-        // recording is best-effort; a failing index must not break the tool
-      }
-    }
-  }
-  const disposer = ctx.on('tools/result', (exec, result) => {
-    try {
-      collectScopePaths(exec, result)
-    } catch {
-      // observation never throws into the tool registry
-    }
-  })
-  return { disposer, provide, service }
-}
-
-/** Extract hot-zone candidate paths from a grep/read tool result. The tool
- *  name is `grep` (dsh-tool-fs-search) or `read` (dsh-tool-fs); glob is a
- *  separate tool we intentionally do not observe. */
-function extractHitPaths(exec, result, name) {
-  // Arguments: `exec.arguments` (dsh-tools) with `exec.input` as an alias.
-  const args = (exec && (exec.arguments || exec.input)) || {}
-  const value = result && result.isError ? undefined : result && result.value
-  if (name === 'grep') return grepHitPaths(value, args)
-  if (name === 'read') return readHitPaths(value, args)
-  return []
-}
-
-/** grep hit paths: match files (workdir-relative) + the searched path arg
- *  when the search found something. */
-function grepHitPaths(value, args) {
-  const paths = []
-  const matches = value && Array.isArray(value.matches) ? value.matches : []
-  for (const match of matches) {
-    if (match && typeof match.path === 'string' && match.path) paths.push(match.path)
-  }
-  if (paths.length > 0 && args && typeof args.path === 'string' && args.path) {
-    paths.push(args.path)
   }
-  return paths
 }
 
-/** read hit paths: a non-empty read ⇔ at least one line; record the requested
- *  path arg and the resolved display path (they usually agree). */
-function readHitPaths(value, args) {
-  const paths = []
-  const lines = value && Array.isArray(value.lines) ? value.lines : []
-  if (lines.length > 0) {
-    if (args && typeof args.file_path === 'string' && args.file_path) paths.push(args.file_path)
-    if (value && typeof value.path === 'string' && value.path) paths.push(value.path)
-  }
-  return paths
-}

src/cli/main.ts

@@ -434,19 +434,12 @@ function parseOptions(args: readonly string[]): OptionValues {
   return { flags, options, positionals };
 }
 
-/** Append the FILES / TESSERAE source-section lines of a search result. Kept as
- *  its own helper so humanResult's branching stays under the complexity gate. */
+/** Append the TESSERAE source-section lines of a search result. Kept as its
+ *  own helper so humanResult's branching stays under the complexity gate. */
 function sourceSectionLines(
-  files: Array<{ path: string; excerpt: string }> | undefined,
   tesserae: Array<{ path: string; label: string; line?: number; stale?: boolean }> | undefined,
 ): string[] {
   const lines: string[] = [];
-  if (files && files.length > 0) {
-    lines.push("FILES:");
-    for (const file of files) {
-      lines.push(`${file.path}\t${file.excerpt}`);
-    }
-  }
   if (tesserae && tesserae.length > 0) {
     lines.push("TESSERAE:");
     for (const tessera of tesserae) {
@@ -604,15 +597,14 @@ function humanResult(value: unknown): string {
         memory: { id: string; memoryType: string; tier: number; statement: string };
         node: { canonicalName: string };
       }>;
-      files?: Array<{ path: string; excerpt: string }>;
       tesserae?: Array<{ path: string; label: string; line?: number; stale?: boolean }>;
       timings?: { timings?: Record<string, number>; totalMs?: number };
     };
     const lines = context.results.map(
       ({ memory, node }) =>
         `${memory.id}\t${memory.memoryType}\tL${memory.tier}\t${node.canonicalName}\t${memory.statement}`,
     );
-    lines.push(...sourceSectionLines(context.files, context.tesserae));
+    lines.push(...sourceSectionLines(context.tesserae));
     if (context.timings) {
       const sections = Object.entries(context.timings.timings ?? {})
         .sort((left, right) => right[1] - left[1])