Replace SOMA with the LexCAT lexical runtime (v0.0.14)

.gitattributes

@@ -1 +1,2 @@
-.github/workflows/*.lock.yml linguist-generated=true merge=ours
+.github/workflows/*.lock.yml linguist-generated=true merge=ours
+vendor/lexcat/** -text

.github/workflows/ci.yml

@@ -38,33 +38,37 @@ jobs:
       - name: Run offline checks
         run: npm run check:offline
 
-  soma-runtimes:
-    name: SOMA ${{ matrix.name }}
+  lexcat-runtimes:
+    name: LexCAT ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - name: Linux x64
             os: ubuntu-latest
-            archive: soma-v0.3.0-linux-x86_64.tar.gz
-            executable: soma
+            platform: linux
+            arch: x64
+            archive: lexcat-v0.0.14-linux-x86_64.tar.gz
+            executable: lexcat
           - name: Linux arm64
             os: ubuntu-24.04-arm
-            archive: soma-v0.3.0-linux-arm64.tar.gz
-            executable: soma
+            platform: linux
+            arch: arm64
+            archive: lexcat-v0.0.14-linux-arm64.tar.gz
+            executable: lexcat
           - name: macOS arm64
             os: macos-15
-            archive: soma-v0.3.0-macos-arm64.tar.gz
-            executable: soma
+            platform: darwin
+            arch: arm64
+            archive: lexcat-v0.0.14-macos-arm64.tar.gz
+            executable: lexcat
           - name: Windows x64
             os: windows-latest
-            archive: soma-v0.3.0-windows-x86_64.zip
-            executable: soma.exe
-          - name: Windows arm64
-            os: windows-11-arm
-            archive: soma-v0.3.0-windows-arm64.zip
-            executable: soma.exe
+            platform: win32
+            arch: x64
+            archive: lexcat-v0.0.14-windows-x86_64.zip
+            executable: lexcat.exe
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -74,56 +78,103 @@ jobs:
         with:
           node-version: 24
 
-      - name: Cache retrieval model
-        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
-        with:
-          path: ${{ runner.temp }}/soma-model
-          key: soma-model-v0.3.0-${{ runner.os }}-${{ runner.arch }}
-
-      - name: Exercise shipped SOMA binary
+      - name: Exercise shipped LexCAT binary
         shell: bash
         run: |
           set -euo pipefail
-          runtime_dir="$RUNNER_TEMP/soma-runtime"
+          runtime_dir="$RUNNER_TEMP/lexcat-runtime"
           mkdir -p "$runtime_dir/corpus"
-          archive="vendor/soma/${{ matrix.archive }}"
+          archive="vendor/lexcat/${{ matrix.archive }}"
           if [[ "$archive" == *.zip ]]; then
             7z x "$archive" -o"$runtime_dir"
           else
             tar -xzf "$archive" -C "$runtime_dir"
           fi
           chmod +x "$runtime_dir/${{ matrix.executable }}"
-          "$runtime_dir/${{ matrix.executable }}" --version | grep -F "0.3.0"
+          # 0.0.14 stamps the real release version into the binary (upstream #234),
+          # so --version pins both the release number and the index schema, on top
+          # of the manifest checksum verified above.
+          node -e '
+            const { createHash } = require("node:crypto");
+            const { readFileSync } = require("node:fs");
+            const manifest = JSON.parse(readFileSync("vendor/lexcat/manifest.json", "utf8"));
+            const artifact = manifest.artifacts.find((entry) => entry.platform === process.argv[2] && entry.arch === process.argv[3]);
+            if (!artifact) throw new Error("no vendored artifact for " + process.argv[2] + "/" + process.argv[3]);
+            const digest = createHash("sha256").update(readFileSync(process.argv[1])).digest("hex");
+            if (digest !== artifact.executable_sha256) throw new Error("executable checksum mismatch: " + digest);
+          ' "$runtime_dir/${{ matrix.executable }}" "${{ matrix.platform }}" "${{ matrix.arch }}"
+          schema="$(node -p 'JSON.parse(require("node:fs").readFileSync("vendor/lexcat/manifest.json","utf8")).index_schema_version')"
+          version="$(node -p 'JSON.parse(require("node:fs").readFileSync("vendor/lexcat/manifest.json","utf8")).version')"
+          "$runtime_dir/${{ matrix.executable }}" --version | grep -q "index schema $schema"
+          "$runtime_dir/${{ matrix.executable }}" --version | grep -q "lexcat $version"
           cp -R wiki-mirror/. "$runtime_dir/corpus/"
           cd "$runtime_dir"
-          "./${{ matrix.executable }}" index build corpus --name ci-smoke --title-field _stem --include-types md --no-incremental
-          index_db="$(find "$runtime_dir" -path '*/indexes/ci-smoke/index.db' -print -quit)"
-          test -n "$index_db"
-
-      - name: Exercise SOMA retrieval
+          # Mirrors how WikiKB indexes: no config file, counts read out of the
+          # machine-readable report. An empty corpus and a collapsed vocabulary
+          # both build and query at exit 0, so both counts are asserted here.
+          "./${{ matrix.executable }}" --index ci-smoke.db build corpus --json > build.json
+          cat build.json
+          node -e '
+            const { readFileSync } = require("node:fs");
+            const report = JSON.parse(readFileSync("build.json", "utf8"));
+            if (!(report.chunks > 0)) throw new Error("build indexed no chunks: " + JSON.stringify(report));
+            if (!(report.terms > 0)) throw new Error("build indexed no terms: " + JSON.stringify(report));
+          '
+          test -f ci-smoke.db
+
+      - name: Exercise LexCAT retrieval
         shell: bash
         run: |
           set -euo pipefail
-          runtime_dir="$RUNNER_TEMP/soma-runtime"
-          model_dir="$RUNNER_TEMP/soma-model"
-          if [[ ! -f "$model_dir/model.safetensors" ]]; then
-            "$runtime_dir/${{ matrix.executable }}" util models install model2vec-potion-retrieval-32m \
-              --revision 6fc8051fab2a1e0ee76689cf08c853792ac285e7 \
-              --output "$model_dir"
-          fi
-          node -e 'const fs=require("fs"); fs.writeFileSync(process.argv[1], JSON.stringify({query:{model2vec_model_path:process.argv[2]}}));' "$runtime_dir/query-preset.json" "$model_dir"
-          index_db="$(find "$runtime_dir" -path '*/indexes/ci-smoke/index.db' -print -quit)"
-          "$runtime_dir/${{ matrix.executable }}" query --index "$(dirname "$index_db")" --preset "$runtime_dir/query-preset.json" --max-tokens 4000 --output - "agentic workflows" > "$runtime_dir/context.json"
-          node -e 'const fs=require("fs"); const p=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const groups=[...(p.communities||[]),...(p.topics||[])]; const chunks=[...(p.chunks||[]),...groups.flatMap(c=>c.chunks||[])]; if(!chunks.some(c=>String(c.text||"").toLowerCase().includes("agentic workflow"))) process.exit(1);' "$runtime_dir/context.json"
-
-      - name: Exercise WikiKB runtime integration and concurrent model use
+          runtime_dir="$RUNNER_TEMP/lexcat-runtime"
+          cd "$runtime_dir"
+          "./${{ matrix.executable }}" --index ci-smoke.db query "agentic workflows" --n 10 --json > hits.json
+          # --json carries each hit's chunk text, which is how WikiKB reads
+          # results, so the assertion never touches the on-disk schema.
+          node -e '
+            const { readFileSync } = require("node:fs");
+            const { hits } = JSON.parse(readFileSync("hits.json", "utf8"));
+            if (!Array.isArray(hits) || hits.length === 0) throw new Error("LexCAT returned no hits");
+            const matched = hits.some((hit) => String(hit.text ?? "").toLowerCase().includes("agentic workflow"));
+            if (!matched) throw new Error("no retrieved chunk contained the query terms");
+          '
+
+      - name: Exercise incremental reindex
+        shell: bash
+        run: |
+          set -euo pipefail
+          runtime_dir="$RUNNER_TEMP/lexcat-runtime"
+          cd "$runtime_dir"
+          printf -- '---\ntitle: "CI Delta"\nwikikb_path: "concepts/ci-delta.md"\n---\n\n# CI Delta\n\nA sentinel page mentioning vermiculite telemetry.\n' > corpus/ci-delta.md
+          "./${{ matrix.executable }}" --index ci-smoke.db sync corpus --json > sync.json
+          cat sync.json
+          node -e '
+            const { readFileSync } = require("node:fs");
+            const report = JSON.parse(readFileSync("sync.json", "utf8"));
+            if (!(report.chunks > 0) || !(report.terms > 0)) throw new Error("sync left an unsearchable index: " + JSON.stringify(report));
+            if (!(report.added >= 1)) throw new Error("sync did not report the added page: " + JSON.stringify(report));
+          '
+          "./${{ matrix.executable }}" --index ci-smoke.db query "vermiculite telemetry" --n 5 --json > delta.json
+          # Frontmatter staged with a document must come back on the hit payload,
+          # because that is where WikiKB reads a result's wiki path and title.
+          node -e '
+            const { readFileSync } = require("node:fs");
+            const { hits } = JSON.parse(readFileSync("delta.json", "utf8"));
+            const hit = hits.find((entry) => String(entry.doc_id ?? "") === "ci-delta.md");
+            if (!hit) throw new Error("incremental sync did not make the new page retrievable");
+            if (hit.payload?.fields?.wikikb_path !== "concepts/ci-delta.md") {
+              throw new Error("frontmatter payload missing: " + JSON.stringify(hit.payload));
+            }
+          '
+
+      - name: Exercise WikiKB runtime integration
         shell: bash
         run: |
           set -euo pipefail
           npm ci
           npm run build:wkb
-          WIKIKB_SOMA_MODEL_DIR="$RUNNER_TEMP/soma-model" node --test \
-            --test-name-pattern='vendored SOMA indexes|concurrent retrieval' \
+          node --test \
+            --test-name-pattern='vendored LexCAT indexes|concurrent retrieval' \
             tools/wikikb-local/test/smoke.mjs
 
   workflows:

.gitignore

@@ -2,6 +2,6 @@
 .env
 tests/integration/.env
 node_modules/
-.soma/
+.lexcat/
 tools/wikikb-local/dist/
 release/

LICENSE

@@ -3,11 +3,11 @@ WikiKB License
 Except for the third-party material identified below, WikiKB is licensed under
 the MIT License:
 
-- The SOMA binary archives and executables under `vendor/soma/` are
+- The LexCAT binary archives and executables under `vendor/lexcat/` are
   not licensed under the MIT License. Microsoft has authorized their
   redistribution as unchanged compiled components of WikiKB; separate use,
   modification, sublicensing, or relicensing is not granted here. See
-  `vendor/soma/THIRD_PARTY_NOTICES.txt`.
+  `vendor/lexcat/THIRD_PARTY_NOTICES.txt`.
 
 The MIT License follows.
 

README.md

@@ -1,8 +1,8 @@
 # WikiKB
 
-WikiKB is an efficient, semantic knowledge base for humans and agents, stored in a GitHub repository's wiki. Content can be ingested or queried from the command line, through GitHub Issues, or as an agent skill.
+WikiKB is an efficient, retrieval-backed knowledge base for humans and agents, stored in a GitHub repository's wiki. Content can be ingested or queried from the command line, through GitHub Issues, or as an agent skill.
 
-An LLM provider is not required to ingest content, or to search the knowledge base. A small, local embedding model runs entirely on CPU, either on the client machine or within GitHub Actions. 
+An LLM provider is not required to ingest content, or to search the knowledge base. A local, model-free LexCAT BM25 engine runs entirely on CPU, either on the client machine or within GitHub Actions.
 
 Retrieval-augmented generation (RAG) operations, such as summarization and question-answering, can use any configured AI provider.
 
@@ -29,7 +29,7 @@ export PATH="$HOME/.local/bin:$PATH"
 export WIKIKB_GITHUB_TOKEN="$(gh auth token)"
 ```
 
-The manual installer places a checkout-backed launcher in `~/.local/bin`; `WKB_INSTALL_DIR` changes the destination. The release includes the SOMA executables for macOS arm64, Linux x64/arm64, and Windows x64/arm64. Other platforms are not supported.
+The manual installer places a checkout-backed launcher in `~/.local/bin`; `WKB_INSTALL_DIR` changes the destination. The release includes the LexCAT executables for linux/x64, darwin/arm64, and win32/x64. Other platforms must set `WIKIKB_LEXCAT_BIN` to an operator-approved executable.
 
 ### GitHub CLI extension
 
@@ -56,7 +56,7 @@ This writes `~/.agents/skills/wikikb-memory/SKILL.md` and its agent metadata. Ex
 wkb add ai-research owner/repository
 wkb ai-research sync
 wkb ai-research search "hybrid retrieval methods" --top 5
-wkb ai-research query "How does graph-based retrieval differ from vector search?" --no-ai
+wkb ai-research query "How does graph-based retrieval differ from keyword search?" --no-ai
 ```
 
 Here, `ai-research` is the local name registered for `owner/repository`. The
@@ -80,7 +80,7 @@ discussions:
 
 ```bash
 wkb ai-research.sources.tool-discussions ingest-issues tool-owner/tool-repository --state all --limit 50 --comments
-wkb ai-research.sources.tool-discussions search "embedding quality" --top 10
+wkb ai-research.sources.tool-discussions search "ranking quality" --top 10
 ```
 
 Finally, retrieve relevant entries from the whole `ai-research` wiki and
@@ -102,7 +102,7 @@ wkb ai-research summarize "Summarize the main approaches to retrieval-augmented
 | `wkb skills install [--force] [--path directory]` | Install the WikiKB agent skill |
 | `wkb <target> sync` | Clone or update the wiki |
 | `wkb <target> status` | Show local state |
-| `wkb <target> index [--force]` | Restore, update, and share an index |
+| `wkb <target> index [--force]` | Restore, rebuild, and share an index |
 | `wkb <target> search <query> [--top N] [--tag tags]` | Return ranked context |
 | `wkb <target> query <question> [options]` | Retrieve context and optionally answer |
 | `wkb <target> summarize\|rewrite\|extract\|timeline <request>` | Retrieve and run a prompt task |
@@ -117,7 +117,7 @@ Run `wkb --help` for options.
 
 ## Retrieval And Cache
 
-WikiKB uses the local SOMA indexing and retrieval backend, distributed as checksum-verified platform binaries.
+WikiKB uses the local LexCAT model-free lexical BM25 indexing and retrieval backend, distributed as checksum-verified platform binaries.
 
 Generated indexes are stored as bounded, checksum-verified archives on the wiki repository's parentless `wikikb-cache-v1` branch. The cache branch contains no wiki Markdown. Reads sync, restore a compatible index, or build and publish one. Offline work stays local and retries later; `--no-push` content never enters the shared cache.
 
@@ -140,6 +140,6 @@ npm ci
 npm run release:check
 ```
 
-The live suite is documented in [Integration Tests](tests/integration/README.md). The supported artifact contains only the CLI, Agentic Workflows, approved runtime binaries, and supporting source. WikiKB source is MIT-licensed; the vendored SOMA binaries are distributed under separate terms.
+The live suite is documented in [Integration Tests](tests/integration/README.md). The supported artifact contains only the CLI, Agentic Workflows, approved runtime binaries, and supporting source. WikiKB source is MIT-licensed; the vendored LexCAT binaries are distributed under separate terms.
 
 Reference: [Architecture](docs/architecture.md), [Agent Memory](docs/agent-memory.md), [Release Scope](docs/release-scope.md), [Release Checklist](docs/release-checklist.md), [Contributing](CONTRIBUTING.md), and [License](LICENSE).

SKILL.md

@@ -11,7 +11,7 @@ export PATH="$HOME/.local/bin:$PATH"
 export WIKIKB_GITHUB_TOKEN="$(gh auth token)"
 ```
 
-Node.js 22+ and SOMA are required. Retrieval has no alternate backend.
+Node.js 22+ and LexCAT are required. Retrieval has no alternate backend.
 
 ## Agent Contract
 
@@ -42,7 +42,7 @@ wkb prompts list|init|path|show <name>
 Dots select up to five namespace levels and include descendants. Tag filters use AND semantics.
 Generation options are `--ai`, `--provider`, `--model`, `--show-prompt`, `--rewrite-query`, `--prompt`, and `--task`. Use `query --no-ai` to return retrieved evidence without generation.
 
-Reads sync the wiki and restore or refresh its shared index. Writes push by default and refresh the selected index. `--no-push` content remains uncommitted and cannot enter the shared cache. A requested push that cannot be published fails.
+Reads sync the wiki and restore or fully rebuild its shared index. Writes push by default and rebuild the selected index. `--no-push` content remains uncommitted and cannot enter the shared cache. A requested push that cannot be published fails.
 
 ## AI
 

docs/agent-memory.md

@@ -20,7 +20,7 @@ wkb project.decisions search "What did we decide?"
 
 Use `query` only for a generated answer. It requires an explicit AI provider and model; `search` never calls a generation provider. See [Configuration](configuration.md).
 
-SOMA is the only retrieval backend. `queries/` pages remain generated claims; verify them against cited sources or concepts.
+LexCAT is the only retrieval backend. It performs model-free lexical BM25 retrieval. `queries/` pages remain generated claims; verify them against cited sources or concepts.
 
 ## Write
 

docs/architecture.md

@@ -8,7 +8,7 @@ WikiKB implements Karpathy's [LLM Wiki](https://gist.github.com/karpathy/442a6bf
 | --- | --- |
 | GitHub wiki | Source, concept, query, and navigation pages |
 | `wkb` | Sync, retrieval, prompting, ingestion, and maintenance |
-| SOMA | Only indexing and retrieval backend |
+| LexCAT | Only indexing and retrieval backend |
 | `wikikb-cache-v1` | Shared generated indexes |
 | `~/.wikikb` | Registry, clones, prompts, runtime, and local indexes |
 | Agentic Workflows | Issue-driven reads and controlled writes |
@@ -31,9 +31,9 @@ The shared branch contains pairs only:
 .wikikb-cache/v1/indexes/<index>.tar.gz
 ```
 
-Each manifest binds an archive to exact Markdown, namespace, indexing contract, runtime, size, and checksums. Snapshots are parentless, pushed with `--force-with-lease`, retain at most eight indexes, and contain no Markdown.
+Each manifest binds an archive containing a single SQLite index file to exact Markdown, namespace, indexing contract, runtime, size, and checksums. Snapshots are parentless, pushed with `--force-with-lease`, retain at most eight indexes, and contain no Markdown.
 
-Reads sync pending commits, digest the selected Markdown, then reuse, restore, or build an index. A new index is shared only after its Markdown is remote. Retrieval stops on any runtime, model, output, integrity, or empty-context failure.
+Reads sync pending commits, digest the selected Markdown, then reuse, restore, or fully rebuild an index. A new index is shared only after its Markdown is remote. Retrieval stops on any runtime, output, integrity, or empty-context failure.
 
 Writes normalize a file, public HTTPS URL, or issue into `sources/`, stage only operation-owned paths, and push unless `--no-push` is set. URL redirects are revalidated, private/local destinations are rejected, explicit titles are honored, and same-title sources cannot overwrite each other. A requested push must reach the wiki or the command fails; uncommitted content cannot enter the shared cache.