From 1ba6d9fbe5f5357f3d6913cae98954bb2626fcb6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 11 Oct 2026 12:29:15 +0700 Subject: [PATCH] fix(release): optimize semantic query reuse and bound model inference Release highlights: - make repeated semantic searches faster while preserving retrieved evidence and citations - enforce the configured batch limit for every semantic embedding call Release details: - **Core:** reuse single-query vectors in a bounded project- and model-scoped memory cache - **Core:** preserve batch order, cancellation, and model cleanup without caching search results - **Documentation:** document cache limits and the current-index retrieval guarantee - **Toolchain:** update Handlebars and HTTP cache semantics and remove the old workspace sprintf-js chain through Mammoth's compatible CLI dependency override - **CI:** lint the pull request commit range directly from its base and head revisions Verification: - pass pnpm validate with 481 Core tests and 24 landing tests plus coverage, API, MCP, builds, packaging, license policy, and release smoke checks - pass Linux x64, macOS ARM64, Commitlint, and CodeQL on the feature pull request - compare three alternating local runs with 288 repeated and 288 unique embeddings and 144 searches per variant, preserving all vector and search fingerprints - measure repeated-search median latency from 29.43 ms to 25.32 ms on the local reference base - verify project/model isolation, cache budgets, cancellation, disposal, and refreshed evidence --- .github/workflows/ci.yml | 19 +- docs/configuration.md | 4 + packages/ragmir-core/src/embeddings.test.ts | 203 ++++++++++++++++++++ packages/ragmir-core/src/embeddings.ts | 81 +++++++- pnpm-lock.yaml | 31 +-- pnpm-workspace.yaml | 6 +- 6 files changed, 292 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ff7ccc..f5e1598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,21 +127,4 @@ jobs: run: pnpm install --frozen-lockfile - name: Lint commit messages - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - - # A release PR promotes an already-reviewed develop history through one merge commit. - # Lint its first-parent commits so inherited history is not checked a second time. - if [[ "$BASE_REF" == "main" && "$HEAD_REF" == release/* ]]; then - while IFS= read -r commit; do - git show -s --format=%B "$commit" | pnpm exec commitlint --verbose - done < <(git rev-list --first-parent "$BASE_SHA..$HEAD_SHA") - exit 0 - fi - - pnpm exec commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose + run: pnpm exec commitlint --from "${{ github.event.pull_request.base.sha }}" --to "${{ github.event.pull_request.head.sha }}" --verbose diff --git a/docs/configuration.md b/docs/configuration.md index 6f36fee..d6651c8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -148,6 +148,10 @@ mutable and therefore unverified. Transformers pipelines are shared per exact model identity and disposed when the final `RagmirClient` owner closes. Cache retirement waits for active inference leases, so a model switch or shutdown does not dispose a session still serving a request. +Repeated single-query embeddings use a project-scoped, in-memory LRU cache tied to that model +identity, capped at 128 entries and 65,536 vector values. The cache retains hashed query keys and +vectors, never search results; each search still reads the current index generation. Every semantic +model call respects `embeddingBatchSize`, including callers that supply a larger input array. ## Local extractors diff --git a/packages/ragmir-core/src/embeddings.test.ts b/packages/ragmir-core/src/embeddings.test.ts index 757b042..443acc0 100644 --- a/packages/ragmir-core/src/embeddings.test.ts +++ b/packages/ragmir-core/src/embeddings.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { RagmirClient } from "./client.js" import { clearTransformersCache, disposeTransformersCache, @@ -105,6 +106,208 @@ describe("local hash embeddings", () => { }) describe("embedding model adapters", () => { + it("reuses a single-query embedding without caching document vectors or sharing mutable arrays", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [0.25, 0.75]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + + const first = await embedText("same evidence", config) + first[0] = 99 + expect(await embedText("same evidence", config)).toEqual([0.25, 0.75]) + await embedTexts(["same evidence"], config) + await embedTexts(["same evidence"], config) + + expect(extractor).toHaveBeenCalledTimes(3) + }) + + it("isolates query vectors by project root and exact model identity", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + await embedText("same evidence", config) + await embedText("same evidence", { ...config, projectRoot: "/tmp/another-project" }) + await embedText("same evidence", { ...config, embeddingModelRevision: "new-revision" }) + await embedText("same evidence", { + ...config, + embeddingModelDigest: `sha256:${"a".repeat(64)}`, + }) + + expect(extractor).toHaveBeenCalledTimes(4) + expect(transformersMock.pipeline).toHaveBeenCalledTimes(3) + }) + + it("evicts the least recently used query when the entry limit is reached", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + for (let index = 0; index < 128; index += 1) await embedText(`query ${index}`, config) + await embedText("query 0", config) + await embedText("query 128", config) + await embedText("query 0", config) + expect(extractor).toHaveBeenCalledTimes(129) + + await embedText("query 1", config) + expect(extractor).toHaveBeenCalledTimes(130) + }) + + it("bounds cached vector values independently from the entry count", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => Array.from({ length: 40_000 }, () => 1)), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + await embedText("first", config) + await embedText("second", config) + await embedText("second", config) + expect(extractor).toHaveBeenCalledTimes(2) + + await embedText("first", config) + expect(extractor).toHaveBeenCalledTimes(3) + }) + + it("does not cache a vector larger than the total cache budget", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => Array.from({ length: 65_537 }, () => 1)), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + await embedText("oversized", config) + await embedText("oversized", config) + + expect(extractor).toHaveBeenCalledTimes(2) + }) + + it("retries failed inference and drops query vectors when the model is disposed", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + extractor.mockRejectedValueOnce(new Error("inference failed")) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + + await expect(embedText("same evidence", config)).rejects.toThrow("inference failed") + await expect(embedText("same evidence", config)).resolves.toEqual([1, 0]) + await embedText("same evidence", config) + expect(extractor).toHaveBeenCalledTimes(2) + await disposeTransformersCache() + await embedText("same evidence", config) + expect(extractor).toHaveBeenCalledTimes(3) + }) + + it("rejects an aborted cached query without running inference", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig() + await embedText("same evidence", config) + const controller = new AbortController() + controller.abort() + + await expect(embedText("same evidence", config, controller.signal)).rejects.toMatchObject({ + code: "ABORTED", + }) + expect(extractor).toHaveBeenCalledTimes(1) + }) + + it("refreshes search evidence after ingestion while reusing the query vector", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const modelConfig = await transformerArtifactConfig() + const root = path.dirname(modelConfig.embeddingModelPath) + const config = testConfig(root, { + embeddingProvider: "transformers", + embeddingModel: modelConfig.embeddingModel, + embeddingModelRevision: modelConfig.embeddingModelRevision, + embeddingModelPath: modelConfig.embeddingModelPath, + }) + await mkdir(config.rawDir, { recursive: true }) + const sourcePath = path.join(config.rawDir, "decision.md") + await writeFile(sourcePath, "Production approval requires the original evidence.\n") + const reader = await RagmirClient.createWithConfig(config) + let writer: RagmirClient | undefined + try { + await reader.ingest() + const first = await reader.search("production approval") + const warm = await reader.search("production approval") + expect(warm).toEqual(first) + + await writeFile(sourcePath, "Production approval now requires signed refreshed evidence.\n") + writer = await RagmirClient.createWithConfig(config) + await writer.ingest({ rebuild: true }) + await writer.close() + const inferenceCalls = extractor.mock.calls.length + const refreshed = await reader.search("production approval") + + expect(refreshed[0]?.text).toContain("signed refreshed evidence") + expect(refreshed[0]?.evidence.id).not.toBe(first[0]?.evidence.id) + expect(extractor).toHaveBeenCalledTimes(inferenceCalls) + } finally { + await writer?.close() + await reader.close() + } + }) + + it("bounds every semantic model call and preserves input order across batches", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map((text) => [Number(text.match(/\d+$/u)?.[0]), 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const config = transformerConfig({ embeddingBatchSize: 2 }) + + expect(await embedTexts(["row 0", "row 1", "row 2", "row 3", "row 4"], config)).toEqual([ + [0, 0], + [1, 0], + [2, 0], + [3, 0], + [4, 0], + ]) + expect(extractor.mock.calls.map(([texts]) => texts.length)).toEqual([2, 2, 1]) + }) + + it("enforces the hard batch ceiling even for a directly supplied config", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + + await embedTexts( + Array.from({ length: 129 }, () => "evidence"), + transformerConfig({ embeddingBatchSize: 1_000 }), + ) + expect(extractor.mock.calls.map(([texts]) => texts.length)).toEqual([128, 1]) + }) + + it("stops between semantic batches and releases the model after cancellation", async () => { + const controller = new AbortController() + const dispose = vi.fn(async () => undefined) + const extractor = vi.fn(async (texts: string[]) => { + controller.abort() + return { tolist: () => texts.map(() => [1, 0]) } + }) + transformersMock.pipeline.mockResolvedValue(Object.assign(extractor, { dispose })) + + await expect( + embedTexts( + ["first", "second", "third"], + transformerConfig({ embeddingBatchSize: 2 }), + "document", + controller.signal, + ), + ).rejects.toMatchObject({ code: "ABORTED" }) + expect(extractor).toHaveBeenCalledTimes(1) + await disposeTransformersCache() + expect(dispose).toHaveBeenCalledOnce() + }) + it("adds the asymmetric E5 retrieval prefixes", () => { const model = "intfloat/multilingual-e5-small" diff --git a/packages/ragmir-core/src/embeddings.ts b/packages/ragmir-core/src/embeddings.ts index b5774b9..f739858 100644 --- a/packages/ragmir-core/src/embeddings.ts +++ b/packages/ragmir-core/src/embeddings.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto" import { createReadStream } from "node:fs" import { mkdir, readdir, stat } from "node:fs/promises" import path from "node:path" +import { MAX_EMBEDDING_BATCH_SIZE } from "./defaults.js" import { throwIfAborted } from "./operation.js" import { tokenize } from "./text.js" import type { Config, IngestionEmbeddingModelState } from "./types.js" @@ -19,6 +20,8 @@ const MAX_LOCAL_HASH_FEATURE_CACHE_ENTRIES = 16_384 * pipeline is disposed as soon as its final lease is released. */ const MAX_TRANSFORMERS_PIPELINES = 1 +const MAX_QUERY_EMBEDDING_CACHE_ENTRIES = 128 +const MAX_QUERY_EMBEDDING_CACHE_VALUES = 65_536 const transformersPipelines = new Map() const transformersModelOwners = new Map() @@ -38,13 +41,20 @@ interface TransformersPipelineEntry { retired: boolean idleWaiters: Array<() => void> disposal: Promise | undefined + queryEmbeddings: QueryEmbeddingCache } interface TransformersPipelineLease { extractor: TransformersExtractor + queryEmbeddings: QueryEmbeddingCache release(): Promise } +interface QueryEmbeddingCache { + vectors: Map + values: number +} + interface LocalHashFeatureLocation { index: number sign: 1 | -1 @@ -77,7 +87,7 @@ export async function embedTexts( return texts.map((text) => localHashEmbedding(text, featureCache)) } - const embeddings = await embedWithTransformers(texts, config, inputType) + const embeddings = await embedWithTransformers(texts, config, inputType, signal) throwIfAborted(signal) return embeddings }) @@ -151,25 +161,74 @@ async function embedWithTransformers( texts: string[], config: Config, inputType: EmbeddingInputType, + signal?: AbortSignal, ): Promise { const lease = await acquireTransformersPipeline(config) try { - const preparedTexts = texts.map((text) => - prepareEmbeddingText(text, config.embeddingModel, inputType), - ) - const output = await lease.extractor(preparedTexts, { pooling: "mean", normalize: true }) - const rows = tensorToEmbeddingRows(output) - - if (rows.length !== texts.length) { - throw new Error(`Expected ${texts.length} embeddings, received ${rows.length}.`) + throwIfAborted(signal) + // Cache only single-query vectors. Search still reads and ranks the current index snapshot. + const cacheKey = + inputType === "query" && texts.length === 1 + ? createHash("sha256") + .update(JSON.stringify([config.projectRoot, texts[0]])) + .digest("hex") + : null + if (cacheKey !== null) { + const cached = cachedQueryEmbedding(lease.queryEmbeddings, cacheKey) + if (cached) return [cached] } - return rows + const embeddings: number[][] = [] + const batchSize = Math.max(1, Math.min(config.embeddingBatchSize, MAX_EMBEDDING_BATCH_SIZE)) + for (let offset = 0; offset < texts.length; offset += batchSize) { + throwIfAborted(signal) + const preparedTexts = texts + .slice(offset, offset + batchSize) + .map((text) => prepareEmbeddingText(text, config.embeddingModel, inputType)) + const output = await lease.extractor(preparedTexts, { pooling: "mean", normalize: true }) + throwIfAborted(signal) + const rows = tensorToEmbeddingRows(output) + if (rows.length !== preparedTexts.length) { + throw new Error(`Expected ${preparedTexts.length} embeddings, received ${rows.length}.`) + } + embeddings.push(...rows) + } + const queryEmbedding = embeddings[0] + if (cacheKey !== null && queryEmbedding) { + cacheQueryEmbedding(lease.queryEmbeddings, cacheKey, queryEmbedding) + } + return embeddings } finally { await lease.release() } } +function cachedQueryEmbedding(cache: QueryEmbeddingCache, key: string): number[] | undefined { + const vector = cache.vectors.get(key) + if (!vector) return undefined + cache.vectors.delete(key) + cache.vectors.set(key, vector) + return [...vector] +} + +function cacheQueryEmbedding(cache: QueryEmbeddingCache, key: string, vector: number[]): void { + if (vector.length > MAX_QUERY_EMBEDDING_CACHE_VALUES) return + const previous = cache.vectors.get(key) + cache.values -= previous?.length ?? 0 + cache.vectors.delete(key) + while ( + cache.vectors.size >= MAX_QUERY_EMBEDDING_CACHE_ENTRIES || + cache.values + vector.length > MAX_QUERY_EMBEDDING_CACHE_VALUES + ) { + const oldest = cache.vectors.entries().next().value + if (!oldest) break + cache.vectors.delete(oldest[0]) + cache.values -= oldest[1].length + } + cache.vectors.set(key, [...vector]) + cache.values += vector.length +} + export function prepareEmbeddingText( text: string, model: string, @@ -198,6 +257,7 @@ async function acquireTransformersPipeline(config: Config): Promise { @@ -214,6 +274,7 @@ async function acquireTransformersPipeline(config: Config): Promiseargparse: 2.0.1 yaml: ^2.9.1 packageExtensionsChecksum: sha256-uvpf4QPuEv+rnz0FdBdbR0iSoPTEXOVffanNl1HYoio= @@ -1933,9 +1934,6 @@ packages: arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} - argparse@1.0.10: - resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} - argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -2665,8 +2663,8 @@ packages: h3@1.15.11: resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} - handlebars@4.7.9: - resolution: {integrity: sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==} + handlebars@4.7.10: + resolution: {integrity: sha512-P5VJMVM7qgBn6vjXMw8WG9uVI+ncf2pi72j4de4yz5ZULLj2RGqLYaKOYGsgyrViQ0tePOVlN1tDCCXXtFqXKg==} engines: {node: '>=0.4.7'} hasBin: true @@ -2727,8 +2725,8 @@ packages: htmlparser2@10.1.0: resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==} - http-cache-semantics@4.2.0: - resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + http-cache-semantics@4.3.0: + resolution: {integrity: sha512-M5t5LlJpS1UHMjvwRQVdFHvPISGeLAxNcrWuJkeGh0KxsqCHZ1O3NXZU/8x7cD0BDcGW8kapxMKTvwlqrNkHkA==} http-errors@2.0.1: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} @@ -3906,9 +3904,6 @@ packages: split2@1.0.0: resolution: {integrity: sha512-NKywug4u4pX/AZBB1FCPzZ6/7O+Xhz1qMVbzTvvKvikjO99oPN87SkK08mEY9P63/5lWjK+wgOOgApnTg5r6qg==} - sprintf-js@1.0.3: - resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -6188,10 +6183,6 @@ snapshots: arg@5.0.2: {} - argparse@1.0.10: - dependencies: - sprintf-js: 1.0.3 - argparse@2.0.1: {} argue-cli@3.2.0: {} @@ -6241,7 +6232,7 @@ snapshots: get-tsconfig: 5.0.0-beta.4 github-slugger: 2.0.0 html-escaper: 3.0.3 - http-cache-semantics: 4.2.0 + http-cache-semantics: 4.3.0 js-yaml: 4.3.2 jsonc-parser: 3.3.1 magic-string: 1.4.2 @@ -6491,7 +6482,7 @@ snapshots: dependencies: '@simple-libs/stream-utils': 1.2.0 conventional-commits-filter: 5.0.0 - handlebars: 4.7.9 + handlebars: 4.7.10 meow: 13.2.0 semver: 7.8.5 @@ -7027,7 +7018,7 @@ snapshots: ufo: 1.6.4 uncrypto: 0.1.3 - handlebars@4.7.9: + handlebars@4.7.10: dependencies: minimist: 1.2.8 neo-async: 2.6.2 @@ -7101,7 +7092,7 @@ snapshots: domutils: 3.2.2 entities: 7.0.1 - http-cache-semantics@4.2.0: {} + http-cache-semantics@4.3.0: {} http-errors@2.0.1: dependencies: @@ -7429,7 +7420,7 @@ snapshots: mammoth@1.13.0: dependencies: '@xmldom/xmldom': 0.8.15 - argparse: 1.0.10 + argparse: 2.0.1 base64-js: 1.5.1 dingbat-to-unicode: 1.0.2 jszip: 3.10.2 @@ -8200,8 +8191,6 @@ snapshots: dependencies: through2: 2.0.5 - sprintf-js@1.0.3: {} - statuses@2.0.2: {} std-env@4.3.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 21074a2..11f8896 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,9 +8,6 @@ audit: # No patched release exists for these advisories. Re-check them on every dependency update and # remove each entry as soon as a fixed version is available. ignore: - # http-cache-semantics <=4.2.0: build-time dependency of the private static landing through - # Astro. No shared HTTP cache serves responses from it. - - GHSA-ch52-4w7c-c8xp # braces <=3.0.3: deeply nested patterns can exhaust the stack. Only semantic-release tooling and # fast-glob reach it, and fast-glob only receives project-owned source patterns from Ragmir # configuration, never MCP input or indexed content. @@ -18,6 +15,9 @@ audit: overrides: "adm-zip@<0.6.1": ^0.6.1 + # Mammoth only imports argparse from its unused CLI. Version 2 retains the legacy CLI aliases + # and removes the vulnerable sprintf-js dependency from the workspace toolchain. + "mammoth>argparse": 2.0.1 yaml: ^2.9.1 packageExtensions: