Improve memory grounding, retrieval, and evaluation fidelity

ts/package.json

@@ -86,7 +86,8 @@
     "test:live": "pnpm -r ---no-bail  -no-sort --stream --workspace-concurrency=1 run test:live",
     "test:local": "pnpm run test:local:packages && pnpm run test:local:scripts",
     "test:local:packages": "node tools/scripts/runLocalTestPackages.mjs",
-    "test:local:scripts": "node --test tools/scripts/test/*.spec.mjs",
+    "test:local:scripts": "node --test tools/scripts/test/*.spec.mjs && pnpm run test:memory-quality",
+    "test:memory-quality": "node --test tools/memory-maintenance-qualification/quality-*.test.mjs tools/memory-maintenance-qualification/live-resume.test.mjs tools/memory-maintenance-qualification/longmemeval-*.test.mjs tools/memory-maintenance-qualification/ingestion-profile.test.mjs",
     "test:ui": "pnpm -r --no-bail --no-sort --stream --if-present run test:ui"
   },
   "devDependencies": {

ts/packages/agents/browser/src/views/client/library/memoryHubImports.test.ts

@@ -138,16 +138,22 @@ test("Memory Hub extracts a selected local PDF and submits only ordinary Markdow
         configurable: true,
     });
     const memory = jest.mocked(invokeMemory);
+    let submitted!: () => void;
+    const submission = new Promise<void>((resolve) => {
+        submitted = resolve;
+    });
     memory.mockImplementation(async (method) => {
         if (method === "memoryListCorpora")
             return [{ corpusId: "selected", name: "Documents" }] as never;
-        if (method === "memoryImportDocument")
+        if (method === "memoryImportDocument") {
+            submitted();
             return {
                 jobId: "local-job",
                 sourceId: "local-source",
                 revisionId: "local-revision",
                 state: "accepted",
             } as never;
+        }
         throw new Error(`Unexpected memory method ${method}`);
     });
     mockGetDocument.mockReturnValue({
@@ -181,7 +187,7 @@ test("Memory Hub extracts a selected local PDF and submits only ordinary Markdow
     dialog
         .querySelector("form")!
         .dispatchEvent(new Event("submit", { cancelable: true }));
-    await new Promise((resolve) => setTimeout(resolve, 30));
+    await submission;
     await settle();
     expect(memory).toHaveBeenCalledWith("memoryImportDocument", {
         corpusId: "selected",

ts/packages/knowPro/README.md

@@ -19,6 +19,34 @@ The KnowPro implementation of Structured RAG is used to explore types of [**memo
 - [Natural language querying](./src/searchLang.ts)
 - [Answer generation](./src/answerGenerator.ts)
 
+### Temporal source headers
+
+Retained text can supply source dates through single-line headers such as
+`Session date: 2024-05-10`. Leading spaces and tabs after the colon are allowed.
+Each recognized header owns the text through the next header or the end of the
+source. The date must be on the header's own line; an empty header does not
+borrow a date from the following line.
+
+### Embedding batches
+
+Default conversation settings batch up to 128 distinct vocabulary terms,
+source-text chunks, or thread descriptions. Message rebuilds group up to 128
+messages before batching their chunks; an explicit outer batch override still
+applies. Thread rebuilds batch descriptions, while incremental thread additions
+embed the single new description immediately.
+
+Provider request-size limits still apply; the generation helper splits requests
+for providers advertising a smaller maximum. Azure/OpenAI index batches also
+split at 300,000 aggregate tokens, counted with `cl100k_base`, without truncating
+source text. The provider's per-input token limit still applies.
+
+Term embeddings are reused within the same persisted vocabulary index, including
+incremental document additions and reloads. Repeated occurrences retain their
+separate evidence references without requiring a vector for every occurrence.
+This is not a global cache across independent corpora, and rebuilding an index
+can recompute its vocabulary. Term and message-text embedding lanes still run
+sequentially.
+
 ### Examples
 
 Examples of using the above APIs can be found here:

ts/packages/knowPro/package.json

@@ -39,6 +39,7 @@
     "async": "^3.2.5",
     "debug": "^4.4.0",
     "fast-levenshtein": "^3.0.0",
+    "gpt-tokenizer": "^2.9.0",
     "typechat": "^0.1.1"
   },
   "devDependencies": {

ts/packages/knowPro/src/answerContext.ts

@@ -1,13 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import * as kpLib from "@typeagent/knowledge-processor";
 import {
     AnswerContext,
     RelevantKnowledge,
     RelevantMessage,
 } from "./answerContextSchema.js";
 import { jsonStringifyForPrompt } from "./common.js";
+import type { AnswerResponse } from "./answerResponseSchema.js";
 
 export function answerContextToString(
     context: AnswerContext,
@@ -46,7 +46,13 @@ export class AnswerContextChunkBuilder {
     constructor(
         public context: AnswerContext,
         public maxCharsPerChunk: number,
-    ) {}
+    ) {
+        if (!Number.isSafeInteger(maxCharsPerChunk) || maxCharsPerChunk < 4) {
+            throw new RangeError(
+                "Answer context budget must be an integer of at least four characters",
+            );
+        }
+    }
 
     public *getChunks(
         includeKnowledge: boolean = true,
@@ -106,44 +112,94 @@ export class AnswerContextChunkBuilder {
                 if (!message.messageText) {
                     continue;
                 }
-                const messageChunks = kpLib.splitLargeTextIntoChunks(
-                    message.messageText,
-                    this.maxCharsPerChunk,
-                );
-                for (const msgChunk of messageChunks) {
-                    const chunkMessage: RelevantMessage = {
-                        ...message,
-                        messageText: msgChunk,
-                    };
-                    const completedChunk = this.addToCurrentChunk(
-                        chunkMessage,
-                        "messages",
-                    );
-                    if (completedChunk) {
-                        yield completedChunk;
+                const texts =
+                    typeof message.messageText === "string"
+                        ? [message.messageText]
+                        : message.messageText;
+                for (const text of texts) {
+                    for (const chunkMessage of this.splitMessage(
+                        message,
+                        text,
+                    )) {
+                        const completedChunk = this.addToCurrentChunk(
+                            chunkMessage,
+                            "messages",
+                        );
+                        if (completedChunk) {
+                            yield completedChunk;
+                        }
                     }
                 }
             }
         }
     }
 
     private addToCurrentChunk(
-        item: any,
+        item: RelevantKnowledge | RelevantMessage,
         type: keyof AnswerContext,
     ): AnswerContext | undefined {
-        const itemString = jsonStringifyForPrompt(item);
-        const itemSize = itemString.length;
-        if (this.currentChunkCharCount + itemSize > this.maxCharsPerChunk) {
-            const completedChunk = this.currentChunk;
-            this.newChunk();
-            return completedChunk;
+        const singleton = { [type]: [item] };
+        if (answerContextToString(singleton).length > this.maxCharsPerChunk) {
+            throw new RangeError(
+                `Answer context ${type} item exceeds the serialized character budget`,
+            );
         }
-        if (this.currentChunk[type] === undefined) {
-            this.currentChunk[type] = [];
+        const candidate = {
+            ...this.currentChunk,
+            [type]: [...(this.currentChunk[type] ?? []), item],
+        };
+        let completedChunk: AnswerContext | undefined;
+        if (answerContextToString(candidate).length > this.maxCharsPerChunk) {
+            completedChunk = this.currentChunk;
+            this.currentChunk = singleton;
+        } else {
+            this.currentChunk = candidate;
+        }
+        this.currentChunkCharCount = answerContextToString(
+            this.currentChunk,
+        ).length;
+        return completedChunk;
+    }
+
+    private *splitMessage(
+        message: RelevantMessage,
+        text: string,
+    ): IterableIterator<RelevantMessage> {
+        let offset = 0;
+        while (offset < text.length) {
+            let low = 0;
+            let high = text.length - offset;
+            while (low < high) {
+                const length = Math.ceil((low + high) / 2);
+                const part = {
+                    ...message,
+                    messageText: text.slice(offset, offset + length),
+                };
+                if (
+                    answerContextToString({ messages: [part] }).length <=
+                    this.maxCharsPerChunk
+                ) {
+                    low = length;
+                } else {
+                    high = length - 1;
+                }
+            }
+            const last = text.charCodeAt(offset + low - 1);
+            if (
+                offset + low < text.length &&
+                last >= 0xd800 &&
+                last <= 0xdbff
+            ) {
+                low--;
+            }
+            if (low === 0) {
+                throw new RangeError(
+                    "Answer context message metadata or text cannot fit the serialized character budget",
+                );
+            }
+            yield { ...message, messageText: text.slice(offset, offset + low) };
+            offset += low;
         }
-        this.currentChunk[type]!.push(item);
-        this.currentChunkCharCount += itemSize;
-        return undefined;
     }
 
     private newChunk() {
@@ -153,6 +209,9 @@ export class AnswerContextChunkBuilder {
 }
 
 export type AnswerContextOptions = {
+    onCombination?:
+        | ((inputs: AnswerResponse[], output: AnswerResponse) => void)
+        | undefined;
     entitiesTopK?: number | undefined;
     topicsTopK?: number | undefined;
     messagesTopK?: number | undefined;

ts/packages/knowPro/src/answerContextSchema.ts

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 
 import { DateRange } from "./interfaces.js";
+import { MessageTemporalContext } from "./temporalSchema.js";
+import { Knowledge } from "./interfaces.js";
 
 export type AnswerContext = {
     // Relevant entities
@@ -16,6 +18,14 @@ export type AnswerContext = {
 export type EntityNames = string | string[];
 
 export type RelevantKnowledge = {
+    // Original, unmerged statements preserve per-fact attribution and conflicting values.
+    evidence?: {
+        semanticRefOrdinal: number;
+        sourceMessageOrdinal: number;
+        knowledge?: Knowledge;
+        temporalContext?: MessageTemporalContext | undefined;
+    }[];
+    sourceMessageOrdinals?: number[] | undefined;
     // The actual knowledge
     knowledge: any;
     // Entity or entities who mentioned the knowledge
@@ -27,6 +37,8 @@ export type RelevantKnowledge = {
 };
 
 export type RelevantMessage = {
+    temporalContext?: MessageTemporalContext | undefined;
+    sourceMessageOrdinals?: number[] | undefined;
     from?: EntityNames | undefined;
     to?: EntityNames | undefined;
     timestamp?: Date | undefined;