fix(elos): better grounded answers on real course slides; faster MCP sources

mcp/elos-server.ts

@@ -24,7 +24,7 @@ import { VOCABULARY, VOCABULARY_PACKS } from "../src/modules/daily-english/vocab
 import { replaySchedulers, type ReviewEvent } from "../src/modules/scientific-pedagogy/replay.ts";
 import { parseLearnerSnapshot, type LearnerSnapshot } from "../src/modules/learner-snapshot/core.ts";
 
-const MAX_FILES = 200;
+const MAX_FILES = 1000;
 const MAX_FILE_BYTES = 25 * 1024 * 1024;
 const FORBIDDEN = /(^|[\\/])(knowledge-private|\.git|node_modules)([\\/]|$)|\.(secret|private|env)$|(^|[\\/])\.env/i;
 
@@ -75,21 +75,38 @@ async function pdfPages(file: string): Promise<RawPage[]> {
   }
 }
 
-/** Loads the learner's course files (or the synthetic demo documents when no root is configured). */
+/** Extraction cache: a file is parsed again only when its size or modification time changes. */
+const extracted = new Map<string, { key: string; document: StudioDocument | null }>();
+
+/**
+ * Loads the learner's course files (or the synthetic demo documents when no root is configured).
+ * Folder layout: <subject>/<origin or year>/... — the first folder becomes the subject.
+ */
 export async function loadSources(root: string | null): Promise<StudioDocument[]> {
   if (!root) return [...SYNTHETIC_DOCUMENTS];
   const documents: StudioDocument[] = [];
+  const seen = new Set<string>();
   for (const file of listFiles(root)) {
+    seen.add(file);
+    const info = statSync(file);
+    const key = `${info.size}:${info.mtimeMs}`;
+    const cached = extracted.get(file);
+    if (cached?.key === key) { if (cached.document) documents.push(cached.document); continue; }
     const relative = path.relative(root, file);
-    const metadata = { title: relative, subjectLabel: path.dirname(relative) === "." ? "Mes cours" : path.dirname(relative), chapter: "", docType: "COURS" as const };
+    const [subject, origin] = relative.split(path.sep);
+    const metadata = { title: relative, subjectLabel: relative.includes(path.sep) ? subject : "Mes cours", chapter: relative.split(path.sep).length > 2 ? origin : "", docType: "COURS" as const };
+    let document: StudioDocument | null = null;
     try {
       const pdf = /\.pdf$/i.test(file);
       const pages = pdf ? await pdfPages(file) : textToSections(readFileSync(file, "utf8"));
-      documents.push(buildImportedDocument(`file:${relative}`, metadata, pages, pdf ? "page" : "section", new Date(statSync(file).mtimeMs)));
+      document = buildImportedDocument(`file:${relative}`, metadata, pages, pdf ? "page" : "section", new Date(info.mtimeMs));
     } catch {
-      // Unreadable or scanned file: skipped (no OCR in the agent server, nothing invented).
+      document = null; // unreadable or scanned file: skipped (no OCR in the agent server, nothing invented)
     }
+    extracted.set(file, { key, document });
+    if (document) documents.push(document);
   }
+  for (const file of extracted.keys()) if (!seen.has(file)) extracted.delete(file);
   return documents;
 }
 

src/modules/learning-studio/core.ts

@@ -107,11 +107,45 @@ function isAnswerable(hit: PassageHit | undefined, query: string): hit is Passag
   return !!hit && queryCoverage(hit, query) >= MIN_QUERY_COVERAGE;
 }
 
+// Real course material is mostly slides: short titles, repeated pages and quiz questions
+// match a question's words without answering it. Answers prefer explanatory passages.
+const DEFINITION_CUE = /\b(?:est|sont|désigne|désignent|sert à|servent à|permet|permettent|référence|correspond|consiste|garantit|assure|identifie)\b|\s:\s/i;
+
+function isQuestionLike(text: string): boolean {
+  return /\?\s*$/.test(text.trim()) || /\bquestion\s*\d*\s*:/i.test(text) || /^\s*[-•]?\s*[a-d]\)\s/i.test(text);
+}
+
+function answerQuality(text: string): number {
+  let quality = 1;
+  if (text.split(/\s+/).length < 8) quality *= 0.35;
+  if (isQuestionLike(text)) quality *= 0.25;
+  if (DEFINITION_CUE.test(text)) quality *= 1.4;
+  // Oral transcripts (timestamps, spoken markers) rank after written material, without being excluded.
+  const spoken = (text.match(/\b(?:je|moi|on va|alors|euh|voilà|vous voyez|d'accord)\b/gi) ?? []).length;
+  if (/\[\d{1,2}:\d{2}/.test(text) || spoken >= 2) quality *= 0.4;
+  return quality;
+}
+
+/** Passages ranked to answer a question: BM25, then duplicates removed and answer quality applied. */
+export function rankAnswerPassages(documents: readonly StudioDocument[], question: string, limit = 4): PassageHit[] {
+  const seen = new Set<string>();
+  return searchPassages(documents, question, 25)
+    .filter((hit) => {
+      const key = normalizeText(hit.passage.text).replace(/[^a-z0-9]+/g, "");
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    })
+    .map((hit) => ({ ...hit, score: hit.score * answerQuality(hit.passage.text) }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit);
+}
+
 function bestSentences(hit: PassageHit, query: string, max = 2): string[] {
   const terms = new Set(tokenize(query));
   return splitSentences(hit.passage.text)
     .map((sentence, index) => ({ sentence, index, overlap: tokenize(sentence).filter((token) => terms.has(token)).length }))
-    .filter((item) => item.overlap > 0)
+    .filter((item) => item.overlap > 0 && !isQuestionLike(item.sentence) && item.sentence.split(/\s+/).length >= 4)
     .sort((a, b) => b.overlap - a.overlap || a.index - b.index)
     .slice(0, max)
     .sort((a, b) => a.index - b.index)
@@ -125,12 +159,19 @@ export type GroundedAnswer = AskCourseAnswer & Readonly<{ studioCitations: reado
  * relevant is found it says so instead of inventing an answer.
  */
 export function answerFromSources(documents: readonly StudioDocument[], question: string): GroundedAnswer {
-  const hits = searchPassages(documents, question, 4);
+  const hits = rankAnswerPassages(documents, question, 4);
   const top = hits[0]?.score ?? 0;
   const relevant = isAnswerable(hits[0], question)
     ? hits.filter((hit) => hit.score >= top * 0.6 && queryCoverage(hit, question) >= MIN_QUERY_COVERAGE / 2).slice(0, 2)
     : [];
-  const quoted = relevant.flatMap((hit) => bestSentences(hit, question).map((sentence) => ({ hit, sentence })));
+  const said = new Set<string>();
+  const quoted = relevant.flatMap((hit) => bestSentences(hit, question).map((sentence) => ({ hit, sentence })))
+    .filter(({ sentence }) => {
+      const key = normalizeText(sentence).replace(/[^a-z0-9]+/g, "");
+      if (said.has(key)) return false;
+      said.add(key);
+      return true;
+    });
   if (quoted.length === 0) {
     return { answer: "Je ne trouve pas cette information dans les sources sélectionnées.", citations: [], studioCitations: [],
       uncertainty: "Aucun passage suffisamment pertinent : reformulez ou sélectionnez d’autres documents.", claims: [] };
@@ -147,7 +188,7 @@ export function answerFromSources(documents: readonly StudioDocument[], question
 
 /** Socratic mode: a guiding question first, then graduated hints, the answer last. */
 export function socraticPrompt(documents: readonly StudioDocument[], question: string): SocraticPrompt | null {
-  const hit = searchPassages(documents, question, 1)[0];
+  const hit = rankAnswerPassages(documents, question, 1)[0];
   if (!isAnswerable(hit, question)) return null;
   const sentence = bestSentences(hit, question, 1)[0] ?? splitSentences(hit.passage.text)[0];
   const inPassage = deriveFlashcards([hit.document]).filter((item) => item.citations[0].passageId === hit.passage.id && item.citations[0].pageNumber === hit.page.pageNumber);

src/modules/learning-studio/import.ts

@@ -57,8 +57,12 @@ export function titleFromFileName(fileName: string): string {
  * extracts as separate fragments ("modi fi er", "fi chier") and normalises whitespace,
  * keeping paragraph breaks.
  */
+const SPACING_ACCENTS: Readonly<Record<string, string>> = { "\u00B4": "\u0301", "\u0060": "\u0300", "\u02C6": "\u0302", "\u00A8": "\u0308", "\u0301": "\u0301", "\u0300": "\u0300", "\u0302": "\u0302", "\u0308": "\u0308" };
+
 export function cleanExtractedText(text: string): string {
   return text
+    // LaTeX-made PDFs extract accents as separate marks before the letter ("exprim ́ ee" → "exprimée").
+    .replace(/ ?([\u00B4\u0060\u02C6\u00A8\u0300-\u0302\u0308]) ?([aeiouAEIOU])/g, (_match, mark: string, letter: string) => (letter + SPACING_ACCENTS[mark]).normalize("NFC"))
     .normalize("NFKC")
     .replace(/\r\n?/g, "\n")
     // One space on the left means the fragment is inside a word; several mean a word boundary.

tests/unit/learning-studio/import-and-exam.test.ts

@@ -96,3 +96,9 @@ test("mock exam grading reports score, calibration and overconfidence, never mas
   assert.equal(result.attempt.mode, "MOCK_EXAM");
   assert.equal(gradeMockExam(exam, {}, {}, NOW, new Date(NOW.getTime() + 10 * 60_000)).timedOut, true);
 });
+
+test("LaTeX spacing accents are recomposed", () => {
+  assert.equal(cleanExtractedText("La condition de jointure est exprim ́ ee dans la clause WHERE."), "La condition de jointure est exprimée dans la clause WHERE.");
+  assert.equal(cleanExtractedText("donn ́ ees, syst ` eme, contr ˆ ole"), "données, système, contrôle");
+  assert.equal(cleanExtractedText("café déjà vu"), "café déjà vu");
+});

tests/unit/learning-studio/learning-studio.test.ts

@@ -158,3 +158,20 @@ test("studio state survives reloads, ignores canonical-looking entries and corru
 test("tokenizer ignores accents, case and French stop words", () => {
   assert.deepEqual(tokenize("À quoi sert le Masque de sous-réseau ?"), ["masque", "sous", "reseau"]);
 });
+
+test("answers on slide-like sources prefer explanations over titles, quiz questions, transcripts and duplicates", async () => {
+  const { buildImportedDocument } = await import("../../../src/modules/learning-studio/import.ts");
+  const { answerFromSources: answer, rankAnswerPassages } = await import("../../../src/modules/learning-studio/core.ts");
+  const meta = { title: "Slides", subjectLabel: "Réseaux", chapter: "", docType: "COURS" as const };
+  const slides = buildImportedDocument("slides", meta, [
+    { pageNumber: 1, text: "Protocole ARP : même LAN" },
+    { pageNumber: 2, text: "Question 5 : Quelle méthode est utilisée par le protocole ARP ?" },
+    { pageNumber: 3, text: "[12:04.1 - 12:09.0] Alors moi je vous dis, le protocole ARP on va le voir." },
+    { pageNumber: 4, text: "Le protocole ARP permet de trouver l'adresse MAC qui correspond à une adresse IP sur le réseau local." },
+    { pageNumber: 5, text: "Le protocole ARP permet de trouver l'adresse MAC qui correspond à une adresse IP sur le réseau local." },
+  ], "page", new Date(2026, 9, 1));
+  assert.equal(rankAnswerPassages([slides], "Qu'est-ce que le protocole ARP ?", 1)[0].page.pageNumber, 4);
+  const result = answer([slides], "Qu'est-ce que le protocole ARP ?");
+  assert.equal(result.answer, "Le protocole ARP permet de trouver l'adresse MAC qui correspond à une adresse IP sur le réseau local.");
+  assert.equal(result.citations.length, 1);
+});