feat(elos): on-device OCR of scanned PDFs in the MCP server (macOS Vision)

docs/integrations/HERMES.md

@@ -60,6 +60,20 @@ N’y mets pas de documents d’entreprise internes, de données personnelles d
 
 Le serveur est lancé depuis ce dépôt : garde une branche qui contient `mcp/` (par exemple `main-public`).
 
+## PDF scannés (OCR sur le Mac)
+
+Sur macOS, le serveur lit aussi les PDF scannés grâce à l’OCR intégré (framework Vision, via `mcp/ocr-vision.swift`,
+compilé une fois avec `swiftc`) : aucun téléchargement, aucun envoi. Le texte reconnu est mis en cache dans
+`ELOS_CACHE_DIR` (par défaut `../.elos-cache` à côté du dossier de cours, jamais dans le dossier partagé) ; les
+citations issues d’une page OCR portent la mention « verify against the page ». L’OCR se fait en arrière-plan
+(la première question n’attend pas) ; pour tout préparer d’avance :
+
+```bash
+ELOS_SOURCES_DIR=~/ELOS-Hermes/cours npm run mcp:ocr
+```
+
+`ELOS_OCR=off` désactive l’OCR.
+
 ## Coach du matin (tâche planifiée Hermes, sans modèle)
 
 `scripts/morning-coach.ts` construit chaque matin un message à partir du résumé : série, révisions dues, question de

mcp/elos-server.ts

@@ -7,14 +7,17 @@
 // knowledge-private/, secret files and paths outside the root are refused.
 //
 // Run: node --experimental-strip-types --import ./tests/register-loader.mjs mcp/elos-server.ts
-import { readdirSync, readFileSync, statSync } from "node:fs";
+import { execFile, execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { promisify } from "node:util";
 import path from "node:path";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { answerFromSources, deriveFlashcards, deriveQuiz, revisionSheet, socraticPrompt } from "../src/modules/learning-studio/core.ts";
 import { SYNTHETIC_DOCUMENTS } from "../src/modules/learning-studio/fixtures.ts";
-import { buildImportedDocument, textToSections, type RawPage } from "../src/modules/learning-studio/import.ts";
+import { buildImportedDocument, pagesNeedingOcr, textToSections, type RawPage } from "../src/modules/learning-studio/import.ts";
 import { buildAudioOverview } from "../src/modules/learning-studio/overview.ts";
 import { derivePrerequisites, remediationPath } from "../src/modules/learning-studio/prerequisites.ts";
 import type { StudioDocument } from "../src/modules/learning-studio/types.ts";
@@ -41,7 +44,7 @@ export function allowedPath(root: string, candidate: string): string {
   return resolved;
 }
 
-function listFiles(root: string): string[] {
+export function listFiles(root: string): string[] {
   const out: string[] = [];
   const walk = (dir: string) => {
     for (const name of readdirSync(dir).sort()) {
@@ -75,15 +78,85 @@ async function pdfPages(file: string): Promise<RawPage[]> {
   }
 }
 
+// --- On-device OCR for scanned PDFs (macOS Vision via mcp/ocr-vision.swift) -------------------
+// Results are cached outside the shared folder (ELOS_CACHE_DIR, default <sources>/../.elos-cache)
+// and computed in the background so a first question never waits for a long OCR run.
+
+export function cacheDir(root: string, env: NodeJS.ProcessEnv = process.env): string {
+  return path.resolve(env.ELOS_CACHE_DIR?.trim() || path.join(path.dirname(root), ".elos-cache"));
+}
+
+let ocrBinary: string | null | undefined;
+function ensureOcrBinary(cache: string, env: NodeJS.ProcessEnv): string | null {
+  if (ocrBinary !== undefined) return ocrBinary;
+  ocrBinary = null;
+  if (process.platform !== "darwin" || env.ELOS_OCR === "off") return null;
+  const source = path.join(path.dirname(new URL(import.meta.url).pathname), "ocr-vision.swift");
+  const hash = createHash("sha256").update(readFileSync(source)).digest("hex").slice(0, 12);
+  const binary = path.join(cache, `elos-ocr-${hash}`);
+  try {
+    if (!existsSync(binary)) { mkdirSync(cache, { recursive: true, mode: 0o700 }); execFileSync("swiftc", ["-O", source, "-o", binary], { stdio: "ignore", timeout: 300_000 }); }
+    ocrBinary = binary;
+  } catch { ocrBinary = null; }
+  return ocrBinary;
+}
+
+const run = promisify(execFile);
+const ocrQueue: string[] = [];
+let ocrRunning = false;
+
+function ocrCacheFile(cache: string, file: string): string {
+  return path.join(cache, "ocr", `${createHash("sha256").update(readFileSync(file)).digest("hex")}.json`);
+}
+
+export function cachedOcr(cache: string, file: string): RawPage[] | null {
+  try { return (JSON.parse(readFileSync(ocrCacheFile(cache, file), "utf8")) as { pages: RawPage[] }).pages; } catch { return null; }
+}
+
+/** OCRs one PDF now (used by the background queue and by `npm run mcp:ocr`). */
+export async function ocrNow(cache: string, file: string, env: NodeJS.ProcessEnv = process.env): Promise<RawPage[] | null> {
+  const existing = cachedOcr(cache, file);
+  if (existing) return existing;
+  const binary = ensureOcrBinary(cache, env);
+  if (!binary) return null;
+  const { stdout } = await run(binary, [file], { maxBuffer: 64 * 1024 * 1024, timeout: 600_000 });
+  const pages = (JSON.parse(stdout) as { pages: RawPage[] }).pages;
+  mkdirSync(path.join(cache, "ocr"), { recursive: true, mode: 0o700 });
+  writeFileSync(ocrCacheFile(cache, file), JSON.stringify({ file: path.basename(file), pages }), { mode: 0o600 });
+  return pages;
+}
+
+function scheduleOcr(cache: string, file: string, env: NodeJS.ProcessEnv): void {
+  if (ocrQueue.includes(file)) return;
+  ocrQueue.push(file);
+  if (ocrRunning) return;
+  ocrRunning = true;
+  void (async () => {
+    while (ocrQueue.length) {
+      const next = ocrQueue.shift() as string;
+      try { await ocrNow(cache, next, env); extracted.delete(next); } catch { /* unreadable: stays text-only */ }
+    }
+    ocrRunning = false;
+  })();
+}
+
+/** Merges OCR text into the pages that have no text layer; flags them as OCR. */
+function withOcr(pages: RawPage[], ocr: RawPage[] | null): RawPage[] {
+  if (!ocr) return pages;
+  const missing = new Set(pagesNeedingOcr(pages));
+  return pages.map((page) => (missing.has(page.pageNumber) ? { pageNumber: page.pageNumber, text: ocr.find((item) => item.pageNumber === page.pageNumber)?.text ?? "", ocr: true } : page));
+}
+
 /** Extraction cache: a file is parsed again only when its size or modification time changes. */
 const extracted = new Map<string, { key: string; document: StudioDocument | null }>();
 
 /**
  * Loads the learner's course files (or the synthetic demo documents when no root is configured).
  * Folder layout: <subject>/<origin or year>/... — the first folder becomes the subject.
  */
-export async function loadSources(root: string | null): Promise<StudioDocument[]> {
+export async function loadSources(root: string | null, env: NodeJS.ProcessEnv = process.env): Promise<StudioDocument[]> {
   if (!root) return [...SYNTHETIC_DOCUMENTS];
+  const cache = cacheDir(root, env);
   const documents: StudioDocument[] = [];
   const seen = new Set<string>();
   for (const file of listFiles(root)) {
@@ -98,7 +171,12 @@ export async function loadSources(root: string | null): Promise<StudioDocument[]
     let document: StudioDocument | null = null;
     try {
       const pdf = /\.pdf$/i.test(file);
-      const pages = pdf ? await pdfPages(file) : textToSections(readFileSync(file, "utf8"));
+      let pages = pdf ? await pdfPages(file) : textToSections(readFileSync(file, "utf8"));
+      if (pdf && pagesNeedingOcr(pages).length > 0) {
+        const ocr = cachedOcr(cache, file);
+        if (ocr) pages = withOcr(pages, ocr);
+        else scheduleOcr(cache, file, env);
+      }
       document = buildImportedDocument(`file:${relative}`, metadata, pages, pdf ? "page" : "section", new Date(info.mtimeMs));
     } catch {
       document = null; // unreadable or scanned file: skipped (no OCR in the agent server, nothing invented)
@@ -113,7 +191,8 @@ export async function loadSources(root: string | null): Promise<StudioDocument[]
 const json = (value: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(value, null, 2) }] });
 const cite = (documents: readonly StudioDocument[], citation: { documentId: string; pageNumber: number; quote: string }) => {
   const document = documents.find((item) => item.id === citation.documentId);
-  return { source: document?.title ?? citation.documentId, [document?.unit === "section" ? "section" : "page"]: citation.pageNumber, quote: citation.quote };
+  const ocr = document?.pages.find((page) => page.pageNumber === citation.pageNumber)?.ocr === true;
+  return { source: document?.title ?? citation.documentId, [document?.unit === "section" ? "section" : "page"]: citation.pageNumber, quote: citation.quote, ...(ocr ? { ocr: "text recognised from a scan: verify against the page" } : {}) };
 };
 
 const STALE_MS = 24 * 3_600_000;
@@ -141,7 +220,7 @@ export function createServer(env: NodeJS.ProcessEnv = process.env): McpServer {
     description: `Answer a question ONLY with sentences quoted from the learner's course sources, with page/section citations; says so when the sources do not contain the answer. mode "socratic" returns a guiding question and graduated hints instead of the answer. ${sourceNote} ${privacy}`,
     inputSchema: { question: z.string().min(2), mode: z.enum(["sourced", "socratic"]).default("sourced") },
   }, async ({ question, mode }) => {
-    const documents = await loadSources(root);
+    const documents = await loadSources(root, env);
     if (mode === "socratic") {
       const prompt = socraticPrompt(documents, question);
       return json(prompt ? { question: prompt.question, hints: prompt.hints.map((hint) => hint.text), citations: prompt.citations.map((c) => cite(documents, c)) }
@@ -156,7 +235,7 @@ export function createServer(env: NodeJS.ProcessEnv = process.env): McpServer {
     description: `Derive study material from the course sources: flashcards, quiz, revision sheet, audio overview script or prerequisite paths. Every item is cited and DERIVED (never counts as mastery). ${sourceNote} ${privacy}`,
     inputSchema: { kind: z.enum(["flashcards", "quiz", "sheet", "audio_overview", "prerequisites"]), limit: z.number().int().min(1).max(50).default(10) },
   }, async ({ kind, limit }) => {
-    const documents = await loadSources(root);
+    const documents = await loadSources(root, env);
     const cards = deriveFlashcards(documents);
     if (kind === "flashcards") return json(cards.slice(0, limit).map((card) => ({ front: card.term, back: card.definition, citation: cite(documents, card.citations[0]) })));
     if (kind === "quiz") return json(deriveQuiz(documents, limit).map((q) => ({ prompt: q.prompt, choices: q.choices, answer: q.answer, citation: cite(documents, q.citations[0]) })));

mcp/ocr-vision.swift

@@ -0,0 +1,35 @@
+// On-device OCR for scanned PDFs (macOS Vision + PDFKit). Prints JSON:
+// {"pages":[{"pageNumber":1,"text":"..."}]}. No network; used by mcp/elos-server.ts.
+import Foundation
+import PDFKit
+import Vision
+
+guard CommandLine.arguments.count == 2, let document = PDFDocument(url: URL(fileURLWithPath: CommandLine.arguments[1])) else {
+  FileHandle.standardError.write("usage: elos-ocr <file.pdf>\n".data(using: .utf8)!)
+  exit(2)
+}
+
+var pages: [[String: Any]] = []
+for index in 0..<document.pageCount {
+  guard let page = document.page(at: index) else { continue }
+  let bounds = page.bounds(for: .mediaBox)
+  let scale: CGFloat = 2.0
+  let width = Int(bounds.width * scale), height = Int(bounds.height * scale)
+  guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0,
+                                space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { continue }
+  context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1))
+  context.fill(CGRect(x: 0, y: 0, width: width, height: height))
+  context.scaleBy(x: scale, y: scale)
+  page.draw(with: .mediaBox, to: context)
+  guard let image = context.makeImage() else { continue }
+
+  let request = VNRecognizeTextRequest()
+  request.recognitionLevel = .accurate
+  request.recognitionLanguages = ["fr-FR", "en-US"]
+  request.usesLanguageCorrection = true
+  try? VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
+  let lines = (request.results ?? []).compactMap { $0.topCandidates(1).first?.string }
+  pages.append(["pageNumber": index + 1, "text": lines.joined(separator: "\n")])
+}
+let data = try JSONSerialization.data(withJSONObject: ["pages": pages])
+FileHandle.standardOutput.write(data)

mcp/prepare-ocr.ts

@@ -0,0 +1,30 @@
+// Pre-computes on-device OCR (macOS Vision) for every scanned PDF under ELOS_SOURCES_DIR,
+// so the ELOS MCP server never has to wait for it. Results go to the cache directory,
+// outside the shared folder. Usage: ELOS_SOURCES_DIR=... npm run mcp:ocr
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
+import { cacheDir, cachedOcr, listFiles, ocrNow, sourcesRoot } from "./elos-server.ts";
+
+const root = sourcesRoot();
+if (!root) { console.error("ELOS_SOURCES_DIR is not set."); process.exit(1); }
+const cache = cacheDir(root);
+let done = 0, skipped = 0, pages = 0;
+for (const file of listFiles(root).filter((item) => /\.pdf$/i.test(item))) {
+  const task = getDocument({ data: new Uint8Array(readFileSync(file)), useSystemFonts: true });
+  let scanned = false;
+  try {
+    const pdf = await task.promise;
+    let chars = 0;
+    for (let i = 1; i <= pdf.numPages && chars < 20; i += 1) chars += (await (await pdf.getPage(i)).getTextContent()).items.map((item) => ("str" in item ? item.str : "")).join("").trim().length;
+    scanned = chars < 20;
+  } catch { scanned = false; } finally { await task.destroy(); }
+  if (!scanned) continue;
+  if (cachedOcr(cache, file)) { skipped += 1; continue; }
+  const started = Date.now();
+  const result = await ocrNow(cache, file);
+  if (!result) { console.error("OCR unavailable (macOS + swiftc required)."); process.exit(1); }
+  done += 1; pages += result.length;
+  console.log(`OCR ${path.relative(root, file)} · ${result.length} p. · ${((Date.now() - started) / 1000).toFixed(1)} s`);
+}
+console.log(`\n${done} PDF scanné(s) traité(s) (${pages} pages), ${skipped} déjà en cache · cache : ${cache}`);

package.json

@@ -25,6 +25,7 @@
     "start": "cd .next/standalone && NODE_ENV=production node server.js 2>&1 | tee ../../server.log",
     "lint": "eslint .",
     "mcp:elos": "node --experimental-strip-types --no-warnings --import ./tests/register-loader.mjs mcp/elos-server.ts",
+    "mcp:ocr": "node --experimental-strip-types --no-warnings --import ./tests/register-loader.mjs mcp/prepare-ocr.ts",
     "typecheck": "tsc --noEmit",
     "test": "node --test --experimental-strip-types --import ./tests/register-loader.mjs \"tests/unit/**/*.test.ts\" \"tests/integration/**/*.test.ts\" \"tests/integration/**/*.test.cjs\"",
     "test:ci": "node --test --test-concurrency=2 --experimental-strip-types --import ./tests/register-loader.mjs \"tests/unit/**/*.test.ts\" \"tests/integration/**/*.test.ts\" \"tests/integration/**/*.test.cjs\"",

tests/unit/mcp/ocr-cache.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { cacheDir, loadSources } from "../../../mcp/elos-server.ts";
+
+/** Smallest valid PDF with one blank page and no text layer (like a scan). */
+function blankPdf(): Buffer {
+  const objects = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] >>"];
+  let body = "%PDF-1.4\n";
+  const offsets: number[] = [];
+  objects.forEach((object, index) => { offsets.push(body.length); body += `${index + 1} 0 obj\n${object}\nendobj\n`; });
+  const xref = body.length;
+  body += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${offsets.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join("")}`;
+  body += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
+  return Buffer.from(body, "latin1");
+}
+
+test("cached OCR text fills scanned pages, is flagged, and lives outside the shared folder", async () => {
+  const base = mkdtempSync(path.join(tmpdir(), "elos-ocr-"));
+  const root = path.join(base, "cours");
+  mkdirSync(path.join(root, "Reseaux"), { recursive: true });
+  const file = path.join(root, "Reseaux", "scan.pdf");
+  const bytes = blankPdf();
+  writeFileSync(file, bytes);
+  const env: NodeJS.ProcessEnv = { NODE_ENV: "test", ELOS_OCR: "off" }; // never call Vision in tests
+
+  assert.equal((await loadSources(root, env)).length, 0, "a scan without OCR yields nothing (nothing invented)");
+
+  const cache = cacheDir(root, env);
+  assert.equal(cache, path.join(base, ".elos-cache"));
+  mkdirSync(path.join(cache, "ocr"), { recursive: true });
+  const digest = createHash("sha256").update(bytes).digest("hex");
+  writeFileSync(path.join(cache, "ocr", `${digest}.json`), JSON.stringify({ pages: [{ pageNumber: 1, text: "Un VLAN est un réseau local virtuel qui isole des machines." }] }));
+  writeFileSync(file, bytes); // touch: new mtime invalidates the in-memory extraction cache
+
+  const [document] = await loadSources(root, env);
+  assert.equal(document.subjectLabel, "Reseaux");
+  assert.equal(document.pages[0].ocr, true);
+  assert.match(document.pages[0].passages[0].text, /VLAN est un réseau local virtuel/);
+  assert.deepEqual(readdirSync(root, { recursive: true }).sort(), ["Reseaux", path.join("Reseaux", "scan.pdf")], "nothing written in the shared folder");
+});