fix(ci): make benchmark symlink corpora reproducible

.github/workflows/external-eval.yml

@@ -1,6 +1,12 @@
 name: external-eval
 
 on:
+  pull_request:
+    paths:
+      - "benchmarks/**"
+      - "scripts/evaluate-baseline.mjs"
+      - "scripts/lib/**"
+      - ".github/workflows/external-eval.yml"
   workflow_dispatch:
   schedule:
     - cron: "17 5 * * 1"
@@ -21,6 +27,7 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npm run build:core
+      - run: node --test scripts/lib/benchmark-corpus.test.mjs
       - run: node scripts/evaluate-external.mjs --gate --check-recorded
       - run: node scripts/evaluate-external.mjs --suite heldout --gate --check-recorded
       - run: node scripts/evaluate-baseline.mjs --suite external --check-recorded

benchmarks/external/README.md

@@ -44,6 +44,16 @@ npm run evaluate:external:record            # deliberately refresh results.json
 node scripts/evaluate-baseline.mjs --suite external --check-recorded
 ```
 
+Baseline arms share one scanned corpus. Committed symlink aliases whose canonical
+target is already scanned are removed consistently, including when Windows Git
+checks out a link as a target-name text file. This avoids a host-dependent duplicate
+document changing BM25 corpus statistics. No ranking weights or expected paths are
+changed by this normalization.
+
+Use `--case colinhacks/zod` for a focused diagnostic run. Filtered runs cannot record
+or validate a full-suite snapshot. Benchmark changes also run the external and
+held-out snapshot gates in pull requests, before the weekly scheduled check.
+
 The first run shallow-clones each repository at its pinned SHA into the OS temp directory (network required); later runs reuse the clones. Because of the network dependency this is not part of `npm run ci`; the [`external-eval` workflow](../../.github/workflows/external-eval.yml) runs it on a weekly schedule and on manual dispatch. Scheduled and release runs use `--check-recorded`, so a ranking change must deliberately refresh and review [`results.json`](results.json).
 
 ## Results

benchmarks/external/baseline-results.json

@@ -1318,7 +1318,7 @@
             "packages/zod/README.md",
             "packages/resolution/src/index.mts",
             "packages/resolution/src/index.cts",
-            "packages/resolution/src/index.ts"
+            "wiki/optionality.md"
           ],
           "top1Hit": false,
           "top3Hit": false,

scripts/evaluate-baseline.mjs

@@ -33,6 +33,7 @@ import { readFile, writeFile } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { dirname, join, resolve } from "node:path";
 import { materializePinnedRepository } from "./lib/external-cache.mjs";
+import { normalizePinnedAliases } from "./lib/benchmark-corpus.mjs";
 import { classifyExpectedPathMention, splitCohorts } from "./lib/expected-path-mention.mjs";
 import { wilsonInterval } from "./lib/wilson.mjs";
 
@@ -49,7 +50,18 @@ if (!["external", "heldout"].includes(suite)) {
 }
 
 const suiteDir = join(repoRoot, "benchmarks", suite);
-const dataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8"));
+const loadedDataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8"));
+const caseIndex = process.argv.indexOf("--case");
+const caseSlug = caseIndex === -1 ? undefined : process.argv[caseIndex + 1];
+if (caseIndex !== -1 && (!caseSlug || !loadedDataset.cases.some((entry) => entry.slug === caseSlug))) {
+  throw new Error("--case must name a case in the selected suite.");
+}
+if (caseSlug && (process.argv.includes("--record") || process.argv.includes("--check-recorded"))) {
+  throw new Error("Filtered runs cannot record or check the full suite snapshot.");
+}
+const dataset = caseSlug
+  ? { ...loadedDataset, cases: loadedDataset.cases.filter((entry) => entry.slug === caseSlug) }
+  : loadedDataset;
 const recordedResultsPath = join(suiteDir, "baseline-results.json");
 
 const TOP_N = 5;
@@ -246,7 +258,7 @@ for (const benchmark of dataset.cases) {
   const dir = await materializePinnedRepository(benchmark);
   // One scan, shared by every arm, so the comparison isolates ranking and candidate policy
   // rather than what was read off disk.
-  const repo = await scanRepo({ repoRoot: dir });
+  const repo = normalizePinnedAliases(await scanRepo({ repoRoot: dir }));
   if (repo.files.length === 0) {
     throw new Error(`Baseline evaluation could not scan any files for ${benchmark.slug} at ${benchmark.sha}.`);
   }

scripts/lib/benchmark-corpus.mjs

@@ -0,0 +1,38 @@
+import { posix } from "node:path";
+import { execFileSync } from "node:child_process";
+
+function readGit(args, cwd) {
+  // Large pinned repositories exceed Node's default 1 MiB process-output limit.
+  // Preserve raw target bytes, including whitespace in valid symlink names.
+  return execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
+}
+
+// Windows Git may materialize a symlink as its target-name text. Linux's scanner
+// deduplicates the real link against its target. Give every benchmark arm that
+// same corpus, using committed link identities rather than host link support.
+export function normalizePinnedAliases(repo, git = readGit) {
+  const aliases = new Map();
+  for (const entry of git(["ls-tree", "-r", "-z", "HEAD"], repo.root).split("\0")) {
+    const match = /^120000 blob ([0-9a-f]+)\t([\s\S]+)$/.exec(entry);
+    if (!match) continue;
+    const target = git(["cat-file", "blob", match[1]], repo.root);
+    if (posix.isAbsolute(target) || target.includes("\\")) continue;
+    const resolved = posix.normalize(posix.join(posix.dirname(match[2]), target));
+    if (resolved === ".." || resolved.startsWith("../")) continue;
+    aliases.set(match[2], resolved);
+  }
+  const paths = new Set(repo.files.map((file) => file.path));
+  return {
+    ...repo,
+    files: repo.files.filter((file) => {
+      const seen = new Set([file.path]);
+      let target = aliases.get(file.path);
+      while (target !== undefined && !seen.has(target)) {
+        seen.add(target);
+        if (!aliases.has(target)) return !paths.has(target);
+        target = aliases.get(target);
+      }
+      return true;
+    })
+  };
+}

scripts/lib/benchmark-corpus.test.mjs

@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { execFileSync } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { normalizePinnedAliases } from "./benchmark-corpus.mjs";
+
+function fixture(paths, links) {
+  const repo = { root: "/repo", files: paths.map((path) => ({ path })), diagnostics: [] };
+  const targets = Object.values(links);
+  const git = (args) => args[0] === "ls-tree"
+    ? Object.keys(links).map((path, index) => `120000 blob ${index.toString(16)}\t${path}\0`).join("")
+    : targets[parseInt(args[2], 16)];
+  return { repo, git };
+}
+
+test("real links and Windows link placeholders produce the same shared corpus", () => {
+  const links = { "README.md": "packages/zod/README.md" };
+  const windows = fixture(["README.md", "packages/zod/README.md", "src/index.ts"], links);
+  const linux = fixture(["packages/zod/README.md", "src/index.ts"], links);
+  assert.deepEqual(normalizePinnedAliases(windows.repo, windows.git), normalizePinnedAliases(linux.repo, linux.git));
+  assert.equal(windows.repo.files.length, 3);
+});
+
+test("relative chains deduplicate only against an available canonical target", () => {
+  const { repo, git } = fixture(["docs/a.md", "docs/b.md", "real.md", "missing.md", "outside.md", "cycle.md"], {
+    "docs/a.md": "b.md", "docs/b.md": "../real.md", "missing.md": "absent.md",
+    "outside.md": "../outside.md", "cycle.md": "cycle.md"
+  });
+  assert.deepEqual(normalizePinnedAliases(repo, git).files.map((file) => file.path),
+    ["real.md", "missing.md", "outside.md", "cycle.md"]);
+});
+
+test("reads a committed tree larger than the default child-process buffer", async () => {
+  const root = await mkdtemp(join(tmpdir(), "fixmap-benchmark-corpus-"));
+  const git = (args, input) => execFileSync("git", args, { cwd: root, encoding: "utf8", input }).trim();
+  try {
+    git(["init", "--quiet"]);
+    const blob = git(["hash-object", "-w", "--stdin"], "source\n");
+    const link = git(["hash-object", "-w", "--stdin"], "target.md");
+    const entries = Array.from({ length: 16_000 }, (_, index) => `100644 ${blob}\tfiles/long-source-file-name-${index}.ts\n`);
+    entries.push(`100644 ${blob}\ttarget.md\n120000 ${link}\talias.md\n`);
+    git(["update-index", "--index-info"], entries.join(""));
+    const tree = git(["write-tree"]);
+    const commit = git(["-c", "user.name=FixMap Test", "-c", "user.email=test@example.invalid", "commit-tree", tree, "-m", "fixture"]);
+    git(["update-ref", "HEAD", commit]);
+    const repo = { root, files: [{ path: "alias.md" }, { path: "target.md" }] };
+    assert.deepEqual(normalizePinnedAliases(repo).files, [{ path: "target.md" }]);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});