From 263e32ddf2035585b010599fc8b481ab83a3b3df Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 8 Sep 2026 17:49:33 +0530 Subject: [PATCH 1/2] fix(ci): normalize benchmark symlink aliases across platforms --- .github/workflows/external-eval.yml | 7 +++++ benchmarks/external/README.md | 10 +++++++ benchmarks/external/baseline-results.json | 2 +- scripts/evaluate-baseline.mjs | 16 ++++++++++-- scripts/lib/benchmark-corpus.mjs | 32 +++++++++++++++++++++++ scripts/lib/benchmark-corpus.test.mjs | 29 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 scripts/lib/benchmark-corpus.mjs create mode 100644 scripts/lib/benchmark-corpus.test.mjs diff --git a/.github/workflows/external-eval.yml b/.github/workflows/external-eval.yml index ba2bd7f..8137d74 100644 --- a/.github/workflows/external-eval.yml +++ b/.github/workflows/external-eval.yml @@ -1,6 +1,12 @@ name: external-eval on: + pull_request: + paths: + - "benchmarks/**" + - "scripts/evaluate-baseline.mjs" + - "scripts/lib/**" + - ".github/workflows/external-eval.yml" workflow_dispatch: schedule: - cron: "17 5 * * 1" @@ -21,6 +27,7 @@ jobs: cache: npm - run: npm ci - run: npm run build:core + - run: node --test scripts/lib/benchmark-corpus.test.mjs - run: node scripts/evaluate-external.mjs --gate --check-recorded - run: node scripts/evaluate-external.mjs --suite heldout --gate --check-recorded - run: node scripts/evaluate-baseline.mjs --suite external --check-recorded diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md index 7ef4fa4..c5b291a 100644 --- a/benchmarks/external/README.md +++ b/benchmarks/external/README.md @@ -44,6 +44,16 @@ npm run evaluate:external:record # deliberately refresh results.json node scripts/evaluate-baseline.mjs --suite external --check-recorded ``` +Baseline arms share one scanned corpus. Committed symlink aliases whose canonical +target is already scanned are removed consistently, including when Windows Git +checks out a link as a target-name text file. This avoids a host-dependent duplicate +document changing BM25 corpus statistics. No ranking weights or expected paths are +changed by this normalization. + +Use `--case colinhacks/zod` for a focused diagnostic run. Filtered runs cannot record +or validate a full-suite snapshot. Benchmark changes also run the external and +held-out snapshot gates in pull requests, before the weekly scheduled check. + The first run shallow-clones each repository at its pinned SHA into the OS temp directory (network required); later runs reuse the clones. Because of the network dependency this is not part of `npm run ci`; the [`external-eval` workflow](../../.github/workflows/external-eval.yml) runs it on a weekly schedule and on manual dispatch. Scheduled and release runs use `--check-recorded`, so a ranking change must deliberately refresh and review [`results.json`](results.json). ## Results diff --git a/benchmarks/external/baseline-results.json b/benchmarks/external/baseline-results.json index 8f48123..cd5d3c1 100644 --- a/benchmarks/external/baseline-results.json +++ b/benchmarks/external/baseline-results.json @@ -1318,7 +1318,7 @@ "packages/zod/README.md", "packages/resolution/src/index.mts", "packages/resolution/src/index.cts", - "packages/resolution/src/index.ts" + "wiki/optionality.md" ], "top1Hit": false, "top3Hit": false, diff --git a/scripts/evaluate-baseline.mjs b/scripts/evaluate-baseline.mjs index a3f39e1..580ae7a 100644 --- a/scripts/evaluate-baseline.mjs +++ b/scripts/evaluate-baseline.mjs @@ -33,6 +33,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { fileURLToPath, pathToFileURL } from "node:url"; import { dirname, join, resolve } from "node:path"; import { materializePinnedRepository } from "./lib/external-cache.mjs"; +import { normalizePinnedAliases } from "./lib/benchmark-corpus.mjs"; import { classifyExpectedPathMention, splitCohorts } from "./lib/expected-path-mention.mjs"; import { wilsonInterval } from "./lib/wilson.mjs"; @@ -49,7 +50,18 @@ if (!["external", "heldout"].includes(suite)) { } const suiteDir = join(repoRoot, "benchmarks", suite); -const dataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8")); +const loadedDataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8")); +const caseIndex = process.argv.indexOf("--case"); +const caseSlug = caseIndex === -1 ? undefined : process.argv[caseIndex + 1]; +if (caseIndex !== -1 && (!caseSlug || !loadedDataset.cases.some((entry) => entry.slug === caseSlug))) { + throw new Error("--case must name a case in the selected suite."); +} +if (caseSlug && (process.argv.includes("--record") || process.argv.includes("--check-recorded"))) { + throw new Error("Filtered runs cannot record or check the full suite snapshot."); +} +const dataset = caseSlug + ? { ...loadedDataset, cases: loadedDataset.cases.filter((entry) => entry.slug === caseSlug) } + : loadedDataset; const recordedResultsPath = join(suiteDir, "baseline-results.json"); const TOP_N = 5; @@ -246,7 +258,7 @@ for (const benchmark of dataset.cases) { const dir = await materializePinnedRepository(benchmark); // One scan, shared by every arm, so the comparison isolates ranking and candidate policy // rather than what was read off disk. - const repo = await scanRepo({ repoRoot: dir }); + const repo = normalizePinnedAliases(await scanRepo({ repoRoot: dir })); if (repo.files.length === 0) { throw new Error(`Baseline evaluation could not scan any files for ${benchmark.slug} at ${benchmark.sha}.`); } diff --git a/scripts/lib/benchmark-corpus.mjs b/scripts/lib/benchmark-corpus.mjs new file mode 100644 index 0000000..2f8b595 --- /dev/null +++ b/scripts/lib/benchmark-corpus.mjs @@ -0,0 +1,32 @@ +import { posix } from "node:path"; +import { runGit } from "./external-cache.mjs"; + +// Windows Git may materialize a symlink as its target-name text. Linux's scanner +// deduplicates the real link against its target. Give every benchmark arm that +// same corpus, using committed link identities rather than host link support. +export function normalizePinnedAliases(repo, git = runGit) { + const aliases = new Map(); + for (const entry of git(["ls-tree", "-r", "-z", "HEAD"], repo.root).split("\0")) { + const match = /^120000 blob ([0-9a-f]+)\t([\s\S]+)$/.exec(entry); + if (!match) continue; + const target = git(["cat-file", "blob", match[1]], repo.root); + if (posix.isAbsolute(target) || target.includes("\\")) continue; + const resolved = posix.normalize(posix.join(posix.dirname(match[2]), target)); + if (resolved === ".." || resolved.startsWith("../")) continue; + aliases.set(match[2], resolved); + } + const paths = new Set(repo.files.map((file) => file.path)); + return { + ...repo, + files: repo.files.filter((file) => { + const seen = new Set([file.path]); + let target = aliases.get(file.path); + while (target !== undefined && !seen.has(target)) { + seen.add(target); + if (!aliases.has(target)) return !paths.has(target); + target = aliases.get(target); + } + return true; + }) + }; +} diff --git a/scripts/lib/benchmark-corpus.test.mjs b/scripts/lib/benchmark-corpus.test.mjs new file mode 100644 index 0000000..c795096 --- /dev/null +++ b/scripts/lib/benchmark-corpus.test.mjs @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { normalizePinnedAliases } from "./benchmark-corpus.mjs"; + +function fixture(paths, links) { + const repo = { root: "/repo", files: paths.map((path) => ({ path })), diagnostics: [] }; + const targets = Object.values(links); + const git = (args) => args[0] === "ls-tree" + ? Object.keys(links).map((path, index) => `120000 blob ${index.toString(16)}\t${path}\0`).join("") + : targets[parseInt(args[2], 16)]; + return { repo, git }; +} + +test("real links and Windows link placeholders produce the same shared corpus", () => { + const links = { "README.md": "packages/zod/README.md" }; + const windows = fixture(["README.md", "packages/zod/README.md", "src/index.ts"], links); + const linux = fixture(["packages/zod/README.md", "src/index.ts"], links); + assert.deepEqual(normalizePinnedAliases(windows.repo, windows.git), normalizePinnedAliases(linux.repo, linux.git)); + assert.equal(windows.repo.files.length, 3); +}); + +test("relative chains deduplicate only against an available canonical target", () => { + const { repo, git } = fixture(["docs/a.md", "docs/b.md", "real.md", "missing.md", "outside.md", "cycle.md"], { + "docs/a.md": "b.md", "docs/b.md": "../real.md", "missing.md": "absent.md", + "outside.md": "../outside.md", "cycle.md": "cycle.md" + }); + assert.deepEqual(normalizePinnedAliases(repo, git).files.map((file) => file.path), + ["real.md", "missing.md", "outside.md", "cycle.md"]); +}); From f7c9c7a7fdd6cc4c653520b368f05d66185f3cbb Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 8 Sep 2026 17:57:02 +0530 Subject: [PATCH 2/2] fix(ci): bound large benchmark tree reads explicitly --- scripts/lib/benchmark-corpus.mjs | 10 ++++++++-- scripts/lib/benchmark-corpus.test.mjs | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/lib/benchmark-corpus.mjs b/scripts/lib/benchmark-corpus.mjs index 2f8b595..00c2382 100644 --- a/scripts/lib/benchmark-corpus.mjs +++ b/scripts/lib/benchmark-corpus.mjs @@ -1,10 +1,16 @@ import { posix } from "node:path"; -import { runGit } from "./external-cache.mjs"; +import { execFileSync } from "node:child_process"; + +function readGit(args, cwd) { + // Large pinned repositories exceed Node's default 1 MiB process-output limit. + // Preserve raw target bytes, including whitespace in valid symlink names. + return execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }); +} // Windows Git may materialize a symlink as its target-name text. Linux's scanner // deduplicates the real link against its target. Give every benchmark arm that // same corpus, using committed link identities rather than host link support. -export function normalizePinnedAliases(repo, git = runGit) { +export function normalizePinnedAliases(repo, git = readGit) { const aliases = new Map(); for (const entry of git(["ls-tree", "-r", "-z", "HEAD"], repo.root).split("\0")) { const match = /^120000 blob ([0-9a-f]+)\t([\s\S]+)$/.exec(entry); diff --git a/scripts/lib/benchmark-corpus.test.mjs b/scripts/lib/benchmark-corpus.test.mjs index c795096..58995e0 100644 --- a/scripts/lib/benchmark-corpus.test.mjs +++ b/scripts/lib/benchmark-corpus.test.mjs @@ -1,5 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { normalizePinnedAliases } from "./benchmark-corpus.mjs"; function fixture(paths, links) { @@ -27,3 +31,23 @@ test("relative chains deduplicate only against an available canonical target", ( assert.deepEqual(normalizePinnedAliases(repo, git).files.map((file) => file.path), ["real.md", "missing.md", "outside.md", "cycle.md"]); }); + +test("reads a committed tree larger than the default child-process buffer", async () => { + const root = await mkdtemp(join(tmpdir(), "fixmap-benchmark-corpus-")); + const git = (args, input) => execFileSync("git", args, { cwd: root, encoding: "utf8", input }).trim(); + try { + git(["init", "--quiet"]); + const blob = git(["hash-object", "-w", "--stdin"], "source\n"); + const link = git(["hash-object", "-w", "--stdin"], "target.md"); + const entries = Array.from({ length: 16_000 }, (_, index) => `100644 ${blob}\tfiles/long-source-file-name-${index}.ts\n`); + entries.push(`100644 ${blob}\ttarget.md\n120000 ${link}\talias.md\n`); + git(["update-index", "--index-info"], entries.join("")); + const tree = git(["write-tree"]); + const commit = git(["-c", "user.name=FixMap Test", "-c", "user.email=test@example.invalid", "commit-tree", tree, "-m", "fixture"]); + git(["update-ref", "HEAD", commit]); + const repo = { root, files: [{ path: "alias.md" }, { path: "target.md" }] }; + assert.deepEqual(normalizePinnedAliases(repo).files, [{ path: "target.md" }]); + } finally { + await rm(root, { recursive: true, force: true }); + } +});