ci: enforce changed-file coverage

codecov.yml

@@ -17,7 +17,7 @@ coverage:
     patch:
       default:
         target: 90%
-        threshold: 1%
+        threshold: 0%
         if_ci_failed: error
 
 comment:

package.json

@@ -20,7 +20,7 @@
   "scripts": {
     "build": "turbo run build",
     "dev": "turbo run dev --parallel",
-    "test": "turbo run test",
+    "test": "node --test scripts/*.test.mjs && turbo run test",
     "update:all": "pnpm update --recursive --depth Infinity && pnpm sync:spec-artifacts",
     "coverage:changed": "node scripts/check-changed-coverage.mjs",
     "test:watch": "turbo run test:watch --parallel",

scripts/check-changed-coverage.mjs

@@ -18,6 +18,7 @@ const lcovFiles = [
   "packages/platform/coverage/lcov.info",
   "packages/service/coverage/lcov.info"
 ];
+const sourcePaths = ["packages"];
 
 if (!Number.isFinite(target) || target < 0 || target > 100) {
   throw new Error(
@@ -33,6 +34,7 @@ const changed = changedSourceLines(baseRef);
 let covered = 0;
 let executable = 0;
 const missing = [];
+const fileCoverageResults = [];
 
 for (const [file, lines] of [...changed.entries()].sort()) {
   const fileCoverage = coverage.get(file);
@@ -44,14 +46,23 @@ for (const [file, lines] of [...changed.entries()].sort()) {
     });
     continue;
   }
+  let fileCovered = 0;
+  let fileExecutable = 0;
   for (const line of [...lines].sort((left, right) => left - right)) {
     const hits = fileCoverage.get(line);
     if (hits === undefined) continue;
     executable += 1;
+    fileExecutable += 1;
     if (hits === 0) missing.push({ file, lines: [line], reason: "uncovered" });
     else if (isPartiallyCovered(branchCoverage.get(file)?.get(line)))
       missing.push({ file, lines: [line], reason: "partially covered" });
-    else covered += 1;
+    else {
+      covered += 1;
+      fileCovered += 1;
+    }
+  }
+  if (fileExecutable > 0) {
+    fileCoverageResults.push({ file, covered: fileCovered, executable: fileExecutable });
   }
 }
 
@@ -61,21 +72,45 @@ if (executable === 0 && missing.length === 0) {
 }
 
 const percentage = executable === 0 ? 0 : (covered / executable) * 100;
+const failingFiles = fileCoverageResults.filter(
+  ({ covered: fileCovered, executable: fileExecutable }) => {
+    return (fileCovered / fileExecutable) * 100 < target;
+  }
+);
 process.stdout.write(
   `Changed-line coverage: ${covered}/${executable} executable changed source lines (${percentage.toFixed(2)}%).\n`
 );
 
+if (failingFiles.length > 0) {
+  process.stdout.write("Changed files below target:\n");
+  for (const { file, covered: fileCovered, executable: fileExecutable } of failingFiles) {
+    const filePercentage = (fileCovered / fileExecutable) * 100;
+    process.stdout.write(
+      `- ${file}: ${fileCovered}/${fileExecutable} (${filePercentage.toFixed(2)}%)\n`
+    );
+  }
+}
+
 if (missing.length > 0) {
   process.stdout.write("Uncovered changed lines:\n");
   for (const item of collapseMissing(missing)) {
     process.stdout.write(`- ${item.file}:${item.lines} (${item.reason})\n`);
   }
 }
 
-if (missing.some((item) => item.reason === "no coverage data") || percentage < target) {
-  process.stderr.write(
-    `Changed-line coverage target not met: ${percentage.toFixed(2)}% < ${target.toFixed(2)}%.\n`
-  );
+if (
+  missing.some((item) => item.reason === "no coverage data") ||
+  percentage < target ||
+  failingFiles.length > 0
+) {
+  const failures = [];
+  if (percentage < target)
+    failures.push(`aggregate ${percentage.toFixed(2)}% < ${target.toFixed(2)}%`);
+  if (failingFiles.length > 0)
+    failures.push(`${String(failingFiles.length)} changed file(s) below target`);
+  if (missing.some((item) => item.reason === "no coverage data"))
+    failures.push("changed source missing coverage data");
+  process.stderr.write(`Changed-line coverage target not met: ${failures.join("; ")}.\n`);
   process.exit(1);
 }
 
@@ -122,16 +157,20 @@ function normalizeSourcePath(packageRoot, sourceFile) {
 }
 
 function changedSourceLines(ref) {
-  const diff = execFileSync("git", ["diff", "--unified=0", `${ref}...HEAD`, "--", "packages"], {
+  const mergeBase = execFileSync("git", ["merge-base", ref, "HEAD"], {
+    cwd: repoRoot,
+    encoding: "utf8"
+  }).trim();
+  const diff = execFileSync("git", ["diff", "--unified=0", mergeBase, "--", ...sourcePaths], {
     cwd: repoRoot,
     encoding: "utf8"
   });
   const changedLines = new Map();
   let currentFile;
   for (const line of diff.split("\n")) {
-    if (line.startsWith("+++ b/")) {
-      const file = line.slice("+++ b/".length);
-      currentFile = file.includes("/src/") && /\.[cm]?[jt]sx?$/.test(file) ? file : undefined;
+    if (line.startsWith("+++ ")) {
+      const file = line.startsWith("+++ b/") ? line.slice("+++ b/".length) : undefined;
+      currentFile = file !== undefined && isIncludedSource(file) ? file : undefined;
       continue;
     }
     if (currentFile === undefined || !line.startsWith("@@")) continue;
@@ -144,9 +183,27 @@ function changedSourceLines(ref) {
     for (let offset = 0; offset < count; offset += 1) lines.add(start + offset);
     changedLines.set(currentFile, lines);
   }
+  const untracked = execFileSync(
+    "git",
+    ["ls-files", "--others", "--exclude-standard", "--", ...sourcePaths],
+    {
+      cwd: repoRoot,
+      encoding: "utf8"
+    }
+  );
+  for (const file of untracked.split("\n").filter(isIncludedSource)) {
+    const contents = readFileSync(resolve(repoRoot, file), "utf8");
+    const count =
+      contents.length === 0 ? 0 : contents.split("\n").length - (contents.endsWith("\n") ? 1 : 0);
+    changedLines.set(file, new Set(Array.from({ length: count }, (_value, index) => index + 1)));
+  }
   return changedLines;
 }
 
+function isIncludedSource(file) {
+  return file.includes("/src/") && /\.[cm]?[jt]sx?$/.test(file);
+}
+
 function collapseMissing(items) {
   const groups = new Map();
   for (const item of items) {

scripts/check-changed-coverage.test.mjs

@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { execFileSync, spawnSync } from "node:child_process";
+import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const sourceScript = fileURLToPath(new URL("./check-changed-coverage.mjs", import.meta.url));
+
+async function createRepository(t) {
+  const root = await mkdtemp(join(tmpdir(), "changed-coverage-"));
+  t.after(() => rm(root, { recursive: true }));
+  await mkdir(join(root, "scripts"), { recursive: true });
+  await mkdir(join(root, "packages/core/src"), { recursive: true });
+  await mkdir(join(root, "packages/core/coverage"), { recursive: true });
+  await copyFile(sourceScript, join(root, "scripts/check-changed-coverage.mjs"));
+  await writeFile(join(root, ".gitignore"), "**/coverage/\n");
+  await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 1;\n");
+  git(root, "init");
+  git(root, "config", "user.email", "coverage@example.test");
+  git(root, "config", "user.name", "Coverage Test");
+  git(root, "add", ".");
+  git(root, "commit", "-m", "initial");
+  const checker = await readFile(sourceScript, "utf8");
+  const reports = [...checker.matchAll(/["']([^"']+\/coverage\/lcov\.info)["']/g)].map(
+    (match) => match[1]
+  );
+  for (const report of reports) {
+    await mkdir(dirname(join(root, report)), { recursive: true });
+    await writeFile(join(root, report), "");
+  }
+  return root;
+}
+
+function git(root, ...args) {
+  return execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
+}
+
+async function writeCoverage(root, file, hits, branches = [], append = false) {
+  const lines = [
+    `SF:src/${file}`,
+    ...hits.map((count, index) => `DA:${String(index + 1)},${String(count)}`)
+  ];
+  lines.push(...branches, "end_of_record", "");
+  await writeFile(
+    join(root, "packages/core/coverage/lcov.info"),
+    lines.join("\n"),
+    append ? { flag: "a" } : undefined
+  );
+}
+
+function runCheck(root, baseRef, target = "100") {
+  return spawnSync(process.execPath, ["scripts/check-changed-coverage.mjs"], {
+    cwd: root,
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      PATCH_COVERAGE_BASE: baseRef,
+      PATCH_COVERAGE_TARGET: target
+    }
+  });
+}
+
+function check(root, baseRef) {
+  const result = runCheck(root, baseRef);
+  assert.equal(result.status, 0, `${result.stdout}${result.stderr}`);
+  return result.stdout;
+}
+
+function checkFailure(root, baseRef, target = "100") {
+  const result = runCheck(root, baseRef, target);
+  assert.notEqual(result.status, 0);
+  return `${result.stdout}${result.stderr}`;
+}
+
+for (const state of ["committed", "staged", "unstaged"]) {
+  test(`measures ${state} source changes`, async (t) => {
+    const root = await createRepository(t);
+    const baseRef = git(root, "rev-parse", "HEAD");
+    await writeFile(
+      join(root, "packages/core/src/existing.ts"),
+      "export const value = 2;\nexport const added = 3;\n"
+    );
+    if (state !== "unstaged") git(root, "add", "packages/core/src/existing.ts");
+    if (state === "committed") git(root, "commit", "-m", "change source");
+    await writeCoverage(root, "existing.ts", [1, 1]);
+
+    assert.match(
+      check(root, baseRef),
+      /Changed-line coverage: 2\/2 executable changed source lines \(100\.00%\)\./
+    );
+  });
+}
+
+test("measures untracked source files", async (t) => {
+  const root = await createRepository(t);
+  const baseRef = git(root, "rev-parse", "HEAD");
+  await writeFile(
+    join(root, "packages/core/src/new.ts"),
+    "export const first = 1;\nexport const second = 2;\n"
+  );
+  await writeCoverage(root, "new.ts", [1, 1]);
+
+  assert.match(
+    check(root, baseRef),
+    /Changed-line coverage: 2\/2 executable changed source lines \(100\.00%\)\./
+  );
+});
+
+test("rejects a partially covered changed line", async (t) => {
+  const root = await createRepository(t);
+  const baseRef = git(root, "rev-parse", "HEAD");
+  await writeFile(
+    join(root, "packages/core/src/existing.ts"),
+    "export const value = true ? 1 : 2;\n"
+  );
+  await writeCoverage(root, "existing.ts", [1], ["BRDA:1,0,0,1", "BRDA:1,0,1,0"]);
+
+  const output = checkFailure(root, baseRef);
+  assert.match(output, /Changed-line coverage: 0\/1 executable changed source lines \(0\.00%\)\./);
+  assert.match(output, /packages\/core\/src\/existing\.ts:1 \(partially covered\)/);
+});
+
+test("rejects an uncovered changed executable line", async (t) => {
+  const root = await createRepository(t);
+  const baseRef = git(root, "rev-parse", "HEAD");
+  await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n");
+  await writeCoverage(root, "existing.ts", [0]);
+
+  const output = checkFailure(root, baseRef);
+  assert.match(output, /packages\/core\/src\/existing\.ts:1 \(uncovered\)/);
+});
+
+test("rejects a changed source file missing from coverage data", async (t) => {
+  const root = await createRepository(t);
+  const baseRef = git(root, "rev-parse", "HEAD");
+  await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n");
+  await writeCoverage(root, "other.ts", [1]);
+
+  const output = checkFailure(root, baseRef);
+  assert.match(output, /packages\/core\/src\/existing\.ts:1 \(no coverage data\)/);
+});
+
+test("rejects a changed file below target when aggregate coverage meets target", async (t) => {
+  const root = await createRepository(t);
+  const baseRef = git(root, "rev-parse", "HEAD");
+  await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n");
+  await writeFile(
+    join(root, "packages/core/src/new.ts"),
+    Array.from(
+      { length: 9 },
+      (_value, index) => `export const value${String(index)} = ${String(index)};`
+    ).join("\n")
+  );
+  await writeCoverage(root, "existing.ts", [0]);
+  await writeCoverage(
+    root,
+    "new.ts",
+    Array.from({ length: 9 }, () => 1),
+    [],
+    true
+  );
+
+  const output = checkFailure(root, baseRef, "90");
+  assert.match(
+    output,
+    /Changed-line coverage: 9\/10 executable changed source lines \(90\.00%\)\./
+  );
+  assert.match(output, /packages\/core\/src\/existing\.ts: 0\/1 \(0\.00%\)/);
+});