diff --git a/codecov.yml b/codecov.yml index e970bf1..4e982a9 100644 --- a/codecov.yml +++ b/codecov.yml @@ -17,7 +17,7 @@ coverage: patch: default: target: 90% - threshold: 1% + threshold: 0% if_ci_failed: error comment: diff --git a/package.json b/package.json index 35dadb1..305e8bd 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev --parallel", - "test": "turbo run test", + "test": "node --test scripts/*.test.mjs && turbo run test", "update:all": "pnpm update --recursive --depth Infinity && pnpm sync:spec-artifacts", "coverage:changed": "node scripts/check-changed-coverage.mjs", "test:watch": "turbo run test:watch --parallel", diff --git a/scripts/check-changed-coverage.mjs b/scripts/check-changed-coverage.mjs index 1d9c286..3a78d1f 100644 --- a/scripts/check-changed-coverage.mjs +++ b/scripts/check-changed-coverage.mjs @@ -18,6 +18,7 @@ const lcovFiles = [ "packages/platform/coverage/lcov.info", "packages/service/coverage/lcov.info" ]; +const sourcePaths = ["packages"]; if (!Number.isFinite(target) || target < 0 || target > 100) { throw new Error( @@ -33,6 +34,7 @@ const changed = changedSourceLines(baseRef); let covered = 0; let executable = 0; const missing = []; +const fileCoverageResults = []; for (const [file, lines] of [...changed.entries()].sort()) { const fileCoverage = coverage.get(file); @@ -44,14 +46,23 @@ for (const [file, lines] of [...changed.entries()].sort()) { }); continue; } + let fileCovered = 0; + let fileExecutable = 0; for (const line of [...lines].sort((left, right) => left - right)) { const hits = fileCoverage.get(line); if (hits === undefined) continue; executable += 1; + fileExecutable += 1; if (hits === 0) missing.push({ file, lines: [line], reason: "uncovered" }); else if (isPartiallyCovered(branchCoverage.get(file)?.get(line))) missing.push({ file, lines: [line], reason: "partially covered" }); - else covered += 1; + else { + covered += 1; + fileCovered += 1; + } + } + if (fileExecutable > 0) { + fileCoverageResults.push({ file, covered: fileCovered, executable: fileExecutable }); } } @@ -61,10 +72,25 @@ if (executable === 0 && missing.length === 0) { } const percentage = executable === 0 ? 0 : (covered / executable) * 100; +const failingFiles = fileCoverageResults.filter( + ({ covered: fileCovered, executable: fileExecutable }) => { + return (fileCovered / fileExecutable) * 100 < target; + } +); process.stdout.write( `Changed-line coverage: ${covered}/${executable} executable changed source lines (${percentage.toFixed(2)}%).\n` ); +if (failingFiles.length > 0) { + process.stdout.write("Changed files below target:\n"); + for (const { file, covered: fileCovered, executable: fileExecutable } of failingFiles) { + const filePercentage = (fileCovered / fileExecutable) * 100; + process.stdout.write( + `- ${file}: ${fileCovered}/${fileExecutable} (${filePercentage.toFixed(2)}%)\n` + ); + } +} + if (missing.length > 0) { process.stdout.write("Uncovered changed lines:\n"); for (const item of collapseMissing(missing)) { @@ -72,10 +98,19 @@ if (missing.length > 0) { } } -if (missing.some((item) => item.reason === "no coverage data") || percentage < target) { - process.stderr.write( - `Changed-line coverage target not met: ${percentage.toFixed(2)}% < ${target.toFixed(2)}%.\n` - ); +if ( + missing.some((item) => item.reason === "no coverage data") || + percentage < target || + failingFiles.length > 0 +) { + const failures = []; + if (percentage < target) + failures.push(`aggregate ${percentage.toFixed(2)}% < ${target.toFixed(2)}%`); + if (failingFiles.length > 0) + failures.push(`${String(failingFiles.length)} changed file(s) below target`); + if (missing.some((item) => item.reason === "no coverage data")) + failures.push("changed source missing coverage data"); + process.stderr.write(`Changed-line coverage target not met: ${failures.join("; ")}.\n`); process.exit(1); } @@ -122,16 +157,20 @@ function normalizeSourcePath(packageRoot, sourceFile) { } function changedSourceLines(ref) { - const diff = execFileSync("git", ["diff", "--unified=0", `${ref}...HEAD`, "--", "packages"], { + const mergeBase = execFileSync("git", ["merge-base", ref, "HEAD"], { + cwd: repoRoot, + encoding: "utf8" + }).trim(); + const diff = execFileSync("git", ["diff", "--unified=0", mergeBase, "--", ...sourcePaths], { cwd: repoRoot, encoding: "utf8" }); const changedLines = new Map(); let currentFile; for (const line of diff.split("\n")) { - if (line.startsWith("+++ b/")) { - const file = line.slice("+++ b/".length); - currentFile = file.includes("/src/") && /\.[cm]?[jt]sx?$/.test(file) ? file : undefined; + if (line.startsWith("+++ ")) { + const file = line.startsWith("+++ b/") ? line.slice("+++ b/".length) : undefined; + currentFile = file !== undefined && isIncludedSource(file) ? file : undefined; continue; } if (currentFile === undefined || !line.startsWith("@@")) continue; @@ -144,9 +183,27 @@ function changedSourceLines(ref) { for (let offset = 0; offset < count; offset += 1) lines.add(start + offset); changedLines.set(currentFile, lines); } + const untracked = execFileSync( + "git", + ["ls-files", "--others", "--exclude-standard", "--", ...sourcePaths], + { + cwd: repoRoot, + encoding: "utf8" + } + ); + for (const file of untracked.split("\n").filter(isIncludedSource)) { + const contents = readFileSync(resolve(repoRoot, file), "utf8"); + const count = + contents.length === 0 ? 0 : contents.split("\n").length - (contents.endsWith("\n") ? 1 : 0); + changedLines.set(file, new Set(Array.from({ length: count }, (_value, index) => index + 1))); + } return changedLines; } +function isIncludedSource(file) { + return file.includes("/src/") && /\.[cm]?[jt]sx?$/.test(file); +} + function collapseMissing(items) { const groups = new Map(); for (const item of items) { diff --git a/scripts/check-changed-coverage.test.mjs b/scripts/check-changed-coverage.test.mjs new file mode 100644 index 0000000..94545fe --- /dev/null +++ b/scripts/check-changed-coverage.test.mjs @@ -0,0 +1,171 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const sourceScript = fileURLToPath(new URL("./check-changed-coverage.mjs", import.meta.url)); + +async function createRepository(t) { + const root = await mkdtemp(join(tmpdir(), "changed-coverage-")); + t.after(() => rm(root, { recursive: true })); + await mkdir(join(root, "scripts"), { recursive: true }); + await mkdir(join(root, "packages/core/src"), { recursive: true }); + await mkdir(join(root, "packages/core/coverage"), { recursive: true }); + await copyFile(sourceScript, join(root, "scripts/check-changed-coverage.mjs")); + await writeFile(join(root, ".gitignore"), "**/coverage/\n"); + await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 1;\n"); + git(root, "init"); + git(root, "config", "user.email", "coverage@example.test"); + git(root, "config", "user.name", "Coverage Test"); + git(root, "add", "."); + git(root, "commit", "-m", "initial"); + const checker = await readFile(sourceScript, "utf8"); + const reports = [...checker.matchAll(/["']([^"']+\/coverage\/lcov\.info)["']/g)].map( + (match) => match[1] + ); + for (const report of reports) { + await mkdir(dirname(join(root, report)), { recursive: true }); + await writeFile(join(root, report), ""); + } + return root; +} + +function git(root, ...args) { + return execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); +} + +async function writeCoverage(root, file, hits, branches = [], append = false) { + const lines = [ + `SF:src/${file}`, + ...hits.map((count, index) => `DA:${String(index + 1)},${String(count)}`) + ]; + lines.push(...branches, "end_of_record", ""); + await writeFile( + join(root, "packages/core/coverage/lcov.info"), + lines.join("\n"), + append ? { flag: "a" } : undefined + ); +} + +function runCheck(root, baseRef, target = "100") { + return spawnSync(process.execPath, ["scripts/check-changed-coverage.mjs"], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + PATCH_COVERAGE_BASE: baseRef, + PATCH_COVERAGE_TARGET: target + } + }); +} + +function check(root, baseRef) { + const result = runCheck(root, baseRef); + assert.equal(result.status, 0, `${result.stdout}${result.stderr}`); + return result.stdout; +} + +function checkFailure(root, baseRef, target = "100") { + const result = runCheck(root, baseRef, target); + assert.notEqual(result.status, 0); + return `${result.stdout}${result.stderr}`; +} + +for (const state of ["committed", "staged", "unstaged"]) { + test(`measures ${state} source changes`, async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile( + join(root, "packages/core/src/existing.ts"), + "export const value = 2;\nexport const added = 3;\n" + ); + if (state !== "unstaged") git(root, "add", "packages/core/src/existing.ts"); + if (state === "committed") git(root, "commit", "-m", "change source"); + await writeCoverage(root, "existing.ts", [1, 1]); + + assert.match( + check(root, baseRef), + /Changed-line coverage: 2\/2 executable changed source lines \(100\.00%\)\./ + ); + }); +} + +test("measures untracked source files", async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile( + join(root, "packages/core/src/new.ts"), + "export const first = 1;\nexport const second = 2;\n" + ); + await writeCoverage(root, "new.ts", [1, 1]); + + assert.match( + check(root, baseRef), + /Changed-line coverage: 2\/2 executable changed source lines \(100\.00%\)\./ + ); +}); + +test("rejects a partially covered changed line", async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile( + join(root, "packages/core/src/existing.ts"), + "export const value = true ? 1 : 2;\n" + ); + await writeCoverage(root, "existing.ts", [1], ["BRDA:1,0,0,1", "BRDA:1,0,1,0"]); + + const output = checkFailure(root, baseRef); + assert.match(output, /Changed-line coverage: 0\/1 executable changed source lines \(0\.00%\)\./); + assert.match(output, /packages\/core\/src\/existing\.ts:1 \(partially covered\)/); +}); + +test("rejects an uncovered changed executable line", async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n"); + await writeCoverage(root, "existing.ts", [0]); + + const output = checkFailure(root, baseRef); + assert.match(output, /packages\/core\/src\/existing\.ts:1 \(uncovered\)/); +}); + +test("rejects a changed source file missing from coverage data", async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n"); + await writeCoverage(root, "other.ts", [1]); + + const output = checkFailure(root, baseRef); + assert.match(output, /packages\/core\/src\/existing\.ts:1 \(no coverage data\)/); +}); + +test("rejects a changed file below target when aggregate coverage meets target", async (t) => { + const root = await createRepository(t); + const baseRef = git(root, "rev-parse", "HEAD"); + await writeFile(join(root, "packages/core/src/existing.ts"), "export const value = 2;\n"); + await writeFile( + join(root, "packages/core/src/new.ts"), + Array.from( + { length: 9 }, + (_value, index) => `export const value${String(index)} = ${String(index)};` + ).join("\n") + ); + await writeCoverage(root, "existing.ts", [0]); + await writeCoverage( + root, + "new.ts", + Array.from({ length: 9 }, () => 1), + [], + true + ); + + const output = checkFailure(root, baseRef, "90"); + assert.match( + output, + /Changed-line coverage: 9\/10 executable changed source lines \(90\.00%\)\./ + ); + assert.match(output, /packages\/core\/src\/existing\.ts: 0\/1 \(0\.00%\)/); +});