From 5b55c651e45840e5be5e408a847953616cbafed7 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Wed, 30 Sep 2026 23:45:40 +0300 Subject: [PATCH] Map exact c8 TypeScript if arms to CFG coverage --- usvm-ts-fast-check/README.md | 14 + usvm-ts-fast-check/build.gradle.kts | 1 + .../fast-check-adapter/package-lock.json | 8 +- .../fast-check-adapter/package.json | 9 +- .../src/coverage-branches.ts | 336 ++++++++++++++++++ .../test/coverage-branches.test.ts | 262 ++++++++++++++ .../pbt/fastcheck/FastCheckCoverageSession.kt | 97 ++++- .../ts/pbt/fastcheck/FastCheckCoverageTest.kt | 109 +++++- .../properties/coverage/CoverageProperties.ts | 17 + .../coverage/nested-source-under-test.ts | 11 + .../coverage/nonterminal-source-under-test.ts | 7 + 11 files changed, 856 insertions(+), 15 deletions(-) create mode 100644 usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts create mode 100644 usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts create mode 100644 usvm-ts-fast-check/src/test/resources/properties/coverage/nested-source-under-test.ts create mode 100644 usvm-ts-fast-check/src/test/resources/properties/coverage/nonterminal-source-under-test.ts diff --git a/usvm-ts-fast-check/README.md b/usvm-ts-fast-check/README.md index 746c3ea1ab..97bf0d95c6 100644 --- a/usvm-ts-fast-check/README.md +++ b/usvm-ts-fast-check/README.md @@ -7,6 +7,20 @@ runtime. The public property model, validation, registries, coverage contracts and decoders, and property-to-EtsIR mapping remain in [`usvm-ts-pbt`](../usvm-ts-pbt/README.md). +## Branch coverage boundary + +One c8 execution produces the existing source-mapped Istanbul statement report and raw V8 ranges. The bounded +TypeScript branch converter reads those same raw ranges, the executed source snapshot, and the source map. It emits +ordered `if` arms only when the original TypeScript points map back exactly, each arm has a distinct V8 execution +point, and the arm counts add up to the count at the condition. A supported `if` without `else` additionally needs +a single terminating `return` or `throw` in its true arm and a following statement that is reached only on false. +Nested `if` statements are supported under those conditions. Ambiguous counts or ranges and unsupported constructs +produce diagnostics; exact statement mappings remain available. + +Loops, `switch`, conditional expressions, logical short-circuit ranges, and function or script ranges are never +reinterpreted as `if` arms. The converter does not instrument the TypeScript program or execute it a second time. +The resulting edges are coverage artifacts; target-selection integration belongs to #399/#355. + Run the backend checks with: ```shell diff --git a/usvm-ts-fast-check/build.gradle.kts b/usvm-ts-fast-check/build.gradle.kts index bd10be1065..de9c78d804 100644 --- a/usvm-ts-fast-check/build.gradle.kts +++ b/usvm-ts-fast-check/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { implementation(Libs.clikt) implementation(Libs.kotlinx_serialization_json) + testImplementation(Libs.jacodb_ets) testImplementation(Libs.logback) } diff --git a/usvm-ts-fast-check/fast-check-adapter/package-lock.json b/usvm-ts-fast-check/fast-check-adapter/package-lock.json index 594dfb5981..3db9388792 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package-lock.json +++ b/usvm-ts-fast-check/fast-check-adapter/package-lock.json @@ -8,13 +8,14 @@ "name": "@usvm/fast-check-adapter", "version": "0.1.0", "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", "c8": "10.1.3", "fast-check": "4.9.0", - "tsx": "4.23.12" + "tsx": "4.23.12", + "typescript": "5.9.2" }, "devDependencies": { - "@types/node": "18.19.130", - "typescript": "5.9.2" + "@types/node": "18.19.130" }, "engines": { "node": ">=18.18.0" @@ -1304,7 +1305,6 @@ "version": "5.9.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/usvm-ts-fast-check/fast-check-adapter/package.json b/usvm-ts-fast-check/fast-check-adapter/package.json index 1312c0b1c0..44d0002d31 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package.json +++ b/usvm-ts-fast-check/fast-check-adapter/package.json @@ -9,16 +9,17 @@ "build": "tsc --project tsconfig.json", "pretest": "npm run build", "test": "npm run test:compiled", - "test:compiled": "node --test dist/test/entry-point.test.js dist/test/local-source-closure.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js" + "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/local-source-closure.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js" }, "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", "c8": "10.1.3", "fast-check": "4.9.0", - "tsx": "4.23.12" + "tsx": "4.23.12", + "typescript": "5.9.2" }, "devDependencies": { - "@types/node": "18.19.130", - "typescript": "5.9.2" + "@types/node": "18.19.130" }, "overrides": { "c8": { diff --git a/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts new file mode 100644 index 0000000000..7d78736563 --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts @@ -0,0 +1,336 @@ +import { readFile, readdir, realpath } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + generatedPositionFor, + originalPositionFor, + TraceMap, +} from '@jridgewell/trace-mapping'; +import ts from 'typescript'; + +interface V8Range { + startOffset: number; + endOffset: number; + count: number; +} + +interface V8Function { + ranges: V8Range[]; + isBlockCoverage: boolean; +} + +interface V8Script { + url: string; + functions: V8Function[]; +} + +interface SourceMapCacheEntry { + lineLengths: number[]; + data: { + version: 3; + sources: string[]; + sourcesContent?: (string | null)[]; + mappings: string; + names: string[]; + }; +} + +interface V8Report { + result?: V8Script[]; + 'source-map-cache'?: Record; +} + +interface Position { + line: number; + column: number; +} + +interface Range { + start: Position; + end: Position; +} + +interface Branch { + branchId: number; + type: 'if'; + location: Range; + arms: { location: Range; hits: number }[]; +} + +interface BranchDiagnostic { + code: string; + message: string; + path: string; +} + +interface BranchReport { + files: { path: string; branches: Branch[] }[]; + diagnostics: BranchDiagnostic[]; +} + +interface ScriptCoverage { + script: V8Script; + cache: SourceMapCacheEntry; + path: string; +} + +/** Reconstructs only source-mapped TypeScript if arms whose V8 counters agree. */ +export async function convertV8IfBranches(rawDirectory: string, sourceRoots: string[]): Promise { + const roots = await Promise.all(sourceRoots.map(root => realpath(root))); + const names = (await readdir(rawDirectory)).filter(name => name.endsWith('.json')).sort(); + const reports = await Promise.all(names.map(async name => + JSON.parse(await readFile(path.join(rawDirectory, name), 'utf8')) as V8Report, + )); + const scripts = await collectScripts(reports, roots); + const files: BranchReport['files'] = []; + const diagnostics: BranchDiagnostic[] = []; + + for (const [sourcePath, candidates] of scripts) { + if (candidates.length !== 1 || candidates[0] === undefined) { + diagnostics.push(diagnostic( + sourcePath, + 'ambiguous', + 'The TypeScript source has several V8 scripts or a script without source-map data', + )); + continue; + } + + const converted = await convertScript(candidates[0]); + files.push({ path: sourcePath, branches: converted.branches.map((branch, branchId) => ({ + ...branch, + branchId, + })) }); + diagnostics.push(...converted.diagnostics); + } + + return { files, diagnostics }; +} + +async function collectScripts(reports: V8Report[], roots: string[]): Promise> { + const scripts = new Map(); + + for (const report of reports) { + for (const script of report.result ?? []) { + if (!script.url.startsWith('file:')) continue; + + const sourcePath = await realpath(fileURLToPath(script.url)).catch(() => undefined); + if (sourcePath === undefined || !sourcePath.endsWith('.ts')) continue; + if (!roots.some(root => sourcePath.startsWith(`${root}${path.sep}`))) continue; + + const cache = report['source-map-cache']?.[script.url]; + const candidates = scripts.get(sourcePath) ?? []; + candidates.push(cache === undefined ? undefined : { script, cache, path: sourcePath }); + scripts.set(sourcePath, candidates); + } + } + + return scripts; +} + +async function convertScript(coverage: ScriptCoverage): Promise<{ + branches: Branch[]; + diagnostics: BranchDiagnostic[]; +}> { + const source = await readFile(coverage.path, 'utf8'); + const sourceIndex = coverage.cache.data.sources.findIndex(candidate => + candidate.startsWith('file:') && fileURLToPath(candidate) === coverage.path, + ); + const coveredSource = coverage.cache.data.sourcesContent?.[sourceIndex]; + if (coveredSource !== source) { + return { + branches: [], + diagnostics: [diagnostic(coverage.path, 'unsupported', 'Covered TypeScript source differs from the current file')], + }; + } + + const ast = ts.createSourceFile(coverage.path, source, ts.ScriptTarget.Latest, true); + const map = new TraceMap(coverage.cache.data); + const branches: Branch[] = []; + const diagnostics: BranchDiagnostic[] = []; + + function visit(node: ts.Node, unsupportedAncestor: boolean): void { + const unsupported = unsupportedAncestor || isRepeatedOrExceptional(node); + + if (ts.isIfStatement(node)) { + const result = convertIf(node, ast, coverage, map, unsupported); + if (result.branch !== undefined) branches.push(result.branch); + if (result.reason !== undefined) diagnostics.push(diagnostic(coverage.path, result.kind, result.reason)); + } + + ts.forEachChild(node, child => visit(child, unsupported)); + } + + visit(ast, false); + + return { branches, diagnostics }; +} + +function convertIf( + node: ts.IfStatement, + ast: ts.SourceFile, + coverage: ScriptCoverage, + map: TraceMap, + unsupportedAncestor: boolean, +): { branch?: Branch; reason?: string; kind: 'unsupported' | 'ambiguous' } { + if (unsupportedAncestor || hasUnsupportedExpression(node.expression)) { + return { kind: 'unsupported', reason: 'If condition is inside an unsupported control-flow construct' }; + } + + const trueBody = firstExecutable(node.thenStatement); + const falseBody = node.elseStatement === undefined + ? falseSuccessor(node) + : firstExecutable(node.elseStatement); + if (trueBody === undefined) { + return { kind: 'unsupported', reason: 'Empty true arm has no V8 execution point' }; + } + if (falseBody === undefined) { + const reason = node.elseStatement === undefined + ? 'An if without else requires a terminating true arm and a following false successor' + : 'Empty false arm has no V8 execution point'; + + return { kind: 'unsupported', reason }; + } + + const conditionHits = countAt(node.getStart(ast), ast, coverage, map); + const trueHits = countAt(trueBody.getStart(ast), ast, coverage, map); + const falseHits = countAt(falseBody.getStart(ast), ast, coverage, map); + + if (conditionHits === undefined || trueHits === undefined || falseHits === undefined || + trueHits < 0 || falseHits < 0 || trueHits + falseHits !== conditionHits) { + return { kind: 'ambiguous', reason: 'V8 ranges and exact TypeScript source-map points do not identify both if arms' }; + } + + const branch: Branch = { + branchId: 0, + type: 'if', + // The predicate span excludes nested if statements from this branch's EtsIR target candidates. + location: sourceRange(node.expression, ast), + arms: [ + { location: sourceRange(node.thenStatement, ast), hits: trueHits }, + { + location: node.elseStatement === undefined + ? sourceRange(node.expression, ast) + : sourceRange(node.elseStatement, ast), + hits: falseHits, + }, + ], + }; + + return { branch, kind: 'ambiguous' }; +} + +function countAt( + sourceOffset: number, + ast: ts.SourceFile, + coverage: ScriptCoverage, + map: TraceMap, +): number | undefined { + const sourcePosition = ast.getLineAndCharacterOfPosition(sourceOffset); + const sourceUrl = coverage.cache.data.sources.find(candidate => + candidate.startsWith('file:') && fileURLToPath(candidate) === coverage.path, + ); + if (sourceUrl === undefined) return undefined; + + const generated = generatedPositionFor(map, { + source: sourceUrl, + line: sourcePosition.line + 1, + column: sourcePosition.character, + }); + if (generated.line === null || generated.column === null) return undefined; + + const original = originalPositionFor(map, { line: generated.line, column: generated.column }); + if (original.source !== sourceUrl || original.line !== sourcePosition.line + 1 || + original.column !== sourcePosition.character) return undefined; + + const lineLengths = coverage.cache.lineLengths; + if (lineLengths.some(length => !Number.isSafeInteger(length) || length < 0) || + generated.line > lineLengths.length || generated.column > (lineLengths[generated.line - 1] ?? -1)) { + return undefined; + } + + const offset = lineLengths.slice(0, generated.line - 1).reduce((sum, length) => sum + length + 1, 0) + + generated.column; + const functions = coverage.script.functions.filter(fn => fn.isBlockCoverage && + fn.ranges.length > 0 && contains(fn.ranges[0] as V8Range, offset)); + const orderedFunctions = functions.sort((left, right) => + rangeLength(left.ranges[0] as V8Range) - rangeLength(right.ranges[0] as V8Range), + ); + const functionCoverage = orderedFunctions[0]; + if (functionCoverage === undefined) return undefined; + if (orderedFunctions[1] !== undefined && + rangeLength(orderedFunctions[1].ranges[0] as V8Range) === rangeLength(functionCoverage.ranges[0] as V8Range)) { + return undefined; + } + + const ranges = functionCoverage.ranges.filter(range => contains(range, offset)); + const orderedRanges = ranges.sort((left, right) => rangeLength(left) - rangeLength(right)); + const innermost = orderedRanges[0]; + if (innermost === undefined || orderedRanges[1] !== undefined && + rangeLength(orderedRanges[1]) === rangeLength(innermost)) return undefined; + + return Number.isSafeInteger(innermost.count) && innermost.count >= 0 ? innermost.count : undefined; +} + +function firstExecutable(statement: ts.Statement): ts.Statement | undefined { + if (ts.isBlock(statement)) return statement.statements[0]; + + return statement; +} + +function falseSuccessor(node: ts.IfStatement): ts.Statement | undefined { + const consequent = node.thenStatement; + const soleStatement = ts.isBlock(consequent) ? consequent.statements[0] : consequent; + const terminates = soleStatement !== undefined && + (!ts.isBlock(consequent) || consequent.statements.length === 1) && + (ts.isReturnStatement(soleStatement) || ts.isThrowStatement(soleStatement)); + if (!terminates) return undefined; + + const parent = node.parent; + if (!ts.isBlock(parent) && !ts.isSourceFile(parent)) return undefined; + + const index = parent.statements.indexOf(node); + + return index < 0 ? undefined : parent.statements[index + 1]; +} + +const isRepeatedOrExceptional = (node: ts.Node): boolean => + ts.isForStatement(node) || ts.isForInStatement(node) || ts.isForOfStatement(node) || + ts.isWhileStatement(node) || ts.isDoStatement(node) || ts.isSwitchStatement(node) || + ts.isTryStatement(node) || ts.isConditionalExpression(node) || + ts.isBinaryExpression(node) && [ + ts.SyntaxKind.AmpersandAmpersandToken, + ts.SyntaxKind.BarBarToken, + ts.SyntaxKind.QuestionQuestionToken, + ].includes(node.operatorToken.kind); + +function hasUnsupportedExpression(node: ts.Node): boolean { + if (isRepeatedOrExceptional(node)) return true; + + return ts.forEachChild(node, child => hasUnsupportedExpression(child) ? true : undefined) === true; +} + +function sourceRange(node: ts.Node, ast: ts.SourceFile): Range { + const start = ast.getLineAndCharacterOfPosition(node.getStart(ast)); + const end = ast.getLineAndCharacterOfPosition(node.getEnd()); + + return { + start: { line: start.line + 1, column: start.character }, + end: { line: end.line + 1, column: end.character }, + }; +} + +const contains = (range: V8Range, offset: number): boolean => + range.startOffset <= offset && offset < range.endOffset; + +const rangeLength = (range: V8Range): number => range.endOffset - range.startOffset; + +function diagnostic(sourcePath: string, kind: string, message: string): BranchDiagnostic { + return { code: `coverage.branch.${kind}`, message, path: sourcePath }; +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [rawDirectory, ...sourceRoots] = process.argv.slice(2); + if (rawDirectory === undefined) throw new Error('Raw V8 coverage directory is required'); + + process.stdout.write(`${JSON.stringify(await convertV8IfBranches(rawDirectory, sourceRoots))}\n`); +} diff --git a/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts new file mode 100644 index 0000000000..e0a3e06717 --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts @@ -0,0 +1,262 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { cp, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import { convertV8IfBranches } from '../src/coverage-branches.js'; + +const adapterRoot = path.resolve(fileURLToPath(new URL('../', import.meta.url)), '..'); +const c8 = path.join(adapterRoot, 'node_modules/c8/bin/c8.js'); + +async function runCoverage(source: string): Promise<{ + root: string; + raw: string; + sourcePath: string; +}> { + const root = await mkdtemp(path.join(os.tmpdir(), 'usvm-branch-test-')); + const sourcePath = path.join(root, 'subject.ts'); + const raw = path.join(root, 'raw'); + await writeFile(sourcePath, source); + + const result = spawnSync(process.execPath, [ + c8, + '--reporter=json', + `--reports-dir=${path.join(root, 'report')}`, + `--temp-directory=${raw}`, + '--allowExternal', + '--exclude=__usvm_no_default_excludes__', + process.execPath, + '--import', + 'tsx', + sourcePath, + ], { cwd: adapterRoot, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + + return { root, raw, sourcePath }; +} + +test('real c8 and tsx preserve true, false, mixed, and nested TypeScript arms', async () => { + for (const [calls, expected] of [ + ['nested(20);', [[1, 0], [1, 0]]], + ['nested(-1);', [[0, 1], [0, 0]]], + ['nested(20); nested(5); nested(-1);', [[2, 1], [1, 1]]], + ] as const) { + const fixture = await runCoverage(` +export function nested(value: number): string { + if (value > 0) { + if (value > 10) { + return 'large'; + } else { + return 'small'; + } + } else { + return 'negative'; + } +} +${calls} +`); + + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + const branches = report.files[0]?.branches ?? []; + + assert.deepEqual(branches.map(branch => branch.arms.map(arm => arm.hits)), expected); + assert.deepEqual(branches.map(branch => branch.location.start.line), [3, 4]); + assert.deepEqual(report.diagnostics, []); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } + } +}); + +test('if without else observes an actual false successor and rejects throwing conditions', async () => { + const fixture = await runCoverage(` +function gate(value: number): boolean { + if (value < 0) throw new Error('guard failed'); + return value > 0; +} +function subject(value: number): number { + if (gate(value)) { + return 1; + } + return 0; +} +for (const value of [-1, 1]) { + try { subject(value); } catch { /* expected */ } +} +`); + + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.equal(report.files[0]?.branches.some(branch => branch.location.start.line === 7), false); + assert.equal(report.diagnostics.some(diagnostic => diagnostic.code === 'coverage.branch.ambiguous'), true); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test('if without else maps both arms when the true arm terminates', async () => { + for (const [calls, expected] of [ + ['subject(1);', [1, 0]], + ['subject(-1);', [0, 1]], + ['subject(1); subject(-1);', [1, 1]], + ] as const) { + const fixture = await runCoverage(` +function subject(value: number): number { + if (value > 0) { + return 1; + } + return 0; +} +${calls} +`); + + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(report.files[0]?.branches[0]?.arms.map(arm => arm.hits), expected); + assert.deepEqual(report.diagnostics, []); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } + } +}); + +test('duplicate V8 scripts remain ambiguous', async () => { + const fixture = await runCoverage(` +function subject(value: number): number { + if (value > 0) return 1; + else return 0; +} +subject(1); +`); + + try { + const reports = (await readdir(fixture.raw)).filter(name => name.endsWith('.json')); + let scriptReport: string | undefined; + for (const name of reports) { + const report = JSON.parse(await readFile(path.join(fixture.raw, name), 'utf8')) as { + result?: { url: string }[]; + }; + + if (report.result?.some(script => script.url.endsWith('/subject.ts'))) { + scriptReport = name; + break; + } + } + assert.ok(scriptReport); + await cp(path.join(fixture.raw, scriptReport), path.join(fixture.raw, 'duplicate.json')); + + const duplicate = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(duplicate.files, []); + assert.equal(duplicate.diagnostics[0]?.code, 'coverage.branch.ambiguous'); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test('unmapped source-map positions do not produce branch counts', async () => { + const fixture = await runCoverage(` +function subject(value: number): number { + if (value > 0) return 1; + else return 0; +} +subject(1); +`); + + try { + let changed = 0; + for (const name of await readdir(fixture.raw)) { + if (!name.endsWith('.json')) continue; + + const reportPath = path.join(fixture.raw, name); + const report = JSON.parse(await readFile(reportPath, 'utf8')) as { + 'source-map-cache'?: Record; + }; + const entry = Object.entries(report['source-map-cache'] ?? {}) + .find(([url]) => url.endsWith('/subject.ts'))?.[1]; + if (entry === undefined) continue; + + entry.data.mappings = ''; + await writeFile(reportPath, JSON.stringify(report)); + changed += 1; + } + assert.ok(changed > 0); + + const converted = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(converted.files[0]?.branches, []); + assert.equal(converted.diagnostics[0]?.code, 'coverage.branch.ambiguous'); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test('function, script, switch, conditional, and logical ranges are not if arms', async () => { + const fixture = await runCoverage(` +function subject(value: number): boolean { + switch (value) { + case 1: return value > 0 && value < 2; + default: return value === 0 ? true : false; + } +} +subject(1); +`); + + try { + const converted = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(converted.files[0]?.branches, []); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test('if ranges inside loops are explicitly unsupported', async () => { + const fixture = await runCoverage(` +function subject(value: number): number { + for (let index = 0; index < 2; index += 1) { + if (value > index) return index; + else value += 1; + } + return -1; +} +subject(1); +`); + + try { + const converted = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(converted.files[0]?.branches, []); + assert.equal(converted.diagnostics[0]?.code, 'coverage.branch.unsupported'); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test('early exits and UTF-16 columns retain exact source positions', async () => { + const fixture = await runCoverage(` +function subject(value: number): number { + const marker = '😀'; if (value < 0) return -1; + if (value > 0) return marker.length; + return 0; +} +subject(-1); +subject(1); +`); + + try { + const converted = await convertV8IfBranches(fixture.raw, [fixture.root]); + const branches = converted.files[0]?.branches ?? []; + + assert.deepEqual(branches.map(branch => branch.arms.map(arm => arm.hits)), [[1, 1], [1, 0]]); + assert.equal(branches[0]?.location.start.column, 27); + assert.deepEqual(converted.diagnostics, []); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt index d4567c9f77..1c3a127915 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt @@ -1,13 +1,19 @@ package org.usvm.ts.pbt.fastcheck +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString import org.usvm.ts.pbt.FastCheckDiagnosticCode +import org.usvm.ts.pbt.backend.BranchCoverage +import org.usvm.ts.pbt.backend.CoverageDiagnostic import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyRunResult import org.usvm.ts.pbt.coverage.CoverageArtifactException import org.usvm.ts.pbt.coverage.IstanbulCoverageContext import org.usvm.ts.pbt.coverage.decodeIstanbulCoverageReport import org.usvm.ts.pbt.coverage.inspectRawV8SourceMapDiagnostics import org.usvm.ts.pbt.coverage.mergeCoverageDiagnostics +import org.usvm.ts.pbt.manifest.PropertyManifestJson import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -68,7 +74,15 @@ internal class FastCheckCoverageSession private constructor( rawDiagnostics = rawDiagnostics, ) - finalArtifact.copy(diagnostics = diagnostics) + val branchReport = if (finalArtifact.files.isEmpty()) { + RawBranchReport(files = emptyList(), diagnostics = emptyList()) + } else { + convertBranches() + } + appendExactBranches( + artifact = finalArtifact.copy(diagnostics = diagnostics), + report = branchReport, + ) } catch (error: CoverageArtifactException) { fail( code = error.diagnostic.code, @@ -96,6 +110,71 @@ internal class FastCheckCoverageSession private constructor( return paths } + private fun convertBranches(): RawBranchReport { + val converter = workspace.adapterRoot.resolve("dist/src/coverage-branches.js") + val transport = FastCheckProcessTransport( + nodeExecutable = nodeExecutable, + maxRequestBytes = 1, + maxStdoutBytes = MAX_BRANCH_REPORT_BYTES, + maxStderrBytes = MAX_BRANCH_ERROR_BYTES, + shutdownGraceMillis = BRANCH_SHUTDOWN_GRACE_MILLIS, + ) + val output = try { + transport.invoke( + command = listOf(nodeExecutable, converter.toString(), workspace.rawDirectory.toString()) + + request.sourceRoots, + request = "", + timeoutMillis = BRANCH_CONVERSION_TIMEOUT_MILLIS, + reportedTimeoutMillis = BRANCH_CONVERSION_TIMEOUT_MILLIS, + description = "TypeScript branch converter", + ) + } catch (error: FastCheckTransportException) { + return unsupportedBranches("Cannot convert TypeScript branches: ${error.message}") + } + if (output.exitCode != 0) { + return unsupportedBranches("TypeScript branch conversion failed: ${output.stderr.trim()}") + } + + return runCatching { + PropertyManifestJson.json.decodeFromString(output.stdout) + }.getOrElse { error -> + unsupportedBranches("Cannot read TypeScript branch coverage: ${error.message}") + } + } + + private fun appendExactBranches( + artifact: PropertyCoverageArtifact, + report: RawBranchReport, + ): PropertyCoverageArtifact { + val exactByPath = report.files.associate { file -> file.path to file.branches } + val retainedPaths = artifact.files.mapTo(hashSetOf()) { file -> file.path } + val files = artifact.files.map { file -> + val nextId = (file.branches.maxOfOrNull(BranchCoverage::branchId) ?: -1) + 1 + val exactBranches = exactByPath[file.path].orEmpty().mapIndexed { index, branch -> + branch.copy(branchId = nextId + index) + } + + file.copy(branches = file.branches + exactBranches) + } + + return artifact.copy( + files = files, + diagnostics = artifact.diagnostics + report.diagnostics.filter { diagnostic -> + diagnostic.path == null || diagnostic.path in retainedPaths + }, + ) + } + + private fun unsupportedBranches(message: String) = RawBranchReport( + files = emptyList(), + diagnostics = listOf( + CoverageDiagnostic( + code = "coverage.branch.unsupported", + message = message, + ), + ), + ) + override fun close() { workspace.root.toFile().deleteRecursively() } @@ -270,12 +349,28 @@ internal class FastCheckCoverageSession private constructor( ) private const val NODE_VERSION_TIMEOUT_MILLIS = 5_000L + private const val BRANCH_CONVERSION_TIMEOUT_MILLIS = 10_000L + private const val BRANCH_SHUTDOWN_GRACE_MILLIS = 500L + private const val MAX_BRANCH_REPORT_BYTES = 10_000_000 + private const val MAX_BRANCH_ERROR_BYTES = 1_000 private const val MINIMUM_NODE_MAJOR_VERSION = 18 private const val MINIMUM_NODE_MINOR_VERSION = 18 private val NODE_VERSION_PATTERN = Regex("""^v(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$""") } } +@Serializable +private data class RawBranchReport( + val files: List, + val diagnostics: List, +) + +@Serializable +private data class RawBranchFile( + val path: String, + val branches: List, +) + private data class CoverageWorkspace( val root: Path, val configPath: Path, diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt index cfcfa58d4d..4c4dee032c 100644 --- a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt @@ -1,5 +1,8 @@ package org.usvm.ts.pbt.fastcheck +import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.utils.EtsIrProvider +import org.jacodb.ets.utils.loadEtsFileAutoConvert import org.junit.jupiter.api.Test import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.PropertyCoverageRequest @@ -7,7 +10,11 @@ import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult import org.usvm.ts.pbt.backend.PropertyRunStatus import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.manifest.toManifest +import org.usvm.ts.pbt.mapping.EtsMappingStatus +import org.usvm.ts.pbt.mapping.PropertyEtsMapper import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.PropertyDefinition import org.usvm.ts.pbt.model.PropertyId import org.usvm.ts.pbt.model.PropertyInput @@ -58,12 +65,9 @@ class FastCheckCoverageTest { assertEquals(listOf(2), zeroHitBranchLines(nonPositiveFile)) assertEquals(1L, statementHitsAtLine(positiveFile, line = 2)) assertEquals(1L, statementHitsAtLine(nonPositiveFile, line = 2)) - assertTrue( - (positiveFile.branches + nonPositiveFile.branches).all { branch -> - branch.type == "branch" && branch.arms.size == 1 - }, - "The pinned c8/V8 collector must expose its backend-specific single-arm branch shape", - ) + assertEquals(listOf(1L, 0L), exactIf(positiveFile).arms.map { arm -> arm.hits }) + assertEquals(listOf(0L, 1L), exactIf(nonPositiveFile).arms.map { arm -> arm.hits }) + assertTrue(positiveFile.branches.any { branch -> branch.type == "branch" && branch.arms.size == 1 }) } @Test @@ -84,6 +88,97 @@ class FastCheckCoverageTest { assertEquals(listOf(5), zeroHitBranchLines(file)) } + @Test + fun `real backend reports mixed TypeScript branch hits`() { + val result = backend.run( + property = property( + exportName = "alwaysCovers", + domain = IntegerDomain(min = -1, max = 1), + ), + configuration = configuration.copy( + numRuns = 3, + examples = listOf( + listOf(JsConcreteValue.number(-1.0)), + listOf(JsConcreteValue.number(1.0)), + ), + ), + ) + + assertEquals(PropertyRunStatus.SUCCESS, result.status) + val armHits = exactIf(sourceUnderTest(result)).arms.map { arm -> arm.hits } + assertEquals(3L, armHits.sum()) + assertTrue(armHits.all { hits -> hits >= 1L }) + } + + @Test + fun `real TypeScript branch arms map to ordered EtsIR successors`() { + val definition = property( + exportName = "coversPositive", + domain = IntegerDomain(min = 1, max = 1), + ) + val result = backend.run(property = definition, configuration = configuration) + val source = sourceRoot().resolve("properties/coverage/source-under-test.ts").toRealPath() + val file = loadEtsFileAutoConvert(source, provider = EtsIrProvider.TS_FRONTEND) + val mapper = PropertyEtsMapper( + scene = EtsScene(listOf(file)), + sourceRoots = listOf(source.parent), + ) + + val mapped = mapper.map(definition.toManifest(), assertNotNull(result.coverage)) + val branch = mapped.coverage.branches.single { candidate -> candidate.coverage.type == "if" } + + assertEquals(EtsMappingStatus.EXACT, branch.mapping.status) + assertEquals(listOf(1L, 0L), branch.arms.map { arm -> arm.coverage.hits }) + assertEquals(listOf(true, false), branch.arms.map { arm -> arm.mapping.targets.single().outcome }) + } + + @Test + fun `nested TypeScript branches map to distinct EtsIR conditions`() { + val definition = property( + exportName = "coversNested", + domain = IntegerDomain(min = 20, max = 20), + ) + val result = backend.run(property = definition, configuration = configuration) + val source = sourceRoot().resolve("properties/coverage/nested-source-under-test.ts").toRealPath() + val file = loadEtsFileAutoConvert(source, provider = EtsIrProvider.TS_FRONTEND) + val mapper = PropertyEtsMapper( + scene = EtsScene(listOf(file)), + sourceRoots = listOf(source.parent), + ) + + val mapped = mapper.map(definition.toManifest(), assertNotNull(result.coverage)) + val branches = mapped.coverage.branches.filter { branch -> branch.coverage.type == "if" } + + assertEquals(2, branches.size) + assertTrue(branches.all { branch -> branch.mapping.status == EtsMappingStatus.EXACT }) + assertEquals(listOf(1L, 0L), branches[0].arms.map { arm -> arm.coverage.hits }) + assertEquals(listOf(1L, 0L), branches[1].arms.map { arm -> arm.coverage.hits }) + val outcomes = branches.map { branch -> + branch.arms.map { arm -> arm.mapping.targets.single().outcome } + } + assertEquals(listOf(listOf(true, false), listOf(true, false)), outcomes) + } + + @Test + fun `unsupported TypeScript branch retains exact statement coverage`() { + val result = backend.run( + property = property( + exportName = "coversNonterminal", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + + val coverage = assertNotNull(result.coverage) + val file = coverage.files.single { candidate -> + candidate.path.endsWith("properties/coverage/nonterminal-source-under-test.ts") + } + + assertTrue(file.statements.any { statement -> statement.hits > 0 }) + assertTrue(file.branches.none { branch -> branch.type == "if" }) + assertTrue(coverage.diagnostics.any { diagnostic -> diagnostic.code == "coverage.branch.unsupported" }) + } + @Test fun `real c8 reports a missing referenced source map when the final report omits the script`() { val module = "properties/coverage/missing-map-entry.js" @@ -168,6 +263,8 @@ class FastCheckCoverageTest { return artifact.files.single { file -> file.path.endsWith("properties/coverage/source-under-test.ts") } } + private fun exactIf(file: SourceFileCoverage) = file.branches.single { branch -> branch.type == "if" } + private fun statementHitsAtLine(file: SourceFileCoverage, line: Int): Long = file.statements .filter { statement -> statement.location.start.line == line } .maxOf { statement -> statement.hits } diff --git a/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts b/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts index 1b012ceb1c..653db26a2d 100644 --- a/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts +++ b/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts @@ -1,4 +1,6 @@ import { classify } from './source-under-test.ts'; +import { classifyNested } from './nested-source-under-test.ts'; +import { nonterminal } from './nonterminal-source-under-test.ts'; export function coversPositive(value: number): boolean { return classify(value) === 'positive'; @@ -12,3 +14,18 @@ export function failsAfterClassifying(value: number): boolean { classify(value); return false; } + +export function alwaysCovers(value: number): boolean { + classify(value); + return true; +} + +export function coversNested(value: number): boolean { + classifyNested(value); + return true; +} + +export function coversNonterminal(value: number): boolean { + nonterminal(value); + return true; +} diff --git a/usvm-ts-fast-check/src/test/resources/properties/coverage/nested-source-under-test.ts b/usvm-ts-fast-check/src/test/resources/properties/coverage/nested-source-under-test.ts new file mode 100644 index 0000000000..fa46738aeb --- /dev/null +++ b/usvm-ts-fast-check/src/test/resources/properties/coverage/nested-source-under-test.ts @@ -0,0 +1,11 @@ +export function classifyNested(value: number): string { + if (value > 0) { + if (value > 10) { + return 'large'; + } else { + return 'small'; + } + } else { + return 'non-positive'; + } +} diff --git a/usvm-ts-fast-check/src/test/resources/properties/coverage/nonterminal-source-under-test.ts b/usvm-ts-fast-check/src/test/resources/properties/coverage/nonterminal-source-under-test.ts new file mode 100644 index 0000000000..b9dfccaaf9 --- /dev/null +++ b/usvm-ts-fast-check/src/test/resources/properties/coverage/nonterminal-source-under-test.ts @@ -0,0 +1,7 @@ +export function nonterminal(value: number): number { + let result = 0; + if (value > 0) { + result += 1; + } + return result; +}