From 3ad888ca9f7eabf7112597412811c34e87b731e3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Thu, 6 Aug 2026 12:17:48 -0300 Subject: [PATCH 1/2] [WIP] Add a target for easily computing code coverage Signed-off-by: Juan Cruz Viotti --- CMakeLists.txt | 12 +++++ Makefile | 10 ++++ cmake/Sourcemeta.cmake | 2 + cmake/common/compiler/coverage.cmake | 10 ++++ cmake/common/targets/coverage-report.cmake | 28 +++++++++++ cmake/common/targets/coverage.cmake | 57 ++++++++++++++++++++++ cmake/common/targets/test.cmake | 2 + 7 files changed, 121 insertions(+) create mode 100644 cmake/common/compiler/coverage.cmake create mode 100644 cmake/common/targets/coverage-report.cmake create mode 100644 cmake/common/targets/coverage.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b74544ac97..f24cb0087f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ option(SOURCEMETA_CORE_DOCS "Build the Sourcemeta Core docs" OFF) option(SOURCEMETA_CORE_INSTALL "Install the Sourcemeta Core library" ON) option(SOURCEMETA_CORE_ADDRESS_SANITIZER "Build Sourcemeta Core with an address sanitizer" OFF) option(SOURCEMETA_CORE_UNDEFINED_SANITIZER "Build Sourcemeta Core with an undefined behavior sanitizer" OFF) +option(SOURCEMETA_CORE_COVERAGE "Build Sourcemeta Core with code coverage instrumentation" OFF) option(SOURCEMETA_CORE_CONTRIB_GOOGLEBENCHMARK "Build the GoogleBenchmark library for downstream consumers" OFF) include(Sourcemeta) @@ -67,6 +68,11 @@ elseif(SOURCEMETA_CORE_UNDEFINED_SANITIZER) sourcemeta_sanitizer(TYPE undefined) endif() +# Enable coverage instrumentation before defining any target +if(SOURCEMETA_CORE_COVERAGE) + sourcemeta_coverage() +endif() + # TODO: Turn this into a re-usable utility CMake function if(SOURCEMETA_CORE_INSTALL) include(GNUInstallDirs) @@ -430,6 +436,12 @@ if(SOURCEMETA_CORE_TESTS) add_subdirectory(test/packaging) endif() endif() + + if(SOURCEMETA_CORE_COVERAGE) + sourcemeta_target_coverage( + OUTPUT "${PROJECT_BINARY_DIR}/coverage" + EXCLUDE "${PROJECT_SOURCE_DIR}/vendor" "${PROJECT_SOURCE_DIR}/test") + endif() endif() if(SOURCEMETA_CORE_TESTS_CI) diff --git a/Makefile b/Makefile index 5fa52e38cc..49599e7e70 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,16 @@ test: .always benchmark: .always $(CMAKE) --build ./build --config $(PRESET) --target benchmark_all +coverage: .always + $(CMAKE) -S . -B ./build/coverage \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR:BOOL=ON \ + -DSOURCEMETA_CORE_TESTS:BOOL=ON \ + -DSOURCEMETA_CORE_COVERAGE:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=OFF + $(CMAKE) --build ./build/coverage --config Debug + $(CMAKE) --build ./build/coverage --config Debug --target coverage + doxygen: .always $(CMAKE) --build ./build --config $(PRESET) --target doxygen diff --git a/cmake/Sourcemeta.cmake b/cmake/Sourcemeta.cmake index 70eaef5ab9..63a9ff7f68 100644 --- a/cmake/Sourcemeta.cmake +++ b/cmake/Sourcemeta.cmake @@ -4,6 +4,7 @@ include("${SOURCEMETA_UTILITIES_DIRECTORY}/variables.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/defaults.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/simd.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/sanitizer.cmake") +include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/coverage.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/options.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/options/enum.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/commands/copy-file.cmake") @@ -13,6 +14,7 @@ include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/clang-format.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/shellcheck.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/doxygen.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/test.cmake") +include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/coverage.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/googlebenchmark.cmake") include("${SOURCEMETA_UTILITIES_DIRECTORY}/clang-tidy.cmake") diff --git a/cmake/common/compiler/coverage.cmake b/cmake/common/compiler/coverage.cmake new file mode 100644 index 0000000000..13a11502b6 --- /dev/null +++ b/cmake/common/compiler/coverage.cmake @@ -0,0 +1,10 @@ +function(sourcemeta_coverage) + if(SOURCEMETA_COMPILER_LLVM) + # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html + message(STATUS "Enabling coverage: LLVM source-based coverage") + add_compile_options(-fprofile-instr-generate -fcoverage-mapping) + add_link_options(-fprofile-instr-generate) + else() + message(FATAL_ERROR "Unrecognized compiler for coverage instrumentation") + endif() +endfunction() diff --git a/cmake/common/targets/coverage-report.cmake b/cmake/common/targets/coverage-report.cmake new file mode 100644 index 0000000000..bd2a66c0ff --- /dev/null +++ b/cmake/common/targets/coverage-report.cmake @@ -0,0 +1,28 @@ +# Script executed by the `coverage` target after the test suite runs, as +# the raw profile files are only known at that point +file(GLOB COVERAGE_PROFILE_FILES "${COVERAGE_PROFILE_DIRECTORY}/*.profraw") +if(NOT COVERAGE_PROFILE_FILES) + message(FATAL_ERROR "No profile data found in ${COVERAGE_PROFILE_DIRECTORY}") +endif() + +set(COVERAGE_PROFILE_DATA "${COVERAGE_OUTPUT_DIRECTORY}/coverage.profdata") +execute_process(COMMAND "${COVERAGE_LLVM_PROFDATA}" merge -sparse + ${COVERAGE_PROFILE_FILES} -o "${COVERAGE_PROFILE_DATA}" + COMMAND_ERROR_IS_FATAL ANY) + +string(REPLACE "|" ";" COVERAGE_OBJECTS "${COVERAGE_OBJECTS}") +list(POP_FRONT COVERAGE_OBJECTS COVERAGE_MAIN_OBJECT) +set(COVERAGE_OBJECT_ARGUMENTS) +foreach(coverage_object IN LISTS COVERAGE_OBJECTS) + list(APPEND COVERAGE_OBJECT_ARGUMENTS -object "${coverage_object}") +endforeach() + +execute_process(COMMAND "${COVERAGE_LLVM_COV}" show + "${COVERAGE_MAIN_OBJECT}" ${COVERAGE_OBJECT_ARGUMENTS} + "-instr-profile=${COVERAGE_PROFILE_DATA}" + -format=html "-output-dir=${COVERAGE_OUTPUT_DIRECTORY}/html" + "-ignore-filename-regex=${COVERAGE_EXCLUDE}" + -show-branches=count + COMMAND_ERROR_IS_FATAL ANY) + +message(STATUS "Coverage report: ${COVERAGE_OUTPUT_DIRECTORY}/html/index.html") diff --git a/cmake/common/targets/coverage.cmake b/cmake/common/targets/coverage.cmake new file mode 100644 index 0000000000..42a9e25dd5 --- /dev/null +++ b/cmake/common/targets/coverage.cmake @@ -0,0 +1,57 @@ +function(sourcemeta_target_coverage) + cmake_parse_arguments(SOURCEMETA_TARGET_COVERAGE "" "OUTPUT" "EXCLUDE" ${ARGN}) + + if(NOT SOURCEMETA_TARGET_COVERAGE_OUTPUT) + message(FATAL_ERROR "You must pass an output directory using the OUTPUT option") + endif() + + get_property(COVERAGE_TEST_TARGETS GLOBAL PROPERTY SOURCEMETA_TEST_TARGETS) + if(NOT COVERAGE_TEST_TARGETS) + message(FATAL_ERROR "You must register tests before declaring the coverage target") + endif() + + # On Apple platforms, the LLVM tools that understand the profile format + # emitted by the system compiler are only reachable through `xcrun` + if(APPLE) + execute_process(COMMAND xcrun --find llvm-profdata + OUTPUT_VARIABLE COVERAGE_LLVM_PROFDATA + OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) + execute_process(COMMAND xcrun --find llvm-cov + OUTPUT_VARIABLE COVERAGE_LLVM_COV + OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) + else() + find_program(COVERAGE_LLVM_PROFDATA llvm-profdata REQUIRED) + find_program(COVERAGE_LLVM_COV llvm-cov REQUIRED) + endif() + + set(COVERAGE_OBJECTS) + foreach(test_target IN LISTS COVERAGE_TEST_TARGETS) + list(APPEND COVERAGE_OBJECTS "$") + endforeach() + # Joined with a custom separator to survive being passed as a single + # command line argument to the report script + list(JOIN COVERAGE_OBJECTS "|" COVERAGE_OBJECTS) + + set(COVERAGE_EXCLUDE "${PROJECT_BINARY_DIR}") + list(APPEND COVERAGE_EXCLUDE ${SOURCEMETA_TARGET_COVERAGE_EXCLUDE}) + list(JOIN COVERAGE_EXCLUDE "|" COVERAGE_EXCLUDE) + + set(COVERAGE_PROFILE_DIRECTORY "${SOURCEMETA_TARGET_COVERAGE_OUTPUT}/profile") + add_custom_target(coverage VERBATIM USES_TERMINAL + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMAND "${CMAKE_COMMAND}" -E rm -R -f "${COVERAGE_PROFILE_DIRECTORY}" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${COVERAGE_PROFILE_DIRECTORY}" + COMMAND "${CMAKE_COMMAND}" -E env + "LLVM_PROFILE_FILE=${COVERAGE_PROFILE_DIRECTORY}/%p.profraw" + "${CMAKE_CTEST_COMMAND}" --test-dir "${PROJECT_BINARY_DIR}" + --build-config $ --output-on-failure --parallel + COMMAND "${CMAKE_COMMAND}" + "-DCOVERAGE_LLVM_PROFDATA=${COVERAGE_LLVM_PROFDATA}" + "-DCOVERAGE_LLVM_COV=${COVERAGE_LLVM_COV}" + "-DCOVERAGE_PROFILE_DIRECTORY=${COVERAGE_PROFILE_DIRECTORY}" + "-DCOVERAGE_OUTPUT_DIRECTORY=${SOURCEMETA_TARGET_COVERAGE_OUTPUT}" + "-DCOVERAGE_OBJECTS=${COVERAGE_OBJECTS}" + "-DCOVERAGE_EXCLUDE=${COVERAGE_EXCLUDE}" + -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/coverage-report.cmake") + add_dependencies(coverage ${COVERAGE_TEST_TARGETS}) +endfunction() diff --git a/cmake/common/targets/test.cmake b/cmake/common/targets/test.cmake index 542ae97e46..3c00c8c93a 100644 --- a/cmake/common/targets/test.cmake +++ b/cmake/common/targets/test.cmake @@ -32,4 +32,6 @@ function(sourcemeta_test) add_test(NAME "${SOURCEMETA_TEST_PROJECT}.${SOURCEMETA_TEST_NAME}" COMMAND "${TARGET_NAME}") + + set_property(GLOBAL APPEND PROPERTY SOURCEMETA_TEST_TARGETS "${TARGET_NAME}") endfunction() From ced4d8ce287566077c08da0481edacaff8d49e8b Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Fri, 7 Aug 2026 14:48:58 -0300 Subject: [PATCH 2/2] More Signed-off-by: Juan Cruz Viotti --- cmake/common/targets/coverage-report.cmake | 112 +++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/cmake/common/targets/coverage-report.cmake b/cmake/common/targets/coverage-report.cmake index bd2a66c0ff..56d45ba75f 100644 --- a/cmake/common/targets/coverage-report.cmake +++ b/cmake/common/targets/coverage-report.cmake @@ -11,6 +11,111 @@ execute_process(COMMAND "${COVERAGE_LLVM_PROFDATA}" merge -sparse COMMAND_ERROR_IS_FATAL ANY) string(REPLACE "|" ";" COVERAGE_OBJECTS "${COVERAGE_OBJECTS}") + +# Passing every binary to a single `llvm-cov` invocation misattributes +# header-inline and constexpr functions: when the compiler constant folds or +# re-hashes a definition in one translation unit, the shared record can shadow +# the executed one and report covered code as untouched. Exporting one LCOV +# trace per binary and keeping the highest execution count per line sidesteps +# the collision entirely, so a line counts as covered when any binary truly +# ran it +set(COVERAGE_LCOV_DIRECTORY "${COVERAGE_OUTPUT_DIRECTORY}/lcov") +file(REMOVE_RECURSE "${COVERAGE_LCOV_DIRECTORY}") +file(MAKE_DIRECTORY "${COVERAGE_LCOV_DIRECTORY}") +set(COVERAGE_TRACE_INDEX 0) +foreach(coverage_object IN LISTS COVERAGE_OBJECTS) + execute_process(COMMAND "${COVERAGE_LLVM_COV}" export + "${coverage_object}" + "-instr-profile=${COVERAGE_PROFILE_DATA}" + -format=lcov + "-ignore-filename-regex=${COVERAGE_EXCLUDE}" + OUTPUT_FILE "${COVERAGE_LCOV_DIRECTORY}/${COVERAGE_TRACE_INDEX}.lcov" + COMMAND_ERROR_IS_FATAL ANY) + math(EXPR COVERAGE_TRACE_INDEX "${COVERAGE_TRACE_INDEX} + 1") +endforeach() + +# Merge the traces line by line, keeping the highest count observed for every +# line and branch, then emit a merged LCOV trace plus a per file summary +set(COVERAGE_MERGE_PROGRAM "${COVERAGE_OUTPUT_DIRECTORY}/merge.awk") +file(WRITE "${COVERAGE_MERGE_PROGRAM}" [==[ +/^SF:/ { source = substr($0, 4); files[source] = 1; next } +/^DA:/ { + split(substr($0, 4), record, ",") + key = source SUBSEP record[1] + if (!(key in lines) || record[2] + 0 > lines[key] + 0) { + lines[key] = record[2] + 0 + } + next +} +/^BRDA:/ { + split(substr($0, 6), record, ",") + key = source SUBSEP record[1] SUBSEP record[2] SUBSEP record[3] + count = record[4] == "-" ? 0 : record[4] + 0 + if (!(key in branches) || count > branches[key] + 0) { + branches[key] = count + } + next +} +END { + total_lines = 0 + total_covered = 0 + total_branches = 0 + total_branches_covered = 0 + for (key in lines) { + split(key, parts, SUBSEP) + file_lines[parts[1]] += 1 + total_lines += 1 + if (lines[key] > 0) { + file_covered[parts[1]] += 1 + total_covered += 1 + } + } + for (key in branches) { + split(key, parts, SUBSEP) + file_branches[parts[1]] += 1 + total_branches += 1 + if (branches[key] > 0) { + file_branches_covered[parts[1]] += 1 + total_branches_covered += 1 + } + } + for (source in files) { + printf "SF:%s\n", source > merged + for (key in lines) { + split(key, parts, SUBSEP) + if (parts[1] == source) { + printf "DA:%s,%s\n", parts[2], lines[key] > merged + } + } + printf "end_of_record\n" > merged + covered = file_covered[source] + 0 + percentage = file_lines[source] > 0 \ + ? (covered * 100.0) / file_lines[source] : 100 + printf "%8.2f%% %6d/%-6d %s\n", percentage, covered, + file_lines[source], source | "sort -k4" + } + close("sort -k4") + line_percentage = total_lines > 0 \ + ? (total_covered * 100.0) / total_lines : 100 + branch_percentage = total_branches > 0 \ + ? (total_branches_covered * 100.0) / total_branches : 100 + printf "%8.2f%% %6d/%-6d TOTAL lines\n", line_percentage, + total_covered, total_lines + printf "%8.2f%% %6d/%-6d TOTAL branches\n", branch_percentage, + total_branches_covered, total_branches +} +]==]) + +file(GLOB COVERAGE_TRACES "${COVERAGE_LCOV_DIRECTORY}/*.lcov") +execute_process(COMMAND awk + -v "merged=${COVERAGE_OUTPUT_DIRECTORY}/coverage.lcov" + -f "${COVERAGE_MERGE_PROGRAM}" ${COVERAGE_TRACES} + OUTPUT_FILE "${COVERAGE_OUTPUT_DIRECTORY}/summary.txt" + COMMAND_ERROR_IS_FATAL ANY) + +# The browsable report keeps the combined view. Its annotated sources can +# still under count the header-inline cases described above, so the summary +# file carries the authoritative numbers list(POP_FRONT COVERAGE_OBJECTS COVERAGE_MAIN_OBJECT) set(COVERAGE_OBJECT_ARGUMENTS) foreach(coverage_object IN LISTS COVERAGE_OBJECTS) @@ -25,4 +130,11 @@ execute_process(COMMAND "${COVERAGE_LLVM_COV}" show -show-branches=count COMMAND_ERROR_IS_FATAL ANY) +file(STRINGS "${COVERAGE_OUTPUT_DIRECTORY}/summary.txt" COVERAGE_TOTALS + REGEX "TOTAL") +foreach(coverage_total IN LISTS COVERAGE_TOTALS) + message(STATUS "Coverage:${coverage_total}") +endforeach() +message(STATUS "Coverage summary: ${COVERAGE_OUTPUT_DIRECTORY}/summary.txt") +message(STATUS "Coverage trace: ${COVERAGE_OUTPUT_DIRECTORY}/coverage.lcov") message(STATUS "Coverage report: ${COVERAGE_OUTPUT_DIRECTORY}/html/index.html")