[WIP] Add a target for easily computing code coverage

CMakeLists.txt

@@ -51,6 +51,7 @@ option(SOURCEMETA_CORE_DOCS "Build the Sourcemeta Core docs" OFF)
 option(SOURCEMETA_CORE_INSTALL "Install the Sourcemeta Core library" ON)
 option(SOURCEMETA_CORE_ADDRESS_SANITIZER "Build Sourcemeta Core with an address sanitizer" OFF)
 option(SOURCEMETA_CORE_UNDEFINED_SANITIZER "Build Sourcemeta Core with an undefined behavior sanitizer" OFF)
+option(SOURCEMETA_CORE_COVERAGE "Build Sourcemeta Core with code coverage instrumentation" OFF)
 option(SOURCEMETA_CORE_CONTRIB_GOOGLEBENCHMARK "Build the GoogleBenchmark library for downstream consumers" OFF)
 
 include(Sourcemeta)
@@ -67,6 +68,11 @@ elseif(SOURCEMETA_CORE_UNDEFINED_SANITIZER)
   sourcemeta_sanitizer(TYPE undefined)
 endif()
 
+# Enable coverage instrumentation before defining any target
+if(SOURCEMETA_CORE_COVERAGE)
+  sourcemeta_coverage()
+endif()
+
 # TODO: Turn this into a re-usable utility CMake function
 if(SOURCEMETA_CORE_INSTALL)
   include(GNUInstallDirs)
@@ -430,6 +436,12 @@ if(SOURCEMETA_CORE_TESTS)
       add_subdirectory(test/packaging)
     endif()
   endif()
+
+  if(SOURCEMETA_CORE_COVERAGE)
+    sourcemeta_target_coverage(
+      OUTPUT "${PROJECT_BINARY_DIR}/coverage"
+      EXCLUDE "${PROJECT_SOURCE_DIR}/vendor" "${PROJECT_SOURCE_DIR}/test")
+  endif()
 endif()
 
 if(SOURCEMETA_CORE_TESTS_CI)

Makefile

@@ -33,6 +33,16 @@ test: .always
 benchmark: .always
 	$(CMAKE) --build ./build --config $(PRESET) --target benchmark_all
 
+coverage: .always
+	$(CMAKE) -S . -B ./build/coverage \
+		-DCMAKE_BUILD_TYPE:STRING=Debug \
+		-DCMAKE_COMPILE_WARNING_AS_ERROR:BOOL=ON \
+		-DSOURCEMETA_CORE_TESTS:BOOL=ON \
+		-DSOURCEMETA_CORE_COVERAGE:BOOL=ON \
+		-DBUILD_SHARED_LIBS:BOOL=OFF
+	$(CMAKE) --build ./build/coverage --config Debug
+	$(CMAKE) --build ./build/coverage --config Debug --target coverage
+
 doxygen: .always
 	$(CMAKE) --build ./build --config $(PRESET) --target doxygen
 

cmake/Sourcemeta.cmake

@@ -4,6 +4,7 @@ include("${SOURCEMETA_UTILITIES_DIRECTORY}/variables.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/defaults.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/simd.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/sanitizer.cmake")
+include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/coverage.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/compiler/options.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/options/enum.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/commands/copy-file.cmake")
@@ -13,6 +14,7 @@ include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/clang-format.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/shellcheck.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/doxygen.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/test.cmake")
+include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/coverage.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/targets/googlebenchmark.cmake")
 include("${SOURCEMETA_UTILITIES_DIRECTORY}/clang-tidy.cmake")
 

cmake/common/compiler/coverage.cmake

@@ -0,0 +1,10 @@
+function(sourcemeta_coverage)
+  if(SOURCEMETA_COMPILER_LLVM)
+    # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+    message(STATUS "Enabling coverage: LLVM source-based coverage")
+    add_compile_options(-fprofile-instr-generate -fcoverage-mapping)
+    add_link_options(-fprofile-instr-generate)
+  else()
+    message(FATAL_ERROR "Unrecognized compiler for coverage instrumentation")
+  endif()
+endfunction()

cmake/common/targets/coverage-report.cmake

@@ -0,0 +1,140 @@
+# Script executed by the `coverage` target after the test suite runs, as
+# the raw profile files are only known at that point
+file(GLOB COVERAGE_PROFILE_FILES "${COVERAGE_PROFILE_DIRECTORY}/*.profraw")
+if(NOT COVERAGE_PROFILE_FILES)
+  message(FATAL_ERROR "No profile data found in ${COVERAGE_PROFILE_DIRECTORY}")
+endif()
+
+set(COVERAGE_PROFILE_DATA "${COVERAGE_OUTPUT_DIRECTORY}/coverage.profdata")
+execute_process(COMMAND "${COVERAGE_LLVM_PROFDATA}" merge -sparse
+  ${COVERAGE_PROFILE_FILES} -o "${COVERAGE_PROFILE_DATA}"
+  COMMAND_ERROR_IS_FATAL ANY)
+
+string(REPLACE "|" ";" COVERAGE_OBJECTS "${COVERAGE_OBJECTS}")
+
+# Passing every binary to a single `llvm-cov` invocation misattributes
+# header-inline and constexpr functions: when the compiler constant folds or
+# re-hashes a definition in one translation unit, the shared record can shadow
+# the executed one and report covered code as untouched. Exporting one LCOV
+# trace per binary and keeping the highest execution count per line sidesteps
+# the collision entirely, so a line counts as covered when any binary truly
+# ran it
+set(COVERAGE_LCOV_DIRECTORY "${COVERAGE_OUTPUT_DIRECTORY}/lcov")
+file(REMOVE_RECURSE "${COVERAGE_LCOV_DIRECTORY}")
+file(MAKE_DIRECTORY "${COVERAGE_LCOV_DIRECTORY}")
+set(COVERAGE_TRACE_INDEX 0)
+foreach(coverage_object IN LISTS COVERAGE_OBJECTS)
+  execute_process(COMMAND "${COVERAGE_LLVM_COV}" export
+    "${coverage_object}"
+    "-instr-profile=${COVERAGE_PROFILE_DATA}"
+    -format=lcov
+    "-ignore-filename-regex=${COVERAGE_EXCLUDE}"
+    OUTPUT_FILE "${COVERAGE_LCOV_DIRECTORY}/${COVERAGE_TRACE_INDEX}.lcov"
+    COMMAND_ERROR_IS_FATAL ANY)
+  math(EXPR COVERAGE_TRACE_INDEX "${COVERAGE_TRACE_INDEX} + 1")
+endforeach()
+
+# Merge the traces line by line, keeping the highest count observed for every
+# line and branch, then emit a merged LCOV trace plus a per file summary
+set(COVERAGE_MERGE_PROGRAM "${COVERAGE_OUTPUT_DIRECTORY}/merge.awk")
+file(WRITE "${COVERAGE_MERGE_PROGRAM}" [==[
+/^SF:/ { source = substr($0, 4); files[source] = 1; next }
+/^DA:/ {
+  split(substr($0, 4), record, ",")
+  key = source SUBSEP record[1]
+  if (!(key in lines) || record[2] + 0 > lines[key] + 0) {
+    lines[key] = record[2] + 0
+  }
+  next
+}
+/^BRDA:/ {
+  split(substr($0, 6), record, ",")
+  key = source SUBSEP record[1] SUBSEP record[2] SUBSEP record[3]
+  count = record[4] == "-" ? 0 : record[4] + 0
+  if (!(key in branches) || count > branches[key] + 0) {
+    branches[key] = count
+  }
+  next
+}
+END {
+  total_lines = 0
+  total_covered = 0
+  total_branches = 0
+  total_branches_covered = 0
+  for (key in lines) {
+    split(key, parts, SUBSEP)
+    file_lines[parts[1]] += 1
+    total_lines += 1
+    if (lines[key] > 0) {
+      file_covered[parts[1]] += 1
+      total_covered += 1
+    }
+  }
+  for (key in branches) {
+    split(key, parts, SUBSEP)
+    file_branches[parts[1]] += 1
+    total_branches += 1
+    if (branches[key] > 0) {
+      file_branches_covered[parts[1]] += 1
+      total_branches_covered += 1
+    }
+  }
+  for (source in files) {
+    printf "SF:%s\n", source > merged
+    for (key in lines) {
+      split(key, parts, SUBSEP)
+      if (parts[1] == source) {
+        printf "DA:%s,%s\n", parts[2], lines[key] > merged
+      }
+    }
+    printf "end_of_record\n" > merged
+    covered = file_covered[source] + 0
+    percentage = file_lines[source] > 0 \
+      ? (covered * 100.0) / file_lines[source] : 100
+    printf "%8.2f%% %6d/%-6d %s\n", percentage, covered,
+      file_lines[source], source | "sort -k4"
+  }
+  close("sort -k4")
+  line_percentage = total_lines > 0 \
+    ? (total_covered * 100.0) / total_lines : 100
+  branch_percentage = total_branches > 0 \
+    ? (total_branches_covered * 100.0) / total_branches : 100
+  printf "%8.2f%% %6d/%-6d TOTAL lines\n", line_percentage,
+    total_covered, total_lines
+  printf "%8.2f%% %6d/%-6d TOTAL branches\n", branch_percentage,
+    total_branches_covered, total_branches
+}
+]==])
+
+file(GLOB COVERAGE_TRACES "${COVERAGE_LCOV_DIRECTORY}/*.lcov")
+execute_process(COMMAND awk
+  -v "merged=${COVERAGE_OUTPUT_DIRECTORY}/coverage.lcov"
+  -f "${COVERAGE_MERGE_PROGRAM}" ${COVERAGE_TRACES}
+  OUTPUT_FILE "${COVERAGE_OUTPUT_DIRECTORY}/summary.txt"
+  COMMAND_ERROR_IS_FATAL ANY)
+
+# The browsable report keeps the combined view. Its annotated sources can
+# still under count the header-inline cases described above, so the summary
+# file carries the authoritative numbers
+list(POP_FRONT COVERAGE_OBJECTS COVERAGE_MAIN_OBJECT)
+set(COVERAGE_OBJECT_ARGUMENTS)
+foreach(coverage_object IN LISTS COVERAGE_OBJECTS)
+  list(APPEND COVERAGE_OBJECT_ARGUMENTS -object "${coverage_object}")
+endforeach()
+
+execute_process(COMMAND "${COVERAGE_LLVM_COV}" show
+  "${COVERAGE_MAIN_OBJECT}" ${COVERAGE_OBJECT_ARGUMENTS}
+  "-instr-profile=${COVERAGE_PROFILE_DATA}"
+  -format=html "-output-dir=${COVERAGE_OUTPUT_DIRECTORY}/html"
+  "-ignore-filename-regex=${COVERAGE_EXCLUDE}"
+  -show-branches=count
+  COMMAND_ERROR_IS_FATAL ANY)
+
+file(STRINGS "${COVERAGE_OUTPUT_DIRECTORY}/summary.txt" COVERAGE_TOTALS
+  REGEX "TOTAL")
+foreach(coverage_total IN LISTS COVERAGE_TOTALS)
+  message(STATUS "Coverage:${coverage_total}")
+endforeach()
+message(STATUS "Coverage summary: ${COVERAGE_OUTPUT_DIRECTORY}/summary.txt")
+message(STATUS "Coverage trace: ${COVERAGE_OUTPUT_DIRECTORY}/coverage.lcov")
+message(STATUS "Coverage report: ${COVERAGE_OUTPUT_DIRECTORY}/html/index.html")

cmake/common/targets/coverage.cmake

@@ -0,0 +1,57 @@
+function(sourcemeta_target_coverage)
+  cmake_parse_arguments(SOURCEMETA_TARGET_COVERAGE "" "OUTPUT" "EXCLUDE" ${ARGN})
+
+  if(NOT SOURCEMETA_TARGET_COVERAGE_OUTPUT)
+    message(FATAL_ERROR "You must pass an output directory using the OUTPUT option")
+  endif()
+
+  get_property(COVERAGE_TEST_TARGETS GLOBAL PROPERTY SOURCEMETA_TEST_TARGETS)
+  if(NOT COVERAGE_TEST_TARGETS)
+    message(FATAL_ERROR "You must register tests before declaring the coverage target")
+  endif()
+
+  # On Apple platforms, the LLVM tools that understand the profile format
+  # emitted by the system compiler are only reachable through `xcrun`
+  if(APPLE)
+    execute_process(COMMAND xcrun --find llvm-profdata
+      OUTPUT_VARIABLE COVERAGE_LLVM_PROFDATA
+      OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY)
+    execute_process(COMMAND xcrun --find llvm-cov
+      OUTPUT_VARIABLE COVERAGE_LLVM_COV
+      OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY)
+  else()
+    find_program(COVERAGE_LLVM_PROFDATA llvm-profdata REQUIRED)
+    find_program(COVERAGE_LLVM_COV llvm-cov REQUIRED)
+  endif()
+
+  set(COVERAGE_OBJECTS)
+  foreach(test_target IN LISTS COVERAGE_TEST_TARGETS)
+    list(APPEND COVERAGE_OBJECTS "$<TARGET_FILE:${test_target}>")
+  endforeach()
+  # Joined with a custom separator to survive being passed as a single
+  # command line argument to the report script
+  list(JOIN COVERAGE_OBJECTS "|" COVERAGE_OBJECTS)
+
+  set(COVERAGE_EXCLUDE "${PROJECT_BINARY_DIR}")
+  list(APPEND COVERAGE_EXCLUDE ${SOURCEMETA_TARGET_COVERAGE_EXCLUDE})
+  list(JOIN COVERAGE_EXCLUDE "|" COVERAGE_EXCLUDE)
+
+  set(COVERAGE_PROFILE_DIRECTORY "${SOURCEMETA_TARGET_COVERAGE_OUTPUT}/profile")
+  add_custom_target(coverage VERBATIM USES_TERMINAL
+    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+    COMMAND "${CMAKE_COMMAND}" -E rm -R -f "${COVERAGE_PROFILE_DIRECTORY}"
+    COMMAND "${CMAKE_COMMAND}" -E make_directory "${COVERAGE_PROFILE_DIRECTORY}"
+    COMMAND "${CMAKE_COMMAND}" -E env
+      "LLVM_PROFILE_FILE=${COVERAGE_PROFILE_DIRECTORY}/%p.profraw"
+      "${CMAKE_CTEST_COMMAND}" --test-dir "${PROJECT_BINARY_DIR}"
+      --build-config $<CONFIG> --output-on-failure --parallel
+    COMMAND "${CMAKE_COMMAND}"
+      "-DCOVERAGE_LLVM_PROFDATA=${COVERAGE_LLVM_PROFDATA}"
+      "-DCOVERAGE_LLVM_COV=${COVERAGE_LLVM_COV}"
+      "-DCOVERAGE_PROFILE_DIRECTORY=${COVERAGE_PROFILE_DIRECTORY}"
+      "-DCOVERAGE_OUTPUT_DIRECTORY=${SOURCEMETA_TARGET_COVERAGE_OUTPUT}"
+      "-DCOVERAGE_OBJECTS=${COVERAGE_OBJECTS}"
+      "-DCOVERAGE_EXCLUDE=${COVERAGE_EXCLUDE}"
+      -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/coverage-report.cmake")
+  add_dependencies(coverage ${COVERAGE_TEST_TARGETS})
+endfunction()

cmake/common/targets/test.cmake

@@ -32,4 +32,6 @@ function(sourcemeta_test)
 
   add_test(NAME "${SOURCEMETA_TEST_PROJECT}.${SOURCEMETA_TEST_NAME}"
     COMMAND "${TARGET_NAME}")
+
+  set_property(GLOBAL APPEND PROPERTY SOURCEMETA_TEST_TARGETS "${TARGET_NAME}")
 endfunction()