Add code coverage for CMake builds with Clang or GCC #138
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -100,6 +100,43 @@ set(CMAKE_AUTOUIC | |
| ON | ||
| ) | ||
|
|
||
| if (PROJECT_IS_TOP_LEVEL) | ||
|
Member
There was a problem hiding this comment.
per cmake docs, that was added in cmake 3.21 and we have: so this is great! |
||
| set(MYAPP_CLANG_COVERAGE_FLAGS | ||
| " -fprofile-instr-generate \ | ||
| -fcoverage-mapping \ | ||
|
Contributor
Author
There was a problem hiding this comment. In case it's of interest, I'll mention I also tried enabling MC/DC coverage for Clang (adding compile flag
Member
There was a problem hiding this comment. Thank you for these notes. It is an interesting question as to why clang flags are getting "seen"/used during the Android build. I should consider adding some clarifying comments in various spots in the build scripts.... along these lines:
Compiling for Android necessarily involves a cross-compiler (host and target machine are different architecture). The Android compiler is determined by whatever gets downloaded in the Android SDK/NDK/"kit". Refer to tools/ci/install_android_kits.sh Apparently the SDK/NDK/"kit" provides a clang-based cross-compiler. Given that we don't have a strong need for MC/DC at this time, I think we can defer that addition for some later PR. Whenever we decide to experiment with adding MC/DC options, then I see at least two paths forward:
|
||
| " | ||
| ) | ||
| set(MYAPP_GCC_COVERAGE_FLAGS | ||
| " --coverage" | ||
| ) | ||
| if (${CMAKE_C_COMPILER_ID} STREQUAL Clang) | ||
| string( | ||
| APPEND | ||
| CMAKE_C_FLAGS_DEBUG | ||
| "${MYAPP_CLANG_COVERAGE_FLAGS}" | ||
| ) | ||
| elseif(${CMAKE_C_COMPILER_ID} STREQUAL GNU) | ||
| string( | ||
| APPEND | ||
| CMAKE_C_FLAGS_DEBUG | ||
| "${MYAPP_GCC_COVERAGE_FLAGS}" | ||
| ) | ||
| endif() | ||
| if (${CMAKE_CXX_COMPILER_ID} STREQUAL Clang) | ||
| string( | ||
| APPEND | ||
| CMAKE_CXX_FLAGS_DEBUG | ||
| "${MYAPP_CLANG_COVERAGE_FLAGS}" | ||
| ) | ||
| elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) | ||
| string( | ||
| APPEND | ||
| CMAKE_CXX_FLAGS_DEBUG | ||
| "${MYAPP_GCC_COVERAGE_FLAGS}" | ||
| ) | ||
| endif() | ||
|
Member
There was a problem hiding this comment. I have no reason to believe that these new compiler flags would interfere with the recently-added ASan flags, but just to be super confident, I also used the "Bad Code" examples from my prior PR #130 and inserted the bad code (locally only) to make sure ASan still works when code coverage is in the mix. All seems to cooperate just fine! |
||
| endif() # PROJECT_IS_TOP_LEVEL | ||
|
Member
There was a problem hiding this comment. "note on a minor improvement/clean-up": in a follow-up PR, I have moved this block of compiler settings to a new file https://github.com/219-design/qt-qml-project-template-with-ci/pull/139/files#r2599259437 |
||
|
|
||
| # ############################################################################## | ||
| # Begin section: based on the book "Professional CMake" by Craig Scott | ||
| # | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -58,6 +58,7 @@ fi | |
|
|
||
| if [[ -n ${MYAPP_TEMPLATE_PREFER_QMAKE-} ]]; then | ||
| ./build_app.sh | ||
| DEBUG_BUILD_DIR="${THISDIR}/build" | ||
| else | ||
| # Run a Release build (except where it doesn't make sense) | ||
| if [[ "$OSTYPE" != "darwin"* ]]; then | ||
|
|
@@ -70,6 +71,7 @@ else | |
| fi | ||
|
|
||
| ./build_cmake_app.sh # <-- the default invocation (no args) will build debug. | ||
| DEBUG_BUILD_DIR="${THISDIR}/cbuild" | ||
|
|
||
| if [[ -z ${MYAPP_TEMPLATE_QT5-} ]]; then | ||
| # Nice-to-have: figure out why cmake-qt6 moved `app` such that we now need this. | ||
|
|
@@ -78,11 +80,60 @@ else | |
| fi | ||
| fi | ||
|
|
||
| # Note coverage-related operations are structured to be no-ops unless the corresponding | ||
| # instrumentation is present. | ||
| COVERAGE_RUN_TIMESTAMP="$(date +%Y-%m-%d_%H%M%S_%N)" | ||
| CLANG_COVERAGE_DATA_DIR="${DEBUG_BUILD_DIR}/coverage_data/${COVERAGE_RUN_TIMESTAMP}" | ||
| export LLVM_PROFILE_FILE="${CLANG_COVERAGE_DATA_DIR}/%p.profraw" | ||
|
|
||
| # Clear any existing gcov data so it does not contribute to this run's coverage results. | ||
| find ${DEBUG_BUILD_DIR} -name "*.gcda" -o -name "*.gcov" -delete | ||
|
|
||
| process_coverage_data() { | ||
|
Member
There was a problem hiding this comment. "note on a minor improvement/clean-up": in a follow-up PR, I have moved this report-generation logic to its own separate script. In order to do that, it was necessary for me to tackle some long-overdue cleanup regarding "single source of truth" for build output folder. It is no fault of yours that isolating the report-generation was not exactly readily feasible at the time of your PR. see: https://github.com/219-design/qt-qml-project-template-with-ci/pull/139/files#r2599266472 |
||
| if [[ -d "${CLANG_COVERAGE_DATA_DIR}" ]]; then | ||
| llvm-profdata merge $(find ${CLANG_COVERAGE_DATA_DIR} -name "*.profraw") \ | ||
| -o ${CLANG_COVERAGE_DATA_DIR}/coverage.profdata | ||
| CLANG_COVERAGE_REPORTS_DIR="${THISDIR}/coverage_reports/clang-llvm/${COVERAGE_RUN_TIMESTAMP}" | ||
| mkdir -p ${CLANG_COVERAGE_REPORTS_DIR} | ||
|
|
||
| # Individually prefacing each relevant binary file with "-object" is seemingly | ||
| # unavoidable for llvm-cov. In spite of the fact it's willing to treat a single | ||
| # non-option argument as a binary file, it's not willing to do that for files | ||
| # after the first, nor is it willing to accept a directory. | ||
| LLVM_COV_BINARY_FILE_ARGS="" | ||
| for BINARY_FILE in $(find -L ${DEBUG_BUILD_DIR}/stage -type f); do | ||
| LLVM_COV_BINARY_FILE_ARGS+=" -object ${BINARY_FILE}" | ||
| done | ||
|
|
||
| llvm-cov show \ | ||
| -instr-profile ${CLANG_COVERAGE_DATA_DIR}/coverage.profdata \ | ||
| ${LLVM_COV_BINARY_FILE_ARGS} \ | ||
| -sources ${THISDIR}/src \ | ||
| -output-dir=${CLANG_COVERAGE_REPORTS_DIR} \ | ||
| -format=html \ | ||
| -show-branches=count \ | ||
| -show-line-counts-or-regions \ | ||
| -show-directory-coverage | ||
| fi | ||
| if [[ -n $(find ${DEBUG_BUILD_DIR} -name "*.gcda") ]]; then | ||
| GCC_COVERAGE_REPORTS_DIR="${THISDIR}/coverage_reports/gcc-gcov/${COVERAGE_RUN_TIMESTAMP}" | ||
| mkdir -p ${GCC_COVERAGE_REPORTS_DIR} | ||
| pushd "${DEBUG_BUILD_DIR}" | ||
| gcovr \ | ||
| --root ${THISDIR} \ | ||
| --filter "${THISDIR}/src" \ | ||
| --html-nested ${GCC_COVERAGE_REPORTS_DIR}/index.html \ | ||
| --decisions | ||
| popd # ${DEBUG_BUILD_DIR} | ||
| fi | ||
| } | ||
|
|
||
| # run all test binaries that got built in the expected dir: | ||
| tools/auto_test/run_cpp_auto_tests.sh | ||
|
|
||
| if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then | ||
| #################### EARLY EXIT FOR MICROSOFT WINDOWS. (TODO: tools/gui_test for WIN32) | ||
| process_coverage_data | ||
| echo 'EARLY EXIT FOR MICROSOFT WINDOWS. (TODO: tools/gui_test for WIN32)' | ||
| exit 0 | ||
| fi | ||
|
|
@@ -108,6 +159,8 @@ fi | |
| # run gui tests which execute the actual app binary: | ||
| tools/gui_test/launch_gui_for_display.sh "${XDISPLAY}" | ||
|
Member
There was a problem hiding this comment. I am very pleasantly surprised and delighted that when the app launches due to I suppose I shouldn't be surprised. But I got "tunnel vision" thinking coverage was somehow limited to unit tests. But of course there is nothing inherent in gcovr or llvm-cov that cause such a limitation. |
||
|
|
||
| process_coverage_data | ||
|
|
||
| if [[ -n ${MYAPP_TEMPLATE_BUILD_APPIMAGE-} ]]; then | ||
| # this MUST happen last because (on the C.I. server) it destroys folders (intentionally) | ||
| tools/gui_test/test_AppImage.sh "${XDISPLAY}" | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm delighted to see that
${{ env.our_nameid }}is available for this kind of re-use.