From 8d67f1b9ff4e5108ece2ab05aa2d8b7c4bec442b Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Mon, 24 Nov 2025 16:52:17 -0800 Subject: [PATCH 1/7] Fix link created by cookiecutter/init_repo.sh Specifically, correct a broken relative link that was created relative to init_repo.sh's directory, not the directory where the link is being placed. --- cookiecutter/init_repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookiecutter/init_repo.sh b/cookiecutter/init_repo.sh index 0cf71a5..ebfd486 100755 --- a/cookiecutter/init_repo.sh +++ b/cookiecutter/init_repo.sh @@ -7,7 +7,7 @@ set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo THISDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ln -sf src/libstyles/imports/libstyles/images src/lib_app/qml/images +ln -sf ../../../src/libstyles/imports/libstyles/images src/lib_app/qml/images ln -sf .qmake.conf qmake.conf ln -sf .clang-format clang-format ln -sf macos dl_third_party/Qt_desktop/6.5.3/clang_64 From f311351d70bb3903084f81c258425eec69e20c11 Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Wed, 26 Nov 2025 11:14:38 -0800 Subject: [PATCH 2/7] Add cbuild/cbuild_* to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 00dd2fd..0b39ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ dl_third_party build +cbuild +cbuild_* AppImage_staging linuxdeployqt-6-x86_64.AppImage gui_test.log From d2036afef46e5198dbfdbd787aad820bdbd1c29c Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Tue, 25 Nov 2025 14:26:51 -0800 Subject: [PATCH 3/7] Add coverage dependencies to provision scripts --- tools/ci/provision.sh | 3 +++ tools/ci/provision_mac.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tools/ci/provision.sh b/tools/ci/provision.sh index a89bfb7..5198e3e 100755 --- a/tools/ci/provision.sh +++ b/tools/ci/provision.sh @@ -38,11 +38,13 @@ sudo apt-get --assume-yes install libc6:i386 libc6-dev:i386 # libstdc++-10-dev was added for the sake of clang. see: https://stackoverflow.com/q/26333823/10278 sudo apt-get --assume-yes install \ build-essential \ + clang \ clang-format-19 \ cmake \ curl \ graphviz \ g++-11 \ + gcovr \ gdb \ git \ gnupg \ @@ -74,6 +76,7 @@ sudo apt-get --assume-yes install \ libxcb-xkb1 \ libxkbcommon-x11-0 \ libxkbcommon0 \ + llvm \ mesa-common-dev \ openjdk-8-jdk \ openjdk-8-jre \ diff --git a/tools/ci/provision_mac.sh b/tools/ci/provision_mac.sh index 4207753..caf6593 100755 --- a/tools/ci/provision_mac.sh +++ b/tools/ci/provision_mac.sh @@ -18,12 +18,14 @@ brew \ --overwrite python@3.12 \ clang-format@21 \ fontconfig \ + gcovr \ gdbm \ glib \ gnu-sed \ graphviz \ grep \ libtool \ + llvm \ p7zip \ pcre \ pcre2 \ From e6f6d14a24de68b6776f4dea8884be016f0a1384 Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Tue, 25 Nov 2025 16:21:09 -0800 Subject: [PATCH 4/7] Enable coverage for debug builds with Clang or GCC --- .gitignore | 3 +++ CMakeLists.txt | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.gitignore b/.gitignore index 0b39ba4..18e791b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ aqtinstall.log # Qt Creator User File *.user + +# code coverage-related: +**/*.profraw diff --git a/CMakeLists.txt b/CMakeLists.txt index 66dfacb..65a1174 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,43 @@ set(CMAKE_AUTOUIC ON ) +if (PROJECT_IS_TOP_LEVEL) + set(MYAPP_CLANG_COVERAGE_FLAGS + " -fprofile-instr-generate \ + -fcoverage-mapping \ + " + ) + set(MYAPP_GCC_COVERAGE_FLAGS + " --coverage" + ) + if (${CMAKE_C_COMPILER_ID} STREQUAL Clang) + string( + APPEND + CMAKE_C_FLAGS_DEBUG + "${MYAPP_CLANG_COVERAGE_FLAGS}" + ) + elseif(${CMAKE_C_COMPILER_ID} STREQUAL GNU) + string( + APPEND + CMAKE_C_FLAGS_DEBUG + "${MYAPP_GCC_COVERAGE_FLAGS}" + ) + endif() + if (${CMAKE_CXX_COMPILER_ID} STREQUAL Clang) + string( + APPEND + CMAKE_CXX_FLAGS_DEBUG + "${MYAPP_CLANG_COVERAGE_FLAGS}" + ) + elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) + string( + APPEND + CMAKE_CXX_FLAGS_DEBUG + "${MYAPP_GCC_COVERAGE_FLAGS}" + ) + endif() +endif() # PROJECT_IS_TOP_LEVEL + # ############################################################################## # Begin section: based on the book "Professional CMake" by Craig Scott # From 333a7801f2a378d7374e86d9b6305bc59e90f820 Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Tue, 25 Nov 2025 17:52:05 -0800 Subject: [PATCH 5/7] Extend run_all_tests.sh to create coverage reports --- .gitignore | 1 + README.md | 1 + index.md | 11 ++++++++++ run_all_tests.sh | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+) diff --git a/.gitignore b/.gitignore index 18e791b..e168b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ aqtinstall.log # code coverage-related: **/*.profraw +coverage_reports diff --git a/README.md b/README.md index 9ef5ca1..2e9f5ea 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ such as: - runtime "flexible asserts" to ensure you do not miss any QML runtime warnings - a wrapper script to make iterating on QML layouts painless with qmlscene - test-runner code to run any unit tests you add. (googletest is provided) + - instrumentation to check code coverage from your tests and generate reports - a basic '.github/workflow' to run tests on github for each commit - a basic GUI Test that launches the app in CI using Xvfb - automated Linux deployment via AppImage folder diff --git a/index.md b/index.md index 2ba807f..3ed7b5b 100644 --- a/index.md +++ b/index.md @@ -43,6 +43,17 @@ This involves five key steps: - [Generate compile\_commands.json during CMake build](https://github.com/219-design/qt-qml-project-template-with-ci/blob/b0abc1b/build_cmake_app.sh#L95-L118) - [Publish compile\_commands.json as a CI build artifact](https://github.com/219-design/qt-qml-project-template-with-ci/blob/4b02a48/.github/workflows/cmakebuild_linux.yml#L54-L67) +### Check Code Coverage + +- Enable compiler code coverage instrumentation and generate `.html` code coverage reports (for some build types). + - Building with coverage instrumentation is supported (and enabled automatically) for CMake debug builds with either Clang or GCC compilers. + - The GCC case uses [`gcov`](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html) for coverage and [`gcovr`](https://gcovr.com/) for report generation. + - The Clang case uses [Clang Source-based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html) for coverage and report generation. + - We use Clang's native "source-based" coverage to give an example of using it and for a second angle on coverage, but note Clang advertises some "`gcov`-compatibility" options that might be worth exploring in multi-compiler situations where sharing coverage-processing code and report formats is preferred to having more perspectives. + - `run_all_tests.sh` generates `.html` code coverage reports for those supported build types. + - These reports are put in sub-directories with timestamp names, created within `coverage_reports/clang-llvm` and `coverage_reports/gcc-gcov` for Clang and GCC builds, respectively. + - Within each timestamp-named sub-directory, you can open `index.html` to view the report summary. Following hyperlinks from there, you can find per-directory reports and per-file reports that show exactly which lines are covered. + ## Leverage Tools For Finding Warnings/Errors/UB/Bugs - [Enable ASan in CMake Debug Build](https://github.com/219-design/qt-qml-project-template-with-ci/blob/b0abc1b/cmake_include/unix_settings.cmake#L44-L80) diff --git a/run_all_tests.sh b/run_all_tests.sh index afc0ff5..7c4b630 100755 --- a/run_all_tests.sh +++ b/run_all_tests.sh @@ -58,6 +58,7 @@ fi if [[ -n ${MYAPP_TEMPLATE_PREFER_QMAKE-} ]]; then ./build_app.sh + DEBUG_BUILD_DIR="${THISDIR}/build" else # Run a Release build (except where it doesn't make sense) if [[ "$OSTYPE" != "darwin"* ]]; then @@ -70,6 +71,7 @@ else fi ./build_cmake_app.sh # <-- the default invocation (no args) will build debug. + DEBUG_BUILD_DIR="${THISDIR}/cbuild" if [[ -z ${MYAPP_TEMPLATE_QT5-} ]]; then # Nice-to-have: figure out why cmake-qt6 moved `app` such that we now need this. @@ -78,11 +80,60 @@ else fi fi +# Note coverage-related operations are structured to be no-ops unless the corresponding +# instrumentation is present. +COVERAGE_RUN_TIMESTAMP="$(date +%Y-%m-%d_%H%M%S_%N)" +CLANG_COVERAGE_DATA_DIR="${DEBUG_BUILD_DIR}/coverage_data/${COVERAGE_RUN_TIMESTAMP}" +export LLVM_PROFILE_FILE="${CLANG_COVERAGE_DATA_DIR}/%p.profraw" + +# Clear any existing gcov data so it does not contribute to this run's coverage results. +find ${DEBUG_BUILD_DIR} -name "*.gcda" -o -name "*.gcov" -delete + +process_coverage_data() { + if [[ -d "${CLANG_COVERAGE_DATA_DIR}" ]]; then + llvm-profdata merge $(find ${CLANG_COVERAGE_DATA_DIR} -name "*.profraw") \ + -o ${CLANG_COVERAGE_DATA_DIR}/coverage.profdata + CLANG_COVERAGE_REPORTS_DIR="${THISDIR}/coverage_reports/clang-llvm/${COVERAGE_RUN_TIMESTAMP}" + mkdir -p ${CLANG_COVERAGE_REPORTS_DIR} + + # Individually prefacing each relevant binary file with "-object" is seemingly + # unavoidable for llvm-cov. In spite of the fact it's willing to treat a single + # non-option argument as a binary file, it's not willing to do that for files + # after the first, nor is it willing to accept a directory. + LLVM_COV_BINARY_FILE_ARGS="" + for BINARY_FILE in $(find -L ${DEBUG_BUILD_DIR}/stage -type f); do + LLVM_COV_BINARY_FILE_ARGS+=" -object ${BINARY_FILE}" + done + + llvm-cov show \ + -instr-profile ${CLANG_COVERAGE_DATA_DIR}/coverage.profdata \ + ${LLVM_COV_BINARY_FILE_ARGS} \ + -sources ${THISDIR}/src \ + -output-dir=${CLANG_COVERAGE_REPORTS_DIR} \ + -format=html \ + -show-branches=count \ + -show-line-counts-or-regions \ + -show-directory-coverage + fi + if [[ -n $(find ${DEBUG_BUILD_DIR} -name "*.gcda") ]]; then + GCC_COVERAGE_REPORTS_DIR="${THISDIR}/coverage_reports/gcc-gcov/${COVERAGE_RUN_TIMESTAMP}" + mkdir -p ${GCC_COVERAGE_REPORTS_DIR} + pushd "${DEBUG_BUILD_DIR}" + gcovr \ + --root ${THISDIR} \ + --filter "${THISDIR}/src" \ + --html-nested ${GCC_COVERAGE_REPORTS_DIR}/index.html \ + --decisions + popd # ${DEBUG_BUILD_DIR} + fi +} + # run all test binaries that got built in the expected dir: tools/auto_test/run_cpp_auto_tests.sh if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then #################### EARLY EXIT FOR MICROSOFT WINDOWS. (TODO: tools/gui_test for WIN32) + process_coverage_data echo 'EARLY EXIT FOR MICROSOFT WINDOWS. (TODO: tools/gui_test for WIN32)' exit 0 fi @@ -108,6 +159,8 @@ fi # run gui tests which execute the actual app binary: tools/gui_test/launch_gui_for_display.sh "${XDISPLAY}" +process_coverage_data + if [[ -n ${MYAPP_TEMPLATE_BUILD_APPIMAGE-} ]]; then # this MUST happen last because (on the C.I. server) it destroys folders (intentionally) tools/gui_test/test_AppImage.sh "${XDISPLAY}" From 699819e859fe3251bd15d63add08e19aa52bf282 Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Wed, 26 Nov 2025 13:53:05 -0800 Subject: [PATCH 6/7] Upload code coverage reports as a CI artifact --- .github/workflows/cmakebuild_linux.yml | 33 ++++++++++++++++++++++++++ .github/workflows/main.yml | 15 ++++++++++++ index.md | 1 + 3 files changed, 49 insertions(+) diff --git a/.github/workflows/cmakebuild_linux.yml b/.github/workflows/cmakebuild_linux.yml index ef5b0a7..965b95a 100644 --- a/.github/workflows/cmakebuild_linux.yml +++ b/.github/workflows/cmakebuild_linux.yml @@ -39,11 +39,35 @@ jobs: MYAPP_TEMPLATE_COMPILERCHOICE_CLANG: 1 run: ./run_all_tests.sh + - name: upload_clang_qt5_coverage_reports + uses: actions/upload-artifact@v4 + with: + name: "${{ env.our_nameid }}.clang.qt5.coverage_reports" + path: coverage_reports/ + if-no-files-found: error + retention-days: 7 + overwrite: false + + - name: clear_clang_qt5_coverage_reports + run: rm -rf coverage_reports + - name: run_all_tests_qt6_android env: MYAPP_TEMPLATE_BUILD_ANDROID: 1 run: ./run_all_tests.sh + - name: upload_qt6_android_coverage_reports + uses: actions/upload-artifact@v4 + with: + name: "${{ env.our_nameid }}.gcc.qt6.android.coverage_reports" + path: coverage_reports/ + if-no-files-found: error + retention-days: 7 + overwrite: false + + - name: clear_qt6_android_coverage_reports + run: rm -rf coverage_reports + # Clang Qt6 build is the LAST one, so the artifact upload will have its commands: - name: run_all_tests_clang env: @@ -65,3 +89,12 @@ jobs: retention-days: 7 # If overwrite==false, the action will fail if an artifact for the given name already exists. overwrite: false + + - name: upload_clang_coverage_reports + uses: actions/upload-artifact@v4 + with: + name: "${{ env.our_nameid }}.clang.qt6.coverage_reports" + path: coverage_reports/ + if-no-files-found: error + retention-days: 7 + overwrite: false diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 84b54eb..97d4007 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,6 +17,12 @@ jobs: steps: - uses: actions/checkout@v1 + - name: compute nameid for this run + env: + nameidinput: "${{ github.head_ref || github.ref_name }}" + run: | + echo "our_nameid=${nameidinput//\//-}" >> $GITHUB_ENV + # Remove apt repos that are known to break from time to time # See https://github.com/actions/virtual-environments/issues/323 - name: Remove broken apt repos @@ -35,6 +41,15 @@ jobs: # For a clang (not gcc) qt6 build, see: cmakebuild_linux.yml run: ./run_all_tests.sh + - name: upload_gcc_coverage_reports + uses: actions/upload-artifact@v4 + with: + name: "${{ env.our_nameid }}.gcc.qt6.coverage_reports" + path: coverage_reports/ + if-no-files-found: error + retention-days: 7 + overwrite: false + # This one must go last, because it does the AppImage tests. - name: run_all_tests_gcc_qt5 env: diff --git a/index.md b/index.md index 3ed7b5b..3d7bf51 100644 --- a/index.md +++ b/index.md @@ -53,6 +53,7 @@ This involves five key steps: - `run_all_tests.sh` generates `.html` code coverage reports for those supported build types. - These reports are put in sub-directories with timestamp names, created within `coverage_reports/clang-llvm` and `coverage_reports/gcc-gcov` for Clang and GCC builds, respectively. - Within each timestamp-named sub-directory, you can open `index.html` to view the report summary. Following hyperlinks from there, you can find per-directory reports and per-file reports that show exactly which lines are covered. + - For each supported `run_all_tests.sh` run in CI, the generated coverage reports are saved as an artifact. ## Leverage Tools For Finding Warnings/Errors/UB/Bugs From 69cc91abdfe221e5dd8f101baf3bece888c6170e Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Tue, 2 Dec 2025 09:29:13 -0800 Subject: [PATCH 7/7] Add links to coverage section of index.md --- index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.md b/index.md index 3d7bf51..53361ab 100644 --- a/index.md +++ b/index.md @@ -46,14 +46,14 @@ This involves five key steps: ### Check Code Coverage - Enable compiler code coverage instrumentation and generate `.html` code coverage reports (for some build types). - - Building with coverage instrumentation is supported (and enabled automatically) for CMake debug builds with either Clang or GCC compilers. + - Building with coverage instrumentation is [supported (and enabled automatically) for CMake debug builds with either Clang or GCC compilers](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/CMakeLists.txt#L103). - The GCC case uses [`gcov`](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html) for coverage and [`gcovr`](https://gcovr.com/) for report generation. - The Clang case uses [Clang Source-based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html) for coverage and report generation. - We use Clang's native "source-based" coverage to give an example of using it and for a second angle on coverage, but note Clang advertises some "`gcov`-compatibility" options that might be worth exploring in multi-compiler situations where sharing coverage-processing code and report formats is preferred to having more perspectives. - - `run_all_tests.sh` generates `.html` code coverage reports for those supported build types. + - `run_all_tests.sh` [generates `.html` code coverage reports for those supported build types](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/run_all_tests.sh#L92). - These reports are put in sub-directories with timestamp names, created within `coverage_reports/clang-llvm` and `coverage_reports/gcc-gcov` for Clang and GCC builds, respectively. - Within each timestamp-named sub-directory, you can open `index.html` to view the report summary. Following hyperlinks from there, you can find per-directory reports and per-file reports that show exactly which lines are covered. - - For each supported `run_all_tests.sh` run in CI, the generated coverage reports are saved as an artifact. + - For each supported `run_all_tests.sh` run in CI, [the generated](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/.github/workflows/cmakebuild_linux.yml#L42) [coverage reports](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/.github/workflows/cmakebuild_linux.yml#L59) [are saved](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/.github/workflows/cmakebuild_linux.yml#L93) [as an artifact](https://github.com/219-design/qt-qml-project-template-with-ci/blob/699819e859fe3251bd15d63add08e19aa52bf282/.github/workflows/main.yml#L44). ## Leverage Tools For Finding Warnings/Errors/UB/Bugs