diff --git a/.github/workflows/reusable-module-ci.yml b/.github/workflows/reusable-module-ci.yml new file mode 100644 index 0000000..2ad2739 --- /dev/null +++ b/.github/workflows/reusable-module-ci.yml @@ -0,0 +1,423 @@ +name: Build and Test + +on: + workflow_call: + inputs: + opendaq-ref: + description: "openDAQ commit, branch or tag. If empty, uses the opendaq_ref file from the repo." + type: string + default: '' + exclude-jobs: + description: "JSON array of wildcard patterns to exclude matrix jobs, e.g. '[\"*-debug\", \"macos-*\"]'" + type: string + default: '[]' + packages: + description: 'JSON array of package requests, e.g. [{"match-jobs": ["ubuntu-*"], "apt-install": ["libpcap-dev"]}]' + type: string + default: '[]' + cmake-presets: + description: 'JSON array of preset configs, e.g. [{"match-jobs": ["*"], "configure-preset": "module", "test-preset": "module-test"}]' + type: string + default: '[]' + +jobs: + generate: + name: Generate + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Generate matrix + id: matrix + shell: python3 {0} + run: | + import json, os + from fnmatch import fnmatch + + exclude_jobs = json.loads("""${{ inputs.exclude-jobs }}""") + user_packages = json.loads("""${{ inputs.packages }}""") + + print(f"Input: exclude-jobs = {json.dumps(exclude_jobs) if exclude_jobs else 'None'}") + print(f"Input: packages = {json.dumps(user_packages) if user_packages else 'None'}") + + # --- General config --- + + _base = { + "runner": None, + "image": None, + "bootstrap": None, + "package-managers": [], + "generator": None, + "toolset": None, + "cc": None, + "cxx": None, + "cmd-sudo": None, + "cmd-apt-envs": ["DEBIAN_FRONTEND=noninteractive", "TZ=Etc/UTC"], + "cmd-apt-install": "apt-get install -y --no-install-recommends", + "cmd-pip-install": "pip install", + "cmd-brew-install": "brew install", + "cmd-choco-install": "choco install -y", + "apt-install-default": [], + "pip-install-default": [], + "brew-install-default": [], + "apt-install": [], + "brew-install": [], + "choco-install": [], + "pip-install": [], + } + + # --- Ubuntu configs --- + + _ubuntu_guest_x86_64_ninja = { + **_base, + "runner": "ubuntu-latest", + "bootstrap": "apt-get update && apt-get install -y git", + "package-managers": ["apt", "pip"], + "cmd-sudo": "", + "generator": "Ninja", + "cc": "gcc", + "cxx": "g++", + "apt-install-default": ["mono-runtime", "libmono-system-json-microsoft4.0-cil", "libmono-system-data4.0-cil", "python3-pip"], + "pip-install-default": ["cmake", "ninja"], + } + + _ubuntu_20_04_x86_64_ninja = { **_ubuntu_guest_x86_64_ninja, "image": "ubuntu:20.04" } + _ubuntu_24_04_x86_64_ninja = { **_ubuntu_guest_x86_64_ninja, "image": "ubuntu:24.04", "cmd-pip-install": "pip install --break-system-packages" } + + # --- macOS configs --- + + _macos_ninja = { + **_base, + "package-managers": ["brew", "pip"], + "brew-install-default": ["mono"], + "generator": "Ninja", + "pip-install-default": ["cmake", "ninja"] + } + + _macos_15_armv8_ninja = {**_macos_ninja, "runner": "macos-15"} + _macos_15_x86_64_ninja = {**_macos_ninja, "runner": "macos-15-intel"} + _macos_26_armv8_ninja = {**_macos_ninja, "runner": "macos-26"} + _macos_26_x86_64_ninja = {**_macos_ninja, "runner": "macos-26-intel"} + + # --- Windows configs --- + + _windows_2025_x86_64_msvs = { + **_base, + "runner": "windows-2025", + "generator": "Visual Studio 17 2022", + "package-managers": ["choco", "pip"], + "pip-install-default": ["cmake"], + } + + # --- Jobs configs --- + + jobs_cfg = { + "windows-2025-x86_64-msvs-v143": {**_windows_2025_x86_64_msvs, "toolset": "v143"}, + + "ubuntu-20.04-x86_64-ninja-gcc-7": {**_ubuntu_20_04_x86_64_ninja, "cc": "gcc-7", "cxx": "g++-7", "apt-install": ["gcc-7", "g++-7"]}, + "ubuntu-20.04-x86_64-ninja-clang-9": {**_ubuntu_20_04_x86_64_ninja, "cc": "clang-9", "cxx": "clang++-9", "apt-install": ["clang-9"]}, + "ubuntu-24.04-x86_64-ninja-gcc-14": {**_ubuntu_24_04_x86_64_ninja, "cc": "gcc-14", "cxx": "g++-14", "apt-install": ["gcc-14", "g++-14"]}, + "ubuntu-24.04-x86_64-ninja-clang-18": {**_ubuntu_24_04_x86_64_ninja, "cc": "clang-18", "cxx": "clang++-18", "apt-install": ["clang-18"]}, + + "macos-15-armv8-ninja-appleclang": {**_macos_15_armv8_ninja, "cc": "/usr/bin/clang", "cxx": "/usr/bin/clang++"}, + "macos-15-x86_64-ninja-appleclang": {**_macos_15_x86_64_ninja, "cc": "/usr/bin/clang", "cxx": "/usr/bin/clang++"}, + "macos-26-armv8-ninja-appleclang": {**_macos_26_armv8_ninja, "cc": "/usr/bin/clang", "cxx": "/usr/bin/clang++"}, + "macos-26-x86_64-ninja-appleclang": {**_macos_26_x86_64_ninja, "cc": "/usr/bin/clang", "cxx": "/usr/bin/clang++"}, + } + + # --- Generate jobs matrix --- + + def matches_any(name, patterns): + return any(fnmatch(name, p) for p in patterns) + + matrix = [] + for cfg_key, cfg in jobs_cfg.items(): + for build_type in ["Release", "Debug"]: + job_name = f"{cfg_key}-{build_type.lower()}" + + if matches_any(job_name, exclude_jobs): + print(f"Matrix job '{job_name}': skipped by 'exclude-jobs' input") + continue + + supported_pms = set(cfg.get("package-managers", [])) + + # collect user packages for this job + user_pkgs = {} + for req in user_packages: + if not matches_any(job_name, req.get("match-jobs", ["*"])): + continue + + user_install = {k: v for k, v in req.items() if k.endswith("-install") and v} + for k, v in user_install.items(): + pm = k.removesuffix("-install") + if pm not in supported_pms: + print(f"::warning::Matrix job '{job_name}': '{pm}' is not supported by this runner, skipping installation of {v}") + continue + user_pkgs.setdefault(k, []).extend(v) + + # merge and build install commands per package manager + install_cmds = {} + for pm in supported_pms: + sudo = cfg.get("cmd-sudo") or "" + envs = " ".join(cfg.get(f"cmd-{pm}-envs", [])) + + install = cfg.get(f"{pm}-install-default", []) + cfg.get(f"{pm}-install", []) + user_pkgs.get(f"{pm}-install", []) + if install: + cmd = cfg.get(f"cmd-{pm}-install", "") + if not cmd: + print(f"::error::Matrix job '{job_name}': 'cmd-{pm}-install' is not defined but packages {install} are requested") + else: + parts = [envs, cmd, *install] + if pm == "apt" and sudo: + parts = [sudo, *parts] + install_cmds[f"cmd-install-{pm}"] = " ".join(p for p in parts if p) + + matrix.append({ + "name": job_name, + "runner": cfg["runner"], + "image": cfg.get("image") or "", + "bootstrap": cfg.get("bootstrap") or "", + "generator": cfg["generator"], + "toolset": cfg.get("toolset") or "", + "cc": cfg.get("cc") or "", + "cxx": cfg.get("cxx") or "", + "build-type": build_type, + **install_cmds, + }) + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"matrix={json.dumps({'include': matrix})}\n") + + print(f"Output: matrix = {json.dumps({'include': matrix}, indent=2)}") + + - name: Generate user presets + id: presets + shell: python3 {0} + run: | + import json + from fnmatch import fnmatch + + cmake_presets = json.loads("""${{ inputs.cmake-presets }}""") + matrix = json.loads("""${{ steps.matrix.outputs.matrix }}""")["include"] + + print(f"Input: cmake-presets = {json.dumps(cmake_presets) if cmake_presets else 'None'}") + + build_dir = "build/output" + user_presets = {"version": 4, "configurePresets": [], "buildPresets": [], "testPresets": []} + + for entry in matrix: + name = entry["name"] + build_type = entry.get("build-type") or "" + generator = entry.get("generator") or "" + toolset = entry.get("toolset") or "" + cc = entry.get("cc") or "" + cxx = entry.get("cxx") or "" + + # find matching cmake-presets config + configure_preset = "" + test_preset = "" + for req in cmake_presets: + if any(fnmatch(name, p) for p in req.get("match-jobs", ["*"])): + configure_preset = req.get("configure-preset", configure_preset) + test_preset = req.get("test-preset", test_preset) + + user_presets["configurePresets"].append({ + "name": name, + **({"inherits": [configure_preset]} if configure_preset else {}), + **({"generator": generator} if generator else {}), + **({"toolset": toolset} if toolset else {}), + "binaryDir": "$" + "{sourceDir}" + f"/{build_dir}", + "cacheVariables": { + **({"CMAKE_BUILD_TYPE": build_type} if build_type else {}), + **({"CMAKE_C_COMPILER": cc} if cc else {}), + **({"CMAKE_CXX_COMPILER": cxx} if cxx else {}), + }, + }) + + user_presets["buildPresets"].append({ + "name": name, + "configurePreset": name, + **({"configuration": build_type} if build_type else {}), + }) + + if test_preset: + user_presets["testPresets"].append({ + "name": name, + "configurePreset": name, + "inherits": [test_preset], + **({"configuration": build_type} if build_type else {}), + }) + + with open("CMakeUserPresets.json", "w") as f: + json.dump(user_presets, f, indent=2) + + - name: Upload presets + uses: actions/upload-artifact@v7 + with: + name: CMakeUserPresets + path: CMakeUserPresets.json + retention-days: 7 + + - name: Job summary + if: always() + shell: bash + run: | + s() { [ "$1" = "success" ] && echo "✅" || ([ "$1" = "skipped" ] && echo "➖" || echo "❌"); } + overall="✅" + [[ "${{ steps.matrix.outcome }}" == "failure" || "${{ steps.presets.outcome }}" == "failure" ]] && overall="❌" + { + echo "
${overall} Generate job inputs" + echo "" + echo '```yaml' + echo "on:" + echo " workflow_call:" + echo " inputs:" + echo " opendaq-ref: ${{ inputs.opendaq-ref || '\"\"' }}" + echo " exclude-jobs: '${{ inputs.exclude-jobs || '[]' }}'" + echo " packages: '${{ inputs.packages || '[]' }}'" + echo " cmake-presets: '${{ inputs.cmake-presets || '[]' }}'" + echo '```' + echo "" + echo "
" + echo "" + echo "
$(s ${{ steps.matrix.outcome }}) Generate matrix (matrix.json)" + echo "" + echo '```json' + echo '${{ steps.matrix.outputs.matrix }}' | jq . 2>/dev/null || echo "Not generated" + echo "" + echo '```' + echo "" + echo "
" + echo "" + echo "
$(s ${{ steps.presets.outcome }}) Generate presets (CMakeUserPresets.json)" + echo "" + echo '```json' + cat CMakeUserPresets.json 2>/dev/null || echo "Not generated" + echo "" + echo '```' + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + + build: + needs: generate + if: ${{ needs.generate.outputs.matrix != '' }} + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + container: + image: ${{ matrix.image }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate.outputs.matrix) }} + + steps: + - name: Bootstrap + if: matrix.bootstrap != '' + run: ${{ matrix.bootstrap }} + + - uses: actions/checkout@v6 + + - name: Override opendaq_ref + if: ${{ inputs.opendaq-ref != '' }} + shell: bash + run: echo "${{ inputs.opendaq-ref }}" > opendaq_ref + + - name: Install dependencies (apt) + if: matrix.cmd-install-apt != '' + run: ${{ matrix.cmd-install-apt }} + + - name: Install dependencies (brew) + if: matrix.cmd-install-brew != '' + run: ${{ matrix.cmd-install-brew }} + + - name: Install dependencies (choco) + if: matrix.cmd-install-choco != '' + run: ${{ matrix.cmd-install-choco }} + + - name: Install dependencies (pip) + if: matrix.cmd-install-pip != '' + run: ${{ matrix.cmd-install-pip }} + + - name: Download CI presets + uses: actions/download-artifact@v8 + with: + name: CMakeUserPresets + + - name: Detect tests + id: detect-tests + shell: bash + run: | + if ctest --list-presets 2>/dev/null | grep -q '"${{ matrix.name }}"'; then + echo "preset=${{ matrix.name }}" >> $GITHUB_OUTPUT + fi + + - name: Configure + id: configure + run: cmake --preset ${{ matrix.name }} + + - name: Build + id: build + run: cmake --build --preset ${{ matrix.name }} + + - name: Setup GTest + id: setup-gtest + if: steps.detect-tests.outputs.preset != '' + shell: python3 {0} + run: | + import os + test_results_dir = os.path.join(os.environ["GITHUB_WORKSPACE"], "build", "test-results") + os.makedirs(test_results_dir, exist_ok=True) + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"test-results-dir={test_results_dir}{os.sep}\n") + + - name: Test + id: test + if: steps.detect-tests.outputs.preset != '' + env: + GTEST_OUTPUT: xml:${{ steps.setup-gtest.outputs.test-results-dir }} + run: ctest --preset ${{ steps.detect-tests.outputs.preset }} + + - name: Upload test results + if: ${{ !cancelled() && steps.detect-tests.outputs.preset != '' }} + uses: actions/upload-artifact@v7 + with: + name: test-results-${{ matrix.name }} + path: ${{ steps.setup-gtest.outputs.test-results-dir }} + retention-days: 7 + + - name: Job summary + if: always() + shell: bash + run: | + s() { [ "$1" = "success" ] && echo "✅" || ([ "$1" = "skipped" ] && echo "➖" || echo "❌"); } + overall="✅" + [[ "${{ steps.configure.outcome }}" == "failure" || "${{ steps.build.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" ]] && overall="❌" + { + echo "
${overall} ${{ matrix.name }}" + echo "" + echo '```bash' + echo "# Download preset (see Generate job summary CMakeUserPresets.json section, if expired):" + echo "gh run download ${{ github.run_id }} -n CMakeUserPresets -D ." + if [ -n "${{ inputs.opendaq-ref }}" ]; then + echo "" + echo "# Override openDAQ ref:" + echo 'echo "${{ inputs.opendaq-ref }}" > opendaq_ref' + fi + echo "" + echo "# $(s ${{ steps.configure.outcome }}) Configure:" + echo "cmake --preset ${{ matrix.name }}" + echo "" + echo "# $(s ${{ steps.build.outcome }}) Build:" + echo "cmake --build --preset ${{ matrix.name }}" + echo "" + if [ "${{ steps.test.outcome }}" = "skipped" ]; then + echo "# $(s ${{ steps.test.outcome }}) Test skipped" + else + echo "# $(s ${{ steps.test.outcome }}) Test:" + echo "ctest --preset ${{ matrix.name }}" + fi + echo '```' + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY"