Feature/modules setup

.github/actions/get-shards/action.yml

@@ -0,0 +1,80 @@
+name: "Get number of shards"
+description: "Get the number of nf-test shards for the current CI job"
+inputs:
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: false
+    default: ""
+  max_shards:
+    description: "Maximum number of shards allowed"
+    required: true
+  paths:
+    description: "Component paths to test"
+    required: false
+outputs:
+  shard:
+    description: "Array of shard numbers"
+    value: ${{ steps.shards.outputs.shard }}
+  total_shards:
+    description: "Total number of shards"
+    value: ${{ steps.shards.outputs.total_shards }}
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        distribution: "temurin"
+        java-version: "17"
+
+    - name: Set up nf-test
+      uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2
+      with:
+        version: ${{ env.NFT_VER }}
+        install-fast-diff: true
+
+    - name: Get number of shards
+      id: shards
+      shell: bash
+      run: |
+        # Prepare tag parameter if tags are provided
+        TAGS=$([ -n "${{ inputs.tags }}" ] && echo "--tag ${{ inputs.tags }}" || echo "")
+
+        # Run nf-test with dynamic parameter
+        nftest_output=$(nf-test test \
+          --dry-run \
+          --profile singularity \
+          ${TAGS} \
+          --filter process,workflow \
+          ${{ inputs.paths }}) || {
+            echo "nf-test command failed with exit code $?"
+            echo "Full output: $nftest_output"
+            exit 1
+        }
+        echo "nf-test dry-run output: $nftest_output"
+
+        # Default values for shard and total_shards
+        shard="[]"
+        total_shards=0
+
+        # Check if there are related tests
+        if echo "$nftest_output" | grep -q 'No tests to execute'; then
+          echo "No related tests found."
+        else
+          # Extract the number of related tests
+          number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
+          if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
+            shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
+            shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
+            total_shards="$shards_to_run"
+          else
+            echo "Unexpected output format. Falling back to default values."
+          fi
+        fi
+
+        # Write to GitHub Actions outputs
+        echo "shard=$shard" >> $GITHUB_OUTPUT
+        echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
+
+        # Debugging output
+        echo "Final shard array: $shard"
+        echo "Total number of shards: $total_shards"

.github/actions/nf-test-action/action.yml

@@ -0,0 +1,110 @@
+name: "nf-test Action"
+description: "Runs nf-test with common setup steps (singularity only)"
+inputs:
+  profile:
+    description: "Profile to use"
+    required: true
+  shard:
+    description: "Shard number for this CI job"
+    required: true
+  total_shards:
+    description: "Total number of test shards(NOT the total number of matrix jobs)"
+    required: true
+  paths:
+    description: "Test paths"
+    required: true
+  tags:
+    description: "Tag to pass as argument for nf-test --tag parameter"
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        distribution: "temurin"
+        java-version: "17"
+
+    - name: Set up Nextflow
+      uses: nf-core/setup-nextflow@b4ec1bc7c16a94435159de94a05253542fddf6ef # v3
+      with:
+        version: "${{ env.NXF_VER }}"
+
+    - name: Set up Python
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+      with:
+        python-version: "3.14"
+
+    - name: Set up nf-test
+      uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2
+      with:
+        version: "${{ env.NFT_VER }}"
+        install-fast-diff: true
+
+    - name: Set up apptainer
+      shell: bash
+      run: |
+        sudo apt update
+        sudo apt install -y software-properties-common
+        sudo add-apt-repository -y ppa:apptainer/ppa
+        sudo apt update
+        sudo apt install -y apptainer apptainer-suid
+        mkdir -p $NXF_SINGULARITY_CACHEDIR
+        mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+    - name: Run nf-test
+      id: run-nf-test
+      shell: bash
+      run: |
+        TAGS=$([ -n "${{ inputs.tags }}" ] && echo "--tag ${{ inputs.tags }}" || echo "")
+
+        nf-test test \
+          --profile=${{ inputs.profile }} \
+          --tap=test.tap \
+          --verbose \
+          --ci \
+          --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
+          --filter process,workflow \
+          $TAGS \
+          ${{ inputs.paths }}
+
+          # Save the absolute path of the test.tap file to the output
+          echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT
+
+    - name: Generate test summary
+      if: ${{ failure() && steps.run-nf-test.conclusion == 'failure' }}
+      shell: bash
+      run: |
+        # Add header if it doesn't exist (using a token file to track this)
+        if [ ! -f ".summary_header" ]; then
+          echo "# 🚀 nf-test Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY
+          echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY
+          touch .summary_header
+        fi
+
+        if [ -f test.tap ]; then
+          while IFS= read -r line; do
+            if [[ $line =~ ^ok ]]; then
+              test_name="${line#ok }"
+              # Remove the test number from the beginning
+              test_name="${test_name#* }"
+              echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+            elif [[ $line =~ ^not\ ok ]]; then
+              test_name="${line#not ok }"
+              # Remove the test number from the beginning
+              test_name="${test_name#* }"
+              echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+            fi
+          done < test.tap
+        else
+          echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+        fi
+
+    - name: Clean up
+      if: always()
+      shell: bash
+      run: |
+        sudo rm -rf /home/ubuntu/tests/

.github/workflows/lint.yml

@@ -0,0 +1,174 @@
+name: Run Linting
+on:
+  pull_request:
+    branches: [main, develop, release/*, feature/*]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  NXF_VER: "25.10.2"
+  NXF_SYNTAX_PARSER: "v2"
+
+jobs:
+  ###################
+  # nf-core linting #
+  ###################
+  nf-core-changes:
+    name: nf-core-changes
+    runs-on: ubuntu-latest
+
+    outputs:
+      # https://github.com/dorny/paths-filter?tab=readme-ov-file#custom-processing-of-changed-files
+      modules: ${{ steps.filter.outputs.modules }}
+      modules_files: ${{ steps.module_names.outputs.result }}
+      subworkflows: ${{ steps.filter.outputs.subworkflows }}
+      subworkflows_files: ${{ steps.subworkflow_names.outputs.result }}
+    steps:
+      - name: Clean workspace
+        run: |
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.* || true
+
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          fetch-depth: 2 # To retrieve the preceding commit.
+
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4
+        id: filter
+        with:
+          filters: |
+            modules:
+              - added|modified: 'modules/UMCUGenetics/**'
+            subworkflows:
+              - added|modified: 'subworkflows/UMCUGenetics/**'
+          token: ""
+          list-files: "json"
+
+      - name: Get module name
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        id: module_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.modules_files }}
+              .filter(x => x.endsWith('main.nf') || x.endsWith('.nf.test.snap'))
+              .map(path => path
+                .replace('modules/UMCUGenetics/', '')
+                .replace(/\/(main\.nf|tests\/.*)$/, '')
+              )
+            )];
+      - name: Get subworkflow name
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        id: subworkflow_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.subworkflows_files }}
+              .filter(x => x.endsWith('main.nf') || x.endsWith('.nf.test.snap'))
+              .map(path => path
+                .replace('subworkflows/UMCUGenetics/', '')
+                .replace(/\/(main\.nf|tests\/.*)$/, '')
+              )
+            )];
+
+      - name: debug
+        run: |
+          echo ${{ steps.filter.outputs.modules_files }}
+          echo ${{ steps.module_names.outputs.result }}
+          echo ${{ steps.filter.outputs.subworkflows_files }}
+          echo ${{ steps.subworkflow_names.outputs.result }}
+
+  nf-core-lint-modules:
+    runs-on: ubuntu-latest
+    name: nf-core lint modules
+    needs: nf-core-changes
+    if: ${{ needs.nf-core-changes.outputs.modules_files != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.nf-core-changes.outputs.modules_files || '[]') }}
+    steps:
+      - name: Clean workspace
+        run: |
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.* || true
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: "3.14"
+
+      - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        id: cache-pip
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Set up Nextflow
+        uses: nf-core/setup-nextflow@v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/UMCUGenetics/nf-core-tools.git
+
+      - name: Lint module ${{ matrix.module }}
+        run: nf-core modules lint ${{ matrix.module }}
+
+  nf-core-lint-subworkflows:
+    runs-on: ubuntu-latest
+    name: nf-core lint subworkflows
+    needs: nf-core-changes
+    if: ${{ needs.nf-core-changes.outputs.subworkflows_files != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        subworkflow: ${{ fromJson(needs.nf-core-changes.outputs.subworkflows_files || '[]') }}
+    steps:
+      - name: Clean workspace
+        run: |
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.* || true
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: "3.14"
+
+      - name: Set up Nextflow
+        uses: nf-core/setup-nextflow@v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/UMCUGenetics/nf-core-tools.git
+
+      - name: Lint subworkflow ${{ matrix.subworkflow }}
+        run: nf-core subworkflows lint ${{ matrix.subworkflow }}
+
+  confirm-pass-lint:
+    runs-on: ubuntu-latest
+    needs: [nf-core-lint-modules, nf-core-lint-subworkflows]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"