From 26cedce9e6fb631d286c9343478fe47494e18bca Mon Sep 17 00:00:00 2001 From: Lukasz Lenart Date: Wed, 14 Oct 2026 10:38:30 +0200 Subject: [PATCH] build(ci): detect changes in-job and cancel superseded PR runs The change detection ran as a job of its own that every build job needed, so each build queued twice on the shared ASF runner pool: once for the two-second detection job, then again for its own runner. Over the last ~40 runs that first wait averaged 37 minutes (max 97) before a build job could even enter the queue. Detection now runs as a step at the start of each job, from a local composite action, and gates the remaining steps. Each job queues once. Every workflow also gets a concurrency group: a new push to a pull request cancels the superseded run, while pushes get a group per run so no branch build is dropped. Co-Authored-By: Claude Opus 5.5 --- .github/actions/detect-changes/action.yml | 94 +++++++++++++++++++++++ .github/workflows/changes.yml | 93 ---------------------- .github/workflows/codeql.yml | 22 ++++-- .github/workflows/maven.yml | 21 ++--- .github/workflows/owasp.yml | 26 ++++--- .github/workflows/sonar.yml | 19 +++-- Jenkinsfile | 2 +- 7 files changed, 149 insertions(+), 128 deletions(-) create mode 100644 .github/actions/detect-changes/action.yml delete mode 100644 .github/workflows/changes.yml diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 0000000000..efea4fe804 --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Decides whether a change needs a build. The one place on GitHub that knows +# which paths do not: .claude/ and the root process documents. Workflows run +# this as a step instead of filtering by path in their triggers, so the list is +# not repeated per workflow and a required check still reports (see maven.yml). +# The Jenkinsfile carries the same pattern for the ASF Jenkins build; keep +# the two in step. + +name: Detect changes +description: "Sets code=true when the change touches anything outside .claude/ and the root process docs" + +inputs: + token: + description: Token used to list the changed files + default: ${{ github.token }} + +outputs: + code: + description: "'true' when the change touches anything outside .claude/ and the root process docs" + value: ${{ steps.filter.outputs.code }} + +runs: + using: composite + steps: + - name: Check which paths the change touches + id: filter + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + run: | + set -eu + # Anything not matched here needs a build. Every pattern is anchored + # to the repository root: a nested file of the same name still builds. + skip='^(\.claude/|CLAUDE\.md$|AGENTS\.md$|SECURITY\.md$|THREAT_MODEL\.md$|$)' + # When the file list cannot be fetched, build: the required check + # must still report, and building is the safe direction. + case "${{ github.event_name }}" in + pull_request) + files=$(gh api --paginate \ + "repos/${{ github.repository }}/pulls/${{ github.event.number }}/files" \ + --jq '.[].filename') || { + echo "Listing the pull request files failed - building." + echo "code=true" >> "$GITHUB_OUTPUT" + exit 0 + } + ;; + push) + # A new branch has no "before" to compare against. The compare API + # lists at most 300 files; a push that large is not docs-only. + before="${{ github.event.before }}" + if [ "$before" = "0000000000000000000000000000000000000000" ]; then + echo "New branch - building." + echo "code=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + files=$(gh api \ + "repos/${{ github.repository }}/compare/${before}...${{ github.sha }}" \ + --jq '.files[].filename') || { + echo "Compare failed - building." + echo "code=true" >> "$GITHUB_OUTPUT" + exit 0 + } + ;; + *) + echo "Event ${{ github.event_name }} - building." + echo "code=true" >> "$GITHUB_OUTPUT" + exit 0 + ;; + esac + echo "Changed files:" + printf '%s\n' "$files" + # Tested by emptiness rather than with `grep -qv`, whose exit status + # is not reliable across grep implementations. + outside=$(printf '%s\n' "$files" | grep -vE "$skip" || true) + if [ -n "$outside" ]; then + echo "code=true" >> "$GITHUB_OUTPUT" + else + echo "Only .claude/ or the process docs changed - skipping the build." + echo "code=false" >> "$GITHUB_OUTPUT" + fi diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml deleted file mode 100644 index f6faebd1a5..0000000000 --- a/.github/workflows/changes.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Decides whether a change needs a build. The one place on GitHub that knows -# which paths do not: .claude/ and the root process documents. Workflows call -# this instead of filtering by path in their triggers, so the list is not -# repeated per workflow and a required check still reports (see maven.yml). -# The Jenkinsfile carries the same pattern for the ASF Jenkins build; keep -# the two in step. - -name: Detect changes - -on: - workflow_call: - outputs: - code: - description: "'true' when the change touches anything outside .claude/ and the root process docs" - value: ${{ jobs.changes.outputs.code }} - -jobs: - changes: - name: Detect changes outside .claude and the process docs - runs-on: ubuntu-latest - outputs: - code: ${{ steps.filter.outputs.code }} - steps: - - name: Check which paths the change touches - id: filter - env: - GH_TOKEN: ${{ github.token }} - run: | - set -eu - # Anything not matched here needs a build. Every pattern is anchored - # to the repository root: a nested file of the same name still builds. - skip='^(\.claude/|CLAUDE\.md$|AGENTS\.md$|SECURITY\.md$|THREAT_MODEL\.md$|$)' - # When the file list cannot be fetched, build: the required check - # must still report, and building is the safe direction. - case "${{ github.event_name }}" in - pull_request) - files=$(gh api --paginate \ - "repos/${{ github.repository }}/pulls/${{ github.event.number }}/files" \ - --jq '.[].filename') || { - echo "Listing the pull request files failed - building." - echo "code=true" >> "$GITHUB_OUTPUT" - exit 0 - } - ;; - push) - # A new branch has no "before" to compare against. The compare API - # lists at most 300 files; a push that large is not docs-only. - before="${{ github.event.before }}" - if [ "$before" = "0000000000000000000000000000000000000000" ]; then - echo "New branch - building." - echo "code=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - files=$(gh api \ - "repos/${{ github.repository }}/compare/${before}...${{ github.sha }}" \ - --jq '.files[].filename') || { - echo "Compare failed - building." - echo "code=true" >> "$GITHUB_OUTPUT" - exit 0 - } - ;; - *) - echo "Event ${{ github.event_name }} - building." - echo "code=true" >> "$GITHUB_OUTPUT" - exit 0 - ;; - esac - echo "Changed files:" - printf '%s\n' "$files" - # Tested by emptiness rather than with `grep -qv`, whose exit status - # is not reliable across grep implementations. - outside=$(printf '%s\n' "$files" | grep -vE "$skip" || true) - if [ -n "$outside" ]; then - echo "code=true" >> "$GITHUB_OUTPUT" - else - echo "Only .claude/ or the process docs changed - skipping the build." - echo "code=false" >> "$GITHUB_OUTPUT" - fi diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7355d7d89a..c66dc24ad7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -28,21 +28,20 @@ on: permissions: contents: read -jobs: - changes: - uses: ./.github/workflows/changes.yml - permissions: - contents: read - pull-requests: read +# Pull requests cancel superseded runs; pushes get unique groups so none is dropped. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: true +jobs: analyze: name: Analyze - needs: changes - if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest permissions: actions: read contents: read + # Read by the detect-changes action to list the pull request files. + pull-requests: read # Needed to upload the results to the code-scanning dashboard. security-events: write strategy: @@ -52,19 +51,26 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + - name: Detect changes + id: changes + uses: ./.github/actions/detect-changes - name: Setup Java JDK + if: steps.changes.outputs.code == 'true' uses: actions/setup-java@v6 with: distribution: temurin java-version: 17 cache: 'maven' - name: Initialize CodeQL + if: steps.changes.outputs.code == 'true' uses: github/codeql-action/init@v4.38.2 with: languages: ${{ matrix.language }} - name: Autobuild + if: steps.changes.outputs.code == 'true' uses: github/codeql-action/autobuild@v4.38.2 - name: Perform CodeQL Analysis + if: steps.changes.outputs.code == 'true' uses: github/codeql-action/analyze@v4.38.2 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3575dd2ffc..b00c01cfa0 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -20,7 +20,7 @@ on: # status check in .asf.yaml, and a workflow skipped by path filtering never # reports its checks - they stay Pending and the pull request can never be # merged. The build job always runs instead, and its steps are skipped by - # condition when changes.yml says nothing needs building. + # condition when the detect-changes action says nothing needs building. pull_request: push: branches: @@ -33,17 +33,18 @@ on: permissions: read-all +# Pull requests cancel superseded runs; pushes get unique groups so none is dropped. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: MAVEN_OPTS: -Xmx2048m -Xms1024m LANG: en_US.utf8 jobs: - changes: - uses: ./.github/workflows/changes.yml - build: name: Build and Test (JDK ${{ matrix.java }})${{ matrix.profile == '-Pjakartaee11' && ' (Jakarta EE 11 + Spring 7)' || matrix.profile }} - needs: changes # No job-level `if:` here on purpose. A matrix job whose condition is false # is skipped *before* the matrix expands, so it reports a single check run # named after the raw `${{ matrix.* }}` template - never "Build and Test @@ -67,23 +68,25 @@ jobs: profile: '-Pjakartaee11' steps: - name: Checkout code - if: needs.changes.outputs.code == 'true' uses: actions/checkout@v7 + - name: Detect changes + id: changes + uses: ./.github/actions/detect-changes - name: Setup Java ${{ matrix.java }} - if: needs.changes.outputs.code == 'true' + if: steps.changes.outputs.code == 'true' uses: actions/setup-java@v6 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Maven Verify on Java ${{ matrix.java }}${{ matrix.profile == '-Pjakartaee11' && ' (Jakarta EE 11 + Spring 7)' || matrix.profile }} - if: needs.changes.outputs.code == 'true' + if: steps.changes.outputs.code == 'true' run: mvn -B -V -DskipAssembly verify ${{ matrix.profile }} --no-transfer-progress - name: Test Summary ${{ matrix.java }} ${{ matrix.profile }} uses: mikepenz/action-junit-report@a9170d5795813c01ab4901ffb045b52bab4ab09d #v6.5.0 continue-on-error: true - if: always() && needs.changes.outputs.code == 'true' + if: always() && steps.changes.outputs.code == 'true' with: annotate_only: true # forked repo cannot write to checks so just do annotations report_paths: | diff --git a/.github/workflows/owasp.yml b/.github/workflows/owasp.yml index 2dde9deb95..34c981bc5e 100644 --- a/.github/workflows/owasp.yml +++ b/.github/workflows/owasp.yml @@ -27,18 +27,18 @@ on: permissions: read-all +# Pull requests cancel superseded runs; pushes get unique groups so none is dropped. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: MAVEN_OPTS: -Xmx2048m -Xms1024m LANG: en_US.utf8 jobs: - changes: - uses: ./.github/workflows/changes.yml - owasp: name: OWASP - needs: changes - if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -46,7 +46,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v7 + - name: Detect changes + id: changes + uses: ./.github/actions/detect-changes - name: Setup Java 25 + if: steps.changes.outputs.code == 'true' uses: actions/setup-java@v6 with: distribution: temurin @@ -55,6 +59,7 @@ jobs: - name: Cache NVD Database id: cache-nvd + if: steps.changes.outputs.code == 'true' uses: actions/cache/restore@v6 with: path: ~/.m2/repository/org/owasp/dependency-check-data @@ -65,35 +70,36 @@ jobs: - name: OWASP Dependency check update cache via NIST_NVD_API_KEY id: nvd-api-update - if: ${{ env.HAVE_NIST_NVD_API_KEY == 'true' }} + if: ${{ steps.changes.outputs.code == 'true' && env.HAVE_NIST_NVD_API_KEY == 'true' }} continue-on-error: true run: mvn -N -V -DskipAssembly -Dmaven.test.skip=true -Powasp-nvd-api -Pdependency-update-only --no-transfer-progress env: NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY}} - name: OWASP Dependency check update cache via Mirror - if: ${{ env.HAVE_NIST_NVD_API_KEY == 'false' || steps.nvd-api-update.outcome == 'failure' }} + if: ${{ steps.changes.outputs.code == 'true' && (env.HAVE_NIST_NVD_API_KEY == 'false' || steps.nvd-api-update.outcome == 'failure') }} run: mvn -N -V -DskipAssembly -Dmaven.test.skip=true -Powasp-nvd-mirror -Pdependency-update-only --no-transfer-progress - name: Cache NVD Database uses: actions/cache/save@v6 - if: ${{ always() }} + if: ${{ always() && steps.changes.outputs.code == 'true' }} with: path: ~/.m2/repository/org/owasp/dependency-check-data key: nvd-cache-${{ runner.os }}-owasp-${{ github.run_id }} - name: OWASP check (Without running tests) + if: steps.changes.outputs.code == 'true' run: mvn -B org.owasp:dependency-check-maven:aggregate -Pdependency-check -Pjakartaee11 -DautoUpdate=false --no-transfer-progress - name: Upload Dependency Check reports uses: actions/upload-artifact@v7 - if: always() + if: always() && steps.changes.outputs.code == 'true' with: name: dependency-check path: target/dependency-check* - name: Add OWASP summary - if: always() + if: always() && steps.changes.outputs.code == 'true' run: | { echo "## OWASP Dependency Check" diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index aa209f81c7..f5a077f79c 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -23,37 +23,42 @@ on: permissions: read-all +# Pull requests cancel superseded runs; pushes get unique groups so none is dropped. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: MAVEN_OPTS: -Xmx2048m -Xms1024m LANG: en_US.utf8 HAVE_SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN != '' }} jobs: - changes: - uses: ./.github/workflows/changes.yml - sonarcloud: name: Scan - needs: changes runs-on: ubuntu-latest - if: ${{ needs.changes.outputs.code == 'true' && !github.event.pull_request.base.repo.fork && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} + if: ${{ !github.event.pull_request.base.repo.fork && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} steps: - uses: actions/checkout@v7 with: fetch-depth: 0 + - name: Detect changes + id: changes + uses: ./.github/actions/detect-changes - uses: actions/setup-java@v6 + if: steps.changes.outputs.code == 'true' with: distribution: temurin java-version: 21 cache: 'maven' - name: SonarCloud Scan - if: ${{ env.HAVE_SONARCLOUD_TOKEN == 'true' }} + if: ${{ steps.changes.outputs.code == 'true' && env.HAVE_SONARCLOUD_TOKEN == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} run: ./mvnw -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage -DskipAssembly - name: SonarCloud Scan -- SKIPPED - if: ${{ env.HAVE_SONARCLOUD_TOKEN != 'true' }} + if: ${{ steps.changes.outputs.code == 'true' && env.HAVE_SONARCLOUD_TOKEN != 'true' }} run: | echo "### SonarCloud not configured" >> $GITHUB_STEP_SUMMARY echo "secrets.SONARCLOUD_TOKEN not existing, cannot push coverage checks" >> $GITHUB_STEP_SUMMARY diff --git a/Jenkinsfile b/Jenkinsfile index b58498fbb3..c7ddbd50b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { script { // Skip the build when a change only touched .claude/ or the root // process docs - agent instructions and policy, not code. The - // pattern mirrors .github/workflows/changes.yml; keep the two in + // pattern mirrors .github/actions/detect-changes; keep the two in // step. Fails open: anything unexpected (no baseline, an // unreachable commit, a git error) reports true and the build // runs as before.