From 3cd28b05ac1a084ba3c35b810d9f050288459881 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Thu, 20 Aug 2026 23:48:58 +0000 Subject: [PATCH] ci: configure scheduled Cloud Build failure reporter Add scheduled Cloud Build failure reporter workflow to monitor nightly and continuous integration test runs and open/update issues on failure. --- .../cloud_build_failure_reporter.yml | 190 ++++++++++++++++++ .github/workflows/schedule_reporter.yml | 30 +++ 2 files changed, 220 insertions(+) create mode 100644 .github/workflows/cloud_build_failure_reporter.yml create mode 100644 .github/workflows/schedule_reporter.yml diff --git a/.github/workflows/cloud_build_failure_reporter.yml b/.github/workflows/cloud_build_failure_reporter.yml new file mode 100644 index 000000000..f97d3c75c --- /dev/null +++ b/.github/workflows/cloud_build_failure_reporter.yml @@ -0,0 +1,190 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Cloud Build Failure Reporter + +on: + workflow_call: + inputs: + trigger_names: + required: true + type: string + workflow_dispatch: + inputs: + trigger_names: + description: 'Cloud Build trigger names separated by comma.' + required: true + default: '' + +jobs: + report: + if: github.repository == 'GoogleCloudPlatform/cloud-sql-proxy' + + permissions: + issues: 'write' + checks: 'read' + contents: 'read' + + runs-on: 'ubuntu-latest' + + steps: + - uses: 'actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd' # v8 + env: + TRIGGER_NAMES: ${{ inputs.trigger_names }} + with: + script: |- + // parse test names + const testNameSubstring = process.env.TRIGGER_NAMES || ''; + const testNameFound = new Map(); // keeps track of whether each test is found + testNameSubstring.split(',').forEach(testName => { + testNameFound.set(testName.trim(), false); + }); + + // label for all issues opened by reporter + const periodicLabel = 'periodic-failure'; + + // check if any reporter opened any issues previously + const prevIssues = await github.paginate(github.rest.issues.listForRepo, { + ...context.repo, + state: 'open', + creator: 'github-actions[bot]', + labels: [periodicLabel] + }); + + // createOrCommentIssue creates a new issue or comments on an existing issue. + const createOrCommentIssue = async function (title, txt) { + if (prevIssues.length < 1) { + console.log('no previous issues found, creating one'); + await github.rest.issues.create({ + ...context.repo, + title: title, + body: txt, + labels: [periodicLabel] + }); + return; + } + // only comment on issue related to the current test + for (const prevIssue of prevIssues) { + if (prevIssue.title.includes(title)) { + console.log( + `found previous issue ${prevIssue.html_url}, adding comment` + ); + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prevIssue.number, + body: txt + }); + return; + } + } + // if previous issues exist for other tests but not this one, create a new issue + console.log('no matching previous issue for this test, creating one'); + await github.rest.issues.create({ + ...context.repo, + title: title, + body: txt, + labels: [periodicLabel] + }); + }; + + // updateIssues comments on any existing issues. No-op if no issue exists. + const updateIssues = async function (checkName, txt) { + if (prevIssues.length < 1) { + console.log('no previous issues found.'); + return; + } + // only comment on issue related to the current test + for (const prevIssue of prevIssues) { + if (prevIssue.title.includes(checkName)) { + console.log(`found previous issue ${prevIssue.html_url}, adding comment`); + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prevIssue.number, + body: txt + }); + } + } + }; + + // Find status of check runs. + // We will find check runs for each commit and then filter for the periodic. + // Checks API only allows for ref and if we use main there could be edge cases where + // the check run happened on a SHA that is different from head. + const commits = await github.paginate(github.rest.repos.listCommits, { + ...context.repo + }); + + const relevantChecks = new Map(); + for (const commit of commits) { + console.log( + `checking runs at ${commit.html_url}: ${commit.commit.message}` + ); + const checks = await github.rest.checks.listForRef({ + ...context.repo, + ref: commit.sha + }); + + // Iterate through each check and find matching names + for (const check of checks.data.check_runs) { + console.log(`Handling test name ${check.name}`); + for (const testName of testNameFound.keys()) { + if (testNameFound.get(testName) === true) { + // skip if a check is already found for this name + continue; + } + if (check.name.includes(testName)) { + relevantChecks.set(check, commit); + testNameFound.set(testName, true); + } + } + } + // Break out of the loop early if all tests are found + const allTestsFound = Array.from(testNameFound.values()).every(value => value === true); + if (allTestsFound) { + break; + } + } + + // Handle each relevant check + relevantChecks.forEach((commit, check) => { + if ( + check.status === 'completed' && + check.conclusion === 'success' + ) { + updateIssues( + check.name, + `[Tests are passing](${check.html_url}) for commit [${commit.sha}](${commit.html_url}).` + ); + } else if (check.status === 'in_progress') { + console.log( + `Check is pending ${check.html_url} for ${commit.html_url}. Retry again later.` + ); + } else { + createOrCommentIssue( + `Cloud Build Failure Reporter: ${check.name} failed`, + `Cloud Build Failure Reporter found test failure for [**${check.name}** ](${check.html_url}) at [${commit.sha}](${commit.html_url}). Please fix the error and then close the issue after the **${check.name}** test passes.` + ); + } + }); + + // no periodic checks found across all commits, report it + const noTestFound = Array.from(testNameFound.values()).every(value => value === false); + if (noTestFound) { + createOrCommentIssue( + `Missing periodic tests: ${testNameSubstring}`, + `No periodic test is found for triggers: ${testNameSubstring}. Last checked from ${ + commits[0].html_url + } to ${commits[commits.length - 1].html_url}` + ); + } diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml new file mode 100644 index 000000000..3bc569eb6 --- /dev/null +++ b/.github/workflows/schedule_reporter.yml @@ -0,0 +1,30 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Schedule Reporter + +on: + schedule: + - cron: '0 6 * * *' # Runs at 6 AM every morning + +jobs: + run_reporter: + if: github.repository == 'GoogleCloudPlatform/cloud-sql-proxy' + permissions: + issues: 'write' + checks: 'read' + contents: 'read' + uses: ./.github/workflows/cloud_build_failure_reporter.yml + with: + trigger_names: "integration-test-pr-proxy"