From fcb9d353a0f90dc6fe5c3d1397da8e920fa4a61e Mon Sep 17 00:00:00 2001 From: Yaron Blum Date: Fri, 21 Aug 2026 15:01:42 +0300 Subject: [PATCH 1/2] FR-26617: harden the venv suite per review Pins --reporter=list so the Total line the size guard greps is not an accident of reporter order, fails on pipe errors, trims the secrets contract to the one secret this workflow references (callers inherit the rest), and runs six specs in two shards instead of three. Co-Authored-By: Claude Opus 5 --- .github/workflows/ai-agent-test-suite.yaml | 42 +++------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ai-agent-test-suite.yaml b/.github/workflows/ai-agent-test-suite.yaml index bf364f0..cccca61 100644 --- a/.github/workflows/ai-agent-test-suite.yaml +++ b/.github/workflows/ai-agent-test-suite.yaml @@ -24,40 +24,7 @@ on: default: false secrets: GH_REPOSITORY_ADMIN_TOKEN: - description: "Github repository admin token" - required: true - DEV_ARGOCD_PASSWORD: - description: "ArgoCD password" - required: true - NPM_TOKEN: - description: "Npm token" - required: true - DOCKER_HUB_ACTION_USER: - description: "Docker hub user" - required: true - DOCKER_HUB_ACTION_PASSWORD: - description: "Docker hub password" - required: true - MAILOSAUR_API_KEY: - description: "MAILOSAUR API KEY" - required: true - MAILOSAUR_SERVER_ID: - description: "MAILOSAUR SERVER ID" - required: true - MAILOSAUR_SERVER_DOMAIN: - description: "MAILOSAUR SERVER DOMAIN" - required: true - ZEPHYR_TOKEN: - description: "Zephyr token" - required: true - AZURE_APP_CLIENT_ID: - description: "Azure app client id" - required: true - AZURE_APP_SECRET: - description: "Azure app secret" - required: true - DD_API_KEY: - description: "Datadog API Key" + description: "Github repository admin token, used to remove the venv" required: true permissions: @@ -87,7 +54,8 @@ jobs: portal_url: ${{ needs.start-venv.outputs.portalUrl }} tests_tag: ${{ inputs.tests_tag }} command: | - resolved=$(yarn test:e2e:ai-agent-sanity:hosted --list | grep -oE 'Total: [0-9]+ test' | grep -oE '[0-9]+') + set -o pipefail + resolved=$(yarn test:e2e:ai-agent-sanity:hosted --list --reporter=list | grep -oE 'Total: [0-9]+ test' | grep -oE '[0-9]+') echo "resolved ${resolved:-0} test(s), expected at least ${{ inputs.expected_tests }}" [ "${resolved:-0}" -ge "${{ inputs.expected_tests }}" ] || { echo "::error::the AI agent suite resolved ${resolved:-0} tests; specs are being skipped on this environment" @@ -111,8 +79,8 @@ jobs: strategy: fail-fast: false matrix: - total: [3] - shard: [1, 2, 3] + total: [2] + shard: [1, 2] secrets: inherit remove-venv: From db5cb01b7c4670c72c22575b01ba359027c9de78 Mon Sep 17 00:00:00 2001 From: Yaron Blum Date: Mon, 24 Aug 2026 21:08:59 +0300 Subject: [PATCH 2/2] FR-26617: never strand a venv when the guard or setup fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remove-venv only ran on full success, so a verify-suite-size failure — which is firing on every app-integrations run today, because the specs are still @skipVenv on master — left the venv to the 2h age cap. Split removal: keep the venv 30min only when the tests themselves failed (worth debugging); remove immediately on a guard/setup failure or success (nothing to debug). Co-Authored-By: Claude Opus 5 --- .github/workflows/ai-agent-test-suite.yaml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ai-agent-test-suite.yaml b/.github/workflows/ai-agent-test-suite.yaml index cccca61..69fc51c 100644 --- a/.github/workflows/ai-agent-test-suite.yaml +++ b/.github/workflows/ai-agent-test-suite.yaml @@ -83,17 +83,33 @@ jobs: shard: [1, 2] secrets: inherit + # Two removal jobs so a failure never strands a venv for the full 2h age cap: + # - tests failed → keep briefly, the venv is worth poking at + # - guard/setup failed, or everything passed → remove now, nothing to debug remove-venv: name: Remove venv needs: [start-venv, verify-suite-size, run-ai-agent-tests] if: | always() && (needs.start-venv.outputs.environmentId) && - !contains(needs.run-ai-agent-tests.result, 'failure') && - !contains(needs.verify-suite-size.result, 'failure') + !contains(needs.run-ai-agent-tests.result, 'failure') uses: frontegg/workflows/.github/workflows/remove-single-venv.yaml@master with: venvId: ${{ needs.start-venv.outputs.environmentId }} gracePeriodInMinutes: "0" secrets: admin_github_token: ${{ secrets.GH_REPOSITORY_ADMIN_TOKEN }} + + remove-venv-after-test-failure: + name: Remove venv (tests failed) + needs: [start-venv, run-ai-agent-tests] + if: | + always() && + (needs.start-venv.outputs.environmentId) && + contains(needs.run-ai-agent-tests.result, 'failure') + uses: frontegg/workflows/.github/workflows/remove-single-venv.yaml@master + with: + venvId: ${{ needs.start-venv.outputs.environmentId }} + gracePeriodInMinutes: "30" + secrets: + admin_github_token: ${{ secrets.GH_REPOSITORY_ADMIN_TOKEN }}