diff --git a/.github/workflows/ai-agent-test-suite.yaml b/.github/workflows/ai-agent-test-suite.yaml index bf364f0..69fc51c 100644 --- a/.github/workflows/ai-agent-test-suite.yaml +++ b/.github/workflows/ai-agent-test-suite.yaml @@ -24,40 +24,7 @@ on: default: false secrets: GH_REPOSITORY_ADMIN_TOKEN: - description: "Github repository admin token" - required: true - DEV_ARGOCD_PASSWORD: - description: "ArgoCD password" - required: true - NPM_TOKEN: - description: "Npm token" - required: true - DOCKER_HUB_ACTION_USER: - description: "Docker hub user" - required: true - DOCKER_HUB_ACTION_PASSWORD: - description: "Docker hub password" - required: true - MAILOSAUR_API_KEY: - description: "MAILOSAUR API KEY" - required: true - MAILOSAUR_SERVER_ID: - description: "MAILOSAUR SERVER ID" - required: true - MAILOSAUR_SERVER_DOMAIN: - description: "MAILOSAUR SERVER DOMAIN" - required: true - ZEPHYR_TOKEN: - description: "Zephyr token" - required: true - AZURE_APP_CLIENT_ID: - description: "Azure app client id" - required: true - AZURE_APP_SECRET: - description: "Azure app secret" - required: true - DD_API_KEY: - description: "Datadog API Key" + description: "Github repository admin token, used to remove the venv" required: true permissions: @@ -87,7 +54,8 @@ jobs: portal_url: ${{ needs.start-venv.outputs.portalUrl }} tests_tag: ${{ inputs.tests_tag }} command: | - resolved=$(yarn test:e2e:ai-agent-sanity:hosted --list | grep -oE 'Total: [0-9]+ test' | grep -oE '[0-9]+') + set -o pipefail + resolved=$(yarn test:e2e:ai-agent-sanity:hosted --list --reporter=list | grep -oE 'Total: [0-9]+ test' | grep -oE '[0-9]+') echo "resolved ${resolved:-0} test(s), expected at least ${{ inputs.expected_tests }}" [ "${resolved:-0}" -ge "${{ inputs.expected_tests }}" ] || { echo "::error::the AI agent suite resolved ${resolved:-0} tests; specs are being skipped on this environment" @@ -111,21 +79,37 @@ jobs: strategy: fail-fast: false matrix: - total: [3] - shard: [1, 2, 3] + total: [2] + shard: [1, 2] secrets: inherit + # Two removal jobs so a failure never strands a venv for the full 2h age cap: + # - tests failed → keep briefly, the venv is worth poking at + # - guard/setup failed, or everything passed → remove now, nothing to debug remove-venv: name: Remove venv needs: [start-venv, verify-suite-size, run-ai-agent-tests] if: | always() && (needs.start-venv.outputs.environmentId) && - !contains(needs.run-ai-agent-tests.result, 'failure') && - !contains(needs.verify-suite-size.result, 'failure') + !contains(needs.run-ai-agent-tests.result, 'failure') uses: frontegg/workflows/.github/workflows/remove-single-venv.yaml@master with: venvId: ${{ needs.start-venv.outputs.environmentId }} gracePeriodInMinutes: "0" secrets: admin_github_token: ${{ secrets.GH_REPOSITORY_ADMIN_TOKEN }} + + remove-venv-after-test-failure: + name: Remove venv (tests failed) + needs: [start-venv, run-ai-agent-tests] + if: | + always() && + (needs.start-venv.outputs.environmentId) && + contains(needs.run-ai-agent-tests.result, 'failure') + uses: frontegg/workflows/.github/workflows/remove-single-venv.yaml@master + with: + venvId: ${{ needs.start-venv.outputs.environmentId }} + gracePeriodInMinutes: "30" + secrets: + admin_github_token: ${{ secrets.GH_REPOSITORY_ADMIN_TOKEN }}