Skip to content

ci(cli): run E2E tests in parallel #1726

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
betegon wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ci(cli): run E2E tests in parallel #1726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0af6b36
ci(cli): run E2E tests in parallel
betegon Oct 29, 2026
2c1cde7
ci(cli): remove unused E2E environment setup
betegon Oct 29, 2026
756b806
ci(cli): prepare E2E and unit inputs once
betegon Oct 30, 2026
1b42251
ci(cli): reuse npm build setup and share Node pins
betegon Oct 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/node-versions/20
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
20.20.2
1 change: 1 addition & 0 deletions .github/node-versions/22
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
22.23.4
1 change: 1 addition & 0 deletions .github/node-versions/24
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
24.22.1
11 changes: 4 additions & 7 deletions .github/workflows/cli-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ permissions:

env:
SENTRY_CLIENT_ID: ${{ vars.SENTRY_CLIENT_ID }}
NODE_VERSION_20: "20.20.2"
NODE_VERSION_22: "22.23.4"
NODE_VERSION_24: "24.22.1"

jobs:
nightly-version:
Expand Down Expand Up @@ -58,7 +55,7 @@ jobs:
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
Expand Down Expand Up @@ -267,7 +264,7 @@ jobs:
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
Expand Down Expand Up @@ -303,7 +300,7 @@ jobs:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: ${{ matrix.node == 24 && env.NODE_VERSION_24 || matrix.node == 20 && env.NODE_VERSION_20 || env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/${{ matrix.node }}
- name: Download npm artifact
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
Expand Down Expand Up @@ -334,7 +331,7 @@ jobs:
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: ${{ env.NODE_VERSION_24 }}
node-version-file: .github/node-versions/24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
Expand Down
107 changes: 71 additions & 36 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@ on:
permissions:
contents: read

env:
NODE_VERSION_20: "20.20.2"
NODE_VERSION_22: "22.23.4"
NODE_VERSION_24: "24.22.1"

jobs:
discover-projects:
name: Discover projects
Expand All @@ -24,14 +19,14 @@ jobs:
runtime-matrix: ${{ steps.projects.outputs.runtime-matrix }}
runtime-count: ${{ steps.projects.outputs.runtime-count }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
fetch-depth: 0
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
- name: Build project matrix
id: projects
env:
Expand All @@ -53,27 +48,27 @@ jobs:
name: Install
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- run: pnpm install --frozen-lockfile

quality:
name: Repository quality
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm run test:ci-projects
Expand All @@ -96,13 +91,13 @@ jobs:
SENTRY_CLI_NO_TELEMETRY: "1"
FOSSILIZE_SIGN: "n"
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm --filter sentry run build -- --target darwin-arm64
Expand All @@ -127,13 +122,13 @@ jobs:
SENTRY_CLIENT_ID: ${{ vars.SENTRY_CLIENT_ID }}
SENTRY_OAUTH_CLIENT_SECRET: ci-test-secret
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- run: cp .env.example .env.local
- run: pnpm install --frozen-lockfile
Expand Down Expand Up @@ -192,10 +187,10 @@ jobs:
PROJECT_NAME: ${{ matrix.name }}
CHECK_SCRIPT: ${{ matrix.policy }}
run: pnpm --filter "$PROJECT_NAME" run "$CHECK_SCRIPT"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
if: matrix.e2e != ''
with:
node-version: ${{ env.NODE_VERSION_24 }}
node-version-file: .github/node-versions/24
cache: pnpm
- name: Run E2E tests on Node 24
if: matrix.e2e != ''
Expand All @@ -214,6 +209,47 @@ jobs:
git diff --exit-code
[[ -z "$(git status --porcelain --untracked-files=all)" ]]

cli-e2e:
name: CLI E2E (Node 24)
needs: discover-projects
if: contains(fromJSON(needs.discover-projects.outputs.matrix).include.*.name, 'sentry')
runs-on: ubuntu-latest
timeout-minutes: 30
env:
SENTRY_CLIENT_ID: ${{ vars.SENTRY_CLIENT_ID }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@v7
with:
node-version-file: .github/node-versions/22
cache: pnpm
- run: pnpm install --frozen-lockfile
- name: Build project dependencies
run: pnpm --filter 'sentry^...' --if-present run build
- run: pnpm --filter sentry run ci:prepare
- name: Build standalone CLI
run: pnpm --filter sentry run tsx script/build.ts --single
- uses: actions/setup-node@v7
with:

@BYK BYK Oct 29, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should defer this step after the standalone build and use that already-built artifact for e2e tests. This should also save us time.

@betegon betegon Oct 30, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Node 24 switch already follows the standalone build and E2E reuses the binary via SENTRY_CLI_BINARY.

also removed the duplicate standalone build from the CLI project job

@BYK BYK Oct 30, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

???

The lines above this literally are:

- name: Build project dependencies
        run: pnpm --filter 'sentry^...' --if-present run build
      - run: pnpm --filter sentry run ci:prepare
      - name: Build standalone CLI
        run: pnpm --filter sentry run tsx script/build.ts --single

Which builds the dependencies and gets a new CLI build. We should be using the build going in for nightly which means we need to download it from a previous job as an artifact.

node-version-file: .github/node-versions/24
- name: Build npm bundle
run: pnpm --filter sentry run tsx script/bundle.ts
- name: Run E2E tests on Node 24
env:
SENTRY_CLI_BINARY: ${{ github.workspace }}/packages/cli/dist-bin/sentry-linux-x64
run: |
test -x "$SENTRY_CLI_BINARY"
test -s packages/cli/dist/bin.cjs
test -s packages/cli/dist/index.cjs
pnpm --filter sentry exec vitest run test/e2e
- name: Check generated files
run: |
git diff --exit-code
[[ -z "$(git status --porcelain --untracked-files=all)" ]]

npm-runtime:
name: NPM package (${{ matrix.name }})
needs: discover-projects
Expand All @@ -227,19 +263,15 @@ jobs:
SENTRY_CLIENT_ID: ${{ vars.SENTRY_CLIENT_ID }}
SENTRY_CLI_NO_TELEMETRY: "1"
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
cache: pnpm
- run: pnpm install --frozen-lockfile
- name: Build package dependencies
env:
PROJECT_NAME: ${{ matrix.name }}
run: pnpm --filter "${PROJECT_NAME}^..." --if-present run build
- name: Build and pack once on Node 22
env:
BUILD_SCRIPT: ${{ matrix.npmRuntime }}
Expand All @@ -254,25 +286,25 @@ jobs:
test -s "${packages[0]}"
[[ "$(stat -c %s "${packages[0]}")" -le 52428800 ]]
mv "${packages[0]}" "$RUNNER_TEMP/npm-package/package.tgz"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_20 }}
node-version-file: .github/node-versions/20
- name: Validate packaged runtime on Node 20
env:
RUNTIME_BIN: ${{ matrix.runtimeBin }}
RUNTIME_PACKAGE: ${{ matrix.name }}
run: bash scripts/validate-npm-runtime.sh
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_22 }}
node-version-file: .github/node-versions/22
- name: Validate packaged runtime on Node 22
env:
RUNTIME_BIN: ${{ matrix.runtimeBin }}
RUNTIME_PACKAGE: ${{ matrix.name }}
run: bash scripts/validate-npm-runtime.sh
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/setup-node@v7
with:
node-version: ${{ env.NODE_VERSION_24 }}
node-version-file: .github/node-versions/24
- name: Validate packaged runtime on Node 24
env:
RUNTIME_BIN: ${{ matrix.runtimeBin }}
Expand All @@ -290,6 +322,7 @@ jobs:
project,
npm-runtime,
cli-asset-catalog,
cli-e2e,
]
runs-on: ubuntu-latest
steps:
Expand All @@ -304,6 +337,7 @@ jobs:
RUNTIME_COUNT: ${{ needs.discover-projects.outputs.runtime-count }}
ASSET_CATALOG_RESULT: ${{ needs.cli-asset-catalog.result }}
ASSET_CATALOG_SELECTED: ${{ contains(fromJSON(needs.discover-projects.outputs.matrix).include.*.name, 'sentry') }}
CLI_E2E_RESULT: ${{ needs.cli-e2e.result }}
run: |
[[ "$DISCOVER_RESULT" == success ]]
[[ "$INSTALL_RESULT" == success ]]
Expand All @@ -322,4 +356,5 @@ jobs:
if [[ "$ASSET_CATALOG_SELECTED" == false ]]; then
expected=skipped
fi
[[ "$ASSET_CATALOG_RESULT" == "$expected" ]]
test "$ASSET_CATALOG_RESULT" = "$expected"
test "$CLI_E2E_RESULT" = "$expected"
14 changes: 14 additions & 0 deletions docs/operations/github-actions.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,20 @@ discovery, installation, repository quality, and every selected project.
Package-specific exceptions live in `package.json#sentryCi`; the standalone
smoke-test suite remains in its own workflow.

`test.yml` and `cli-build.yml` read their exact Node 20, 22, and 24 versions
from `.github/node-versions/`. The npm runtime job calls the self-contained
`sentryCi.npmRuntime` build command once, then validates the same package on
all three Node versions.

`CLI E2E (Node 24)` runs independently of the CLI project job. It builds the
standalone CLI on Node 22, then builds the npm bundle and tests both on Node 24.
Dependencies and generated sources are prepared once per job. The CLI project
job runs typechecks, unit tests, and coverage without rebuilding the standalone
or repeating generation. Public build and test commands still prepare their own
inputs for local use. The final `test` gate
requires E2E to pass whenever the CLI is selected, and requires it to be skipped
otherwise.

### deploy.yml

Runs after a successful `Test` push run on `main`. Checks out the tested commit
Expand Down
11 changes: 9 additions & 2 deletions packages/cli/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,13 @@
},
"packageManager": "pnpm@10.11.0",
"sentryCi": {
"build": false,
"coverage": "coverage/lcov.info",
"e2e": false,
"npmRuntime": "bundle",
"policy": "ci:policy"
"policy": "ci:policy",
"test": "ci:test:unit",
"typecheck": "ci:typecheck"
},
"scripts": {
"tsx": "tsx --env-file-if-exists=.env.local --import ./script/require-shim.mjs",
Expand All @@ -65,7 +69,7 @@
"lint": "pnpm -w exec oxlint --report-unused-disable-directives packages/cli && pnpm -w exec oxfmt --check packages/cli",
"lint:fix": "pnpm -w exec oxlint --fix packages/cli && pnpm -w exec oxfmt --write packages/cli",
"test": "pnpm run test:unit",
"test:unit": "pnpm run generate:docs && pnpm run generate:sdk && vitest run test/lib test/commands test/types test/script --coverage",
"test:unit": "pnpm run generate:docs && pnpm run generate:sdk && pnpm run ci:test:unit",
"test:changed": "pnpm run generate:docs && pnpm run generate:sdk && vitest run --changed",
"test:e2e": "pnpm tsx script/prepare-e2e-bundle.ts && vitest run test/e2e",
"test:init-eval": "vitest run test/init-eval --testTimeout 600000",
Expand Down Expand Up @@ -93,6 +97,9 @@
"check:doc-examples": "pnpm tsx script/check-doc-examples.ts",
"check:installer-docs": "pnpm tsx script/check-installer-docs.ts",
"check:destructive": "pnpm tsx script/check-destructive-commands.ts",
"ci:prepare": "pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk",
"ci:typecheck": "tsc --noEmit",
"ci:test:unit": "vitest run test/lib test/commands test/types test/script --coverage",
"ci:policy": "pnpm run check:patches && pnpm run check:deps && pnpm run check:errors && pnpm run check:fragments && pnpm run check:docs-sections && pnpm run check:env-coverage && pnpm run check:stale-refs && pnpm run check:doc-examples && pnpm run check:installer-docs && pnpm run check:destructive"
},
"devDependencies": {
Expand Down
Loading
Loading