chore(ci): Streamline CI setup to split bundle, layer, tarball generation (#20396)

## Summary

Restructures the CI Build & Test pipeline to split the monolithic build
step into parallel jobs and eliminate redundant work, reducing the
critical path for test execution.

   ### Key changes

   **Split build into parallel jobs:**
- The main `Build` job now only runs `build:transpile`, `build:types`,
and `build:extension` (via `yarn build:ci`)
- **Bundle builds** (`job_build_bundles`) run in a separate parallel job
after `Build` — only when browser integration tests are affected
- **Lambda layer build** (`job_build_layer`) runs in its own job — only
when `@sentry/aws-serverless` or related E2E tests are affected
- **Tarball packaging** (`job_build_tarballs`) is a dedicated job that
replaces the old `job_e2e_prepare` step

   **Smarter build artifact handling:**
- Replaced the hand-maintained `CACHED_BUILD_PATHS` list with a dynamic
Nx-derived artifact path computation
(`scripts/ci-print-build-artifact-paths.mjs`) that reads the Nx project
graph to determine
   exactly which output directories to upload
- Removed the tarball cache (`BUILD_CACHE_TARBALL_KEY`) — tarballs are
now built once and passed as artifacts, not cached across runs
- Each build stage uploads its own artifact (`build-output`,
`build-bundle-output`, `build-layer-output`), so downstream jobs only
download what they need

   **E2E test matrix generation moved into Build:**
- The E2E test matrix (`ci:build-matrix`) is now computed inside the
`Build` job instead of a separate `job_e2e_prepare` job
- This removes `job_e2e_prepare` as a separate step and shaves off the
sequential dependency

   **NX cache improvements:**
- Moved NX cache directory from `.nxcache` to `.nx/cache` (includes
`workspace-data` for better cache hits)
- Changed from `actions/cache` (save+restore) to `actions/cache/save` in
Build + `actions/cache/restore` in downstream jobs
- Cache is now scoped per-PR (`${{ github.head_ref }}-${{ github.run_id
}}`) for more predictable behavior

   **Other cleanups:**
- `getTestMatrix.ts` → `getTestMatrix.mjs` — removed the `ts-node`,
`glob`, and `yaml` dependencies by rewriting as plain ESM with `node:fs`
and `node:util`
- Fixed 1aws-serverless1 to include the lambda extension in the
transpile step, to ensure it is properly cached and restored, and as we
depend on this for the tarball as well.
   - Fixed `angular` dependency resolution for CI builds
   - Fixes `nestjs` types output for proper caching

   ### Before → After pipeline structure

   ```
   BEFORE:
Metadata → Build (transpile+types+bundles+layer+tarballs) → All tests

   AFTER:
     Metadata → Build (transpile+types only)
                  ├── Build bundles ──→ Browser Playwright tests
                  ├── Build layer ───→ (only when needed)
                  ├── Build tarballs → E2E tests
└── Unit/Integration/Remix tests (no extra build needed)
   ```

Somehow, cache restoration with nx seems hit or miss, it works sometimes
but not other times. I opened an issue here:
nrwl/nx#35403 - fixing this would improve this
by a couple minutes, so would be nice to get there...

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: mydea

.github/workflows/build.yml

@@ -10,11 +10,6 @@ on:
 env:
   HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
 
-  NX_CACHE_RESTORE_KEYS: |
-    nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }}
-    nx-Linux-${{ github.ref }}
-    nx-Linux
-
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
 concurrency:
@@ -36,16 +31,10 @@ jobs:
         with:
           node-version-file: 'package.json'
           cache: 'yarn'
+
       - name: Install dependencies
         run: yarn install --ignore-engines --frozen-lockfile
 
-      - name: NX cache
-        uses: actions/cache/restore@v5
-        with:
-          path: .nxcache
-          key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }}
-          restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }}
-
       - name: Build packages
         run: yarn build
 

.github/workflows/flaky-test-detector.yml

@@ -198,7 +198,7 @@ try {
 ```
 
 Test apps in the folder `test-applications` will be automatically picked up by CI in the job `job_e2e_tests` (in `.github/workflows/build.yml`).
-The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.ts`.
+The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.mjs`.
 
 For each test app, CI checks its dependencies (and devDependencies) to see if any of them have changed in the current PR (based on nx affected projects).
 For example, if something is changed in the browser package, only E2E test apps that depend on browser will run, while others will be skipped.

dev-packages/e2e-tests/README.md

@@ -1,42 +1,21 @@
-import { execSync } from 'child_process';
-import * as fs from 'fs';
-import { sync as globSync } from 'glob';
-import * as path from 'path';
-import { dirname } from 'path';
-import { parseArgs } from 'util';
-
-interface MatrixInclude {
-  /** The test application (directory) name. */
-  'test-application': string;
-  /** Optional override for the build command to run. */
-  'build-command'?: string;
-  /** Optional override for the assert command to run. */
-  'assert-command'?: string;
-  /** Optional label for the test run. If not set, defaults to value of `test-application`. */
-  label?: string;
-}
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
 
-interface PackageJsonSentryTestConfig {
-  /** If this is true, the test app is optional. */
-  optional?: boolean;
-  /** Variant configs that should be run in non-optional test runs. */
-  variants?: Partial<MatrixInclude>[];
-  /** Variant configs that should be run in optional test runs. */
-  optionalVariants?: Partial<MatrixInclude>[];
-  /** Skip this test app for matrix generation. */
-  skip?: boolean;
-}
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
- * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests.
- * It checks which test applications are affected by the current changes in the PR and then generates a matrix
+ * Generates a matrix for the GitHub Actions workflow to run the E2E tests.
+ * Checks which test applications are affected by the current changes in the PR and then generates a matrix
  * including all test apps that have at least one dependency that was changed in the PR.
  * If no `--base=xxx` is provided, it will output all test applications.
  *
  * If `--optional=true` is set, it will generate a matrix of optional test applications only.
  * Otherwise, these will be skipped.
  */
-function run(): void {
+function run() {
   const { values } = parseArgs({
     args: process.argv.slice(2),
     options: {
@@ -52,9 +31,7 @@ function run(): void {
   // eslint-disable-next-line no-console
   console.error(`Parsed command line arguments: base=${base}, head=${head}, optional=${optional}`);
 
-  const testApplications = globSync('*/package.json', {
-    cwd: `${__dirname}/../test-applications`,
-  }).map(filePath => dirname(filePath));
+  const testApplications = discoverTestApplicationDirs();
 
   // For GitHub Action debugging (using stderr the 'matrix=...' output is not polluted)
   // eslint-disable-next-line no-console
@@ -67,7 +44,7 @@ function run(): void {
     : testApplications;
 
   const optionalMode = optional === 'true';
-  const includes: MatrixInclude[] = [];
+  const includes = [];
 
   includedTestApplications.forEach(testApp => {
     addIncludesForTestApp(testApp, includes, { optionalMode });
@@ -78,11 +55,18 @@ function run(): void {
   console.log(`matrix=${JSON.stringify({ include: includes })}`);
 }
 
-function addIncludesForTestApp(
-  testApp: string,
-  includes: MatrixInclude[],
-  { optionalMode }: { optionalMode: boolean },
-): void {
+/** Direct children of `test-applications/` that contain a `package.json` (replaces glob one-segment + package.json). */
+function discoverTestApplicationDirs() {
+  const appsRoot = path.join(__dirname, '..', 'test-applications');
+  return fs
+    .readdirSync(appsRoot, { withFileTypes: true })
+    .filter(entry => entry.isDirectory())
+    .map(entry => entry.name)
+    .filter(name => fs.existsSync(path.join(appsRoot, name, 'package.json')))
+    .sort();
+}
+
+function addIncludesForTestApp(testApp, includes, { optionalMode }) {
   const packageJson = getPackageJson(testApp);
 
   const shouldSkip = packageJson.sentryTest?.skip || false;
@@ -108,7 +92,7 @@ function addIncludesForTestApp(
   });
 }
 
-function getSentryDependencies(appName: string): string[] {
+function getSentryDependencies(appName) {
   const packageJson = getPackageJson(appName);
 
   const dependencies = {
@@ -119,11 +103,7 @@ function getSentryDependencies(appName: string): string[] {
   return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
 }
 
-function getPackageJson(appName: string): {
-  dependencies?: { [key: string]: string };
-  devDependencies?: { [key: string]: string };
-  sentryTest?: PackageJsonSentryTestConfig;
-} {
+function getPackageJson(appName) {
   const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
 
   if (!fs.existsSync(fullPath)) {
@@ -133,19 +113,14 @@ function getPackageJson(appName: string): {
   return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
 }
 
-run();
-
-function getAffectedTestApplications(
-  testApplications: string[],
-  { base = 'develop', head }: { base?: string; head?: string },
-): string[] {
+function getAffectedTestApplications(testApplications, { base = 'develop', head }) {
   const additionalArgs = [`--base=${base}`];
 
   if (head) {
     additionalArgs.push(`--head=${head}`);
   }
 
-  let affectedProjects: string[] = [];
+  let affectedProjects = [];
   try {
     affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
       .toString()
@@ -201,7 +176,7 @@ function getAffectedTestApplications(
   return Array.from(testAppsToRun);
 }
 
-function getChangedTestApps(base: string, head?: string): false | Set<string> {
+function getChangedTestApps(base, head) {
   const changedFiles = execSync(`git diff --name-only ${base}${head ? `..${head}` : ''} -- .`, {
     encoding: 'utf-8',
   })
@@ -214,7 +189,7 @@ function getChangedTestApps(base: string, head?: string): false | Set<string> {
   // eslint-disable-next-line no-console
   console.error(`Changed files since ${base}${head ? `..${head}` : ''}: ${JSON.stringify(changedFiles)}`);
 
-  const changedTestApps: Set<string> = new Set();
+  const changedTestApps = new Set();
   const testAppsPrefix = 'dev-packages/e2e-tests/test-applications/';
 
   for (const file of changedFiles) {
@@ -233,3 +208,5 @@ function getChangedTestApps(base: string, head?: string): false | Set<string> {
 
   return changedTestApps;
 }
+
+run();

…-packages/e2e-tests/lib/getTestMatrix.ts …packages/e2e-tests/lib/getTestMatrix.mjsdev-packages/e2e-tests/lib/getTestMatrix.ts renamed to dev-packages/e2e-tests/lib/getTestMatrix.mjs dev-packages/e2e-tests/lib/getTestMatrix.ts renamed to dev-packages/e2e-tests/lib/getTestMatrix.mjs

@@ -12,8 +12,8 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "ts-node validate-packed-tarball-setup.ts",
     "clean": "rimraf tmp node_modules packed && yarn clean:test-applications && yarn clean:pnpm",
-    "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts",
-    "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true",
+    "ci:build-matrix": "node ./lib/getTestMatrix.mjs",
+    "ci:build-matrix-optional": "node ./lib/getTestMatrix.mjs --optional=true",
     "ci:copy-to-temp": "ts-node ./ciCopyToTemp.ts",
     "ci:pnpm-overrides": "ts-node ./ciPnpmOverrides.ts",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.nuxt,.sveltekit,.react-router,.astro,.output,pnpm-lock.yaml,.last-run.json,test-results,.angular,event-dumps}",

dev-packages/e2e-tests/package.json

@@ -61,7 +61,7 @@
     }
   },
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
-  "cacheDirectory": ".nxcache",
+  "cacheDirectory": ".nx/cache",
   "tui": {
     "autoExit": true
   },

nx.json

@@ -1,8 +1,10 @@
 {
   "private": true,
   "scripts": {
-    "build": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types build:bundle build:layer",
+    "build": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types build:bundle",
+    "build:ci": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types",
     "build:bundle": "nx run-many -t build:bundle",
+    "build:layer": "nx run-many -t build:layer",
     "build:dev": "nx run-many -t build:types build:transpile",
     "build:dev:filter": "nx run-many -t build:dev -p",
     "build:transpile": "nx run-many -t build:transpile",
@@ -11,6 +13,7 @@
     "build:dev:watch": "nx run-many -t build:dev:watch",
     "build:tarball": "run-s clean:tarballs build:tarballs",
     "build:tarballs": "nx run-many -t build:tarball",
+    "ci:print-build-artifact-paths": "node ./scripts/ci-print-build-artifact-paths.mjs",
     "changelog": "ts-node ./scripts/get-commit-list.ts",
     "generate-changelog": "ts-node ./scripts/generate-changelog.ts",
     "circularDepCheck": "nx run-many -t circularDepCheck",

package.json

@@ -71,10 +71,7 @@
           "^build:types"
         ],
         "outputs": [
-          "{projectRoot}/build/esm2015",
-          "{projectRoot}/build/fesm2015",
-          "{projectRoot}/build/fesm2020",
-          "{projectRoot}/build/*.d.ts"
+          "{projectRoot}/build"
         ]
       }
     }

packages/angular/package.json

@@ -80,11 +80,12 @@
     "@vercel/nft": "^1.3.0"
   },
   "scripts": {
-    "build": "run-p build:transpile build:types build:extension && run-s build:layer",
+    "build": "run-p build:transpile build:types",
     "build:extension": "rollup -c rollup.lambda-extension.config.mjs && yarn ts-node scripts/buildLambdaExtension.ts",
     "build:layer": "rimraf build/aws && yarn ts-node scripts/buildLambdaLayer.ts",
     "build:dev": "run-p build:transpile build:types",
-    "build:transpile": "rollup -c rollup.npm.config.mjs",
+    "build:transpile": "run-s build:transpile:npm build:extension",
+    "build:transpile:npm": "rollup -c rollup.npm.config.mjs",
     "build:types": "run-s build:types:core build:types:downlevel",
     "build:types:core": "tsc -p tsconfig.types.json",
     "build:types:downlevel": "yarn downlevel-dts build/npm/types build/npm/types-ts3.8 --to ts3.8",
@@ -117,20 +118,10 @@
         ],
         "outputs": [
           "{projectRoot}/build/npm/esm",
-          "{projectRoot}/build/npm/cjs"
-        ]
-      },
-      "build:extension": {
-        "inputs": [
-          "production",
-          "^production"
-        ],
-        "dependsOn": [
-          "^build:transpile"
-        ],
-        "outputs": [
+          "{projectRoot}/build/npm/cjs",
           "{projectRoot}/build/lambda-extension"
-        ]
+        ],
+        "cache": true
       },
       "build:layer": {
         "inputs": [
@@ -139,7 +130,7 @@
         ],
         "dependsOn": [
           "build:transpile",
-          "build:extension"
+          "build:types"
         ],
         "outputs": [
           "{projectRoot}/build/aws"

packages/aws-serverless/package.json

@@ -84,5 +84,24 @@
   "volta": {
     "extends": "../../package.json"
   },
+  "nx": {
+    "targets": {
+      "build:transpile": {
+        "inputs": [
+          "production",
+          "^production"
+        ],
+        "outputs": [
+          "{projectRoot}/build/esm",
+          "{projectRoot}/build/cjs",
+          "{projectRoot}/*.d.ts"
+        ],
+        "dependsOn": [
+          "^build:transpile"
+        ],
+        "cache": true
+      }
+    }
+  },
   "sideEffects": false
 }