diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 544bb7900008..ee57a47747ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,8 +28,6 @@ concurrency: env: HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} - # WARNING: this disables cross os caching as ~ and - # github.workspace evaluate to differents paths CACHED_DEPENDENCY_PATHS: | ${{ github.workspace }}/node_modules ${{ github.workspace }}/packages/*/node_modules @@ -38,25 +36,19 @@ env: # DEPENDENCY_CACHE_KEY: can't be set here because we don't have access to yarn.lock - # WARNING: this disables cross os caching as ~ and - # github.workspace evaluate to differents paths - # packages/utils/cjs and packages/utils/esm: Symlinks to the folders inside of `build`, needed for tests - CACHED_BUILD_PATHS: | + BUILD_PATHS: | ${{ github.workspace }}/dev-packages/*/build ${{ github.workspace }}/packages/*/build - ${{ github.workspace }}/packages/*/lib - ${{ github.workspace }}/packages/ember/*.d.ts - ${{ github.workspace }}/packages/gatsby/*.d.ts + ${{ github.workspace }}/packages/*.d.ts - BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }} + # upload-artifact globs drop the path through the first `*` (see upload-artifact + # README). Tarballs therefore land in the artifact as /*.tgz; download + # tarball-output into packages/ so they resolve to packages//*.tgz. + TARBALL_ARTIFACT_GLOB: packages/*/*.tgz + TARBALL_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages - # GH will use the first restore-key it finds that matches - # So it will start by looking for one from the same branch, else take the newest one it can find elsewhere - # We want to prefer the cache from the current develop branch, if we don't find any on the current branch - NX_CACHE_RESTORE_KEYS: | - nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }} - nx-Linux-${{ github.ref }} - nx-Linux + # We cache NX per-PR for faster CI, other than this we do not want to rely on it + NX_CACHE_KEY: nx-Linux-${{ github.ref }} # https://bsky.app/profile/joyeecheung.bsky.social/post/3lhy6o54fo22h # Apparently some of our CI failures are attributable to a corrupt v8 cache, causing v8 failures with: "Check failed: current == end_slot_index.". @@ -66,6 +58,7 @@ env: jobs: job_get_metadata: uses: ./.github/workflows/ci-metadata.yml + name: Get CI Metadata with: head_commit: ${{ github.event.inputs.commit || github.sha }} permissions: @@ -110,34 +103,40 @@ jobs: base: ${{ github.event.pull_request.base.sha }} head: ${{ env.HEAD_COMMIT }} - - name: NX cache - uses: actions/cache@v5 - # Disable cache when: - # - on release branches - # - when PR has `ci-skip-cache` label or on nightly builds - if: | - needs.job_get_metadata.outputs.is_release == 'false' && - needs.job_get_metadata.outputs.force_skip_cache == 'false' - with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT || github.sha }} - # On develop branch, we want to _store_ the cache (so it can be used by other branches), but never _restore_ from it - restore-keys: - ${{needs.job_get_metadata.outputs.is_base_branch == 'false' && env.NX_CACHE_RESTORE_KEYS || - 'nx-never-restore'}} - - name: Build packages run: yarn build + - name: Store NX cache + uses: actions/cache/save@v5 + # Only cache this per-PR to speed up CI. + if: github.event_name == 'pull_request' + with: + path: .nx + key: ${{ env.NX_CACHE_KEY }} + - name: Upload build artifacts uses: actions/upload-artifact@v7 with: name: build-output - path: ${{ env.CACHED_BUILD_PATHS }} + path: ${{ env.BUILD_PATHS }} retention-days: 4 compression-level: 6 overwrite: true + - name: Determine which test applications should be run + id: matrix + run: + yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && + github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT + working-directory: dev-packages/e2e-tests + + - name: Determine which optional E2E test applications should be run + id: matrix-optional + run: + yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && + github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT + working-directory: dev-packages/e2e-tests + outputs: dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }} changed_node_integration: @@ -164,6 +163,8 @@ jobs: changed_browser_integration: ${{ needs.job_get_metadata.outputs.changed_ci == 'true' || contains(steps.checkForAffected.outputs.affected, '@sentry-internal/browser-integration-tests') }} + e2e-matrix: ${{ steps.matrix.outputs.matrix }} + e2e-matrix-optional: ${{ steps.matrix-optional.outputs.matrix }} job_check_branches: name: Check PR branches @@ -298,7 +299,7 @@ jobs: job_artifacts: name: Upload Artifacts - needs: [job_get_metadata, job_build] + needs: [job_get_metadata, job_build, job_build_tarballs] runs-on: ubuntu-24.04 # Build artifacts are only needed for releasing workflow. if: needs.job_get_metadata.outputs.is_release == 'true' @@ -316,8 +317,11 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Pack tarballs - run: yarn build:tarball + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 + with: + name: tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - name: Archive artifacts uses: actions/upload-artifact@v7 @@ -662,7 +666,7 @@ jobs: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - name: Check for dts files that reference stuff in the temporary build folder run: | - if grep -r --include "*.d.ts" --exclude-dir ".nxcache" 'import("@sentry(-internal)?/[^/]*/build' .; then + if grep -r --include "*.d.ts" --exclude-dir ".nx" 'import("@sentry(-internal)?/[^/]*/build' .; then echo "Found illegal TypeScript import statement." exit 1 fi @@ -834,19 +838,16 @@ jobs: cd packages/remix yarn test:integration:ci - job_e2e_prepare: - name: Prepare E2E tests + job_build_tarballs: + name: Build tarballs # We want to run this if: # - The build job was successful, not skipped if: | always() && needs.job_build.result == 'success' needs: [job_get_metadata, job_build] - runs-on: ubuntu-24.04-large-js + runs-on: ubuntu-24.04 timeout-minutes: 15 - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - matrix-optional: ${{ steps.matrix-optional.outputs.matrix }} steps: - name: Check out base commit (${{ github.event.pull_request.base.sha }}) uses: actions/checkout@v6 @@ -865,44 +866,33 @@ jobs: uses: ./.github/actions/restore-cache with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: NX cache + + - name: Restore NX cache uses: actions/cache/restore@v5 with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }} - # On develop branch, we want to _store_ the cache (so it can be used by other branches), but never _restore_ from it - restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }} + path: .nx + key: ${{ env.NX_CACHE_KEY }} - name: Build tarballs run: yarn build:tarball - - name: Stores tarballs in cache - uses: actions/cache/save@v5 + - name: Upload tarball artifacts + uses: actions/upload-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} - - - name: Determine which E2E test applications should be run - id: matrix - run: - yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && - github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT - working-directory: dev-packages/e2e-tests - - - name: Determine which optional E2E test applications should be run - id: matrix-optional - run: - yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && - github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT - working-directory: dev-packages/e2e-tests + name: tarball-output + path: ${{ env.TARBALL_ARTIFACT_GLOB }} + if-no-files-found: error + retention-days: 4 + compression-level: 6 + overwrite: true job_e2e_tests: name: E2E ${{ matrix.label || matrix.test-application }} Test # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_e2e_prepare.result == 'success' && needs.job_e2e_prepare.outputs.matrix != '{"include":[]}' - needs: [job_get_metadata, job_build, job_e2e_prepare] + always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' + needs: [job_get_metadata, job_build, job_build_tarballs] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -916,7 +906,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' strategy: fail-fast: false - matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }} + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix) }} steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v6 @@ -948,16 +938,11 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Restore tarball cache - uses: actions/cache/restore@v5 - id: restore-tarball-cache + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} - - - name: Build tarballs if not cached - if: steps.restore-tarball-cache.outputs.cache-hit != 'true' - run: yarn build:tarball + name: tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - name: Validate Verdaccio run: yarn test:validate @@ -1022,10 +1007,10 @@ jobs: # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_e2e_prepare.result == 'success' && - needs.job_e2e_prepare.outputs.matrix-optional != '{"include":[]}' && (github.event_name != 'pull_request' || + always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_build_tarballs.result == 'success' && + needs.job_build.outputs.e2e-matrix-optional != '{"include":[]}' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' - needs: [job_get_metadata, job_build, job_e2e_prepare] + needs: [job_get_metadata, job_build, job_build_tarballs] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -1039,7 +1024,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' strategy: fail-fast: false - matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }} + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }} steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) @@ -1058,16 +1043,11 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Restore tarball cache - uses: actions/cache/restore@v5 - id: restore-tarball-cache + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} - - - name: Build tarballs if not cached - if: steps.restore-tarball-cache.outputs.cache-hit != 'true' - run: yarn build:tarball + name: tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - name: Validate Verdaccio run: yarn test:validate diff --git a/.github/workflows/flaky-test-detector.yml b/.github/workflows/flaky-test-detector.yml index c0a8f1f720b1..973c3e296cdc 100644 --- a/.github/workflows/flaky-test-detector.yml +++ b/.github/workflows/flaky-test-detector.yml @@ -10,10 +10,7 @@ on: env: HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} - NX_CACHE_RESTORE_KEYS: | - nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }} - nx-Linux-${{ github.ref }} - nx-Linux + NX_CACHE_KEY: nx-Linux-${{ github.ref }} # Cancel in progress workflows on pull_requests. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value @@ -39,12 +36,11 @@ jobs: - name: Install dependencies run: yarn install --ignore-engines --frozen-lockfile - - name: NX cache + - name: Restore NX cache uses: actions/cache/restore@v5 with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }} - restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }} + path: .nx + key: ${{ env.NX_CACHE_KEY }} - name: Build packages run: yarn build diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 15de0fd49ee0..2ff4b0665cd3 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -198,7 +198,7 @@ try { ``` Test apps in the folder `test-applications` will be automatically picked up by CI in the job `job_e2e_tests` (in `.github/workflows/build.yml`). -The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.ts`. +The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.mjs`. For each test app, CI checks its dependencies (and devDependencies) to see if any of them have changed in the current PR (based on nx affected projects). For example, if something is changed in the browser package, only E2E test apps that depend on browser will run, while others will be skipped. diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.mjs similarity index 73% rename from dev-packages/e2e-tests/lib/getTestMatrix.ts rename to dev-packages/e2e-tests/lib/getTestMatrix.mjs index 86a4bda3e701..b8be7af5f528 100644 --- a/dev-packages/e2e-tests/lib/getTestMatrix.ts +++ b/dev-packages/e2e-tests/lib/getTestMatrix.mjs @@ -1,42 +1,21 @@ -import { execSync } from 'child_process'; -import * as fs from 'fs'; -import { sync as globSync } from 'glob'; -import * as path from 'path'; -import { dirname } from 'path'; -import { parseArgs } from 'util'; - -interface MatrixInclude { - /** The test application (directory) name. */ - 'test-application': string; - /** Optional override for the build command to run. */ - 'build-command'?: string; - /** Optional override for the assert command to run. */ - 'assert-command'?: string; - /** Optional label for the test run. If not set, defaults to value of `test-application`. */ - label?: string; -} +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; -interface PackageJsonSentryTestConfig { - /** If this is true, the test app is optional. */ - optional?: boolean; - /** Variant configs that should be run in non-optional test runs. */ - variants?: Partial[]; - /** Variant configs that should be run in optional test runs. */ - optionalVariants?: Partial[]; - /** Skip this test app for matrix generation. */ - skip?: boolean; -} +const __dirname = path.dirname(fileURLToPath(import.meta.url)); /** - * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests. - * It checks which test applications are affected by the current changes in the PR and then generates a matrix + * Generates a matrix for the GitHub Actions workflow to run the E2E tests. + * Checks which test applications are affected by the current changes in the PR and then generates a matrix * including all test apps that have at least one dependency that was changed in the PR. * If no `--base=xxx` is provided, it will output all test applications. * * If `--optional=true` is set, it will generate a matrix of optional test applications only. * Otherwise, these will be skipped. */ -function run(): void { +function run() { const { values } = parseArgs({ args: process.argv.slice(2), options: { @@ -52,9 +31,7 @@ function run(): void { // eslint-disable-next-line no-console console.error(`Parsed command line arguments: base=${base}, head=${head}, optional=${optional}`); - const testApplications = globSync('*/package.json', { - cwd: `${__dirname}/../test-applications`, - }).map(filePath => dirname(filePath)); + const testApplications = discoverTestApplicationDirs(); // For GitHub Action debugging (using stderr the 'matrix=...' output is not polluted) // eslint-disable-next-line no-console @@ -67,7 +44,7 @@ function run(): void { : testApplications; const optionalMode = optional === 'true'; - const includes: MatrixInclude[] = []; + const includes = []; includedTestApplications.forEach(testApp => { addIncludesForTestApp(testApp, includes, { optionalMode }); @@ -78,11 +55,18 @@ function run(): void { console.log(`matrix=${JSON.stringify({ include: includes })}`); } -function addIncludesForTestApp( - testApp: string, - includes: MatrixInclude[], - { optionalMode }: { optionalMode: boolean }, -): void { +/** Direct children of `test-applications/` that contain a `package.json` (replaces glob one-segment + package.json). */ +function discoverTestApplicationDirs() { + const appsRoot = path.join(__dirname, '..', 'test-applications'); + return fs + .readdirSync(appsRoot, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .filter(name => fs.existsSync(path.join(appsRoot, name, 'package.json'))) + .sort(); +} + +function addIncludesForTestApp(testApp, includes, { optionalMode }) { const packageJson = getPackageJson(testApp); const shouldSkip = packageJson.sentryTest?.skip || false; @@ -108,7 +92,7 @@ function addIncludesForTestApp( }); } -function getSentryDependencies(appName: string): string[] { +function getSentryDependencies(appName) { const packageJson = getPackageJson(appName); const dependencies = { @@ -119,11 +103,7 @@ function getSentryDependencies(appName: string): string[] { return Object.keys(dependencies).filter(key => key.startsWith('@sentry')); } -function getPackageJson(appName: string): { - dependencies?: { [key: string]: string }; - devDependencies?: { [key: string]: string }; - sentryTest?: PackageJsonSentryTestConfig; -} { +function getPackageJson(appName) { const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json'); if (!fs.existsSync(fullPath)) { @@ -133,19 +113,14 @@ function getPackageJson(appName: string): { return JSON.parse(fs.readFileSync(fullPath, 'utf8')); } -run(); - -function getAffectedTestApplications( - testApplications: string[], - { base = 'develop', head }: { base?: string; head?: string }, -): string[] { +function getAffectedTestApplications(testApplications, { base = 'develop', head }) { const additionalArgs = [`--base=${base}`]; if (head) { additionalArgs.push(`--head=${head}`); } - let affectedProjects: string[] = []; + let affectedProjects = []; try { affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`) .toString() @@ -201,7 +176,7 @@ function getAffectedTestApplications( return Array.from(testAppsToRun); } -function getChangedTestApps(base: string, head?: string): false | Set { +function getChangedTestApps(base, head) { const changedFiles = execSync(`git diff --name-only ${base}${head ? `..${head}` : ''} -- .`, { encoding: 'utf-8', }) @@ -214,7 +189,7 @@ function getChangedTestApps(base: string, head?: string): false | Set { // eslint-disable-next-line no-console console.error(`Changed files since ${base}${head ? `..${head}` : ''}: ${JSON.stringify(changedFiles)}`); - const changedTestApps: Set = new Set(); + const changedTestApps = new Set(); const testAppsPrefix = 'dev-packages/e2e-tests/test-applications/'; for (const file of changedFiles) { @@ -233,3 +208,5 @@ function getChangedTestApps(base: string, head?: string): false | Set { return changedTestApps; } + +run(); diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json index ac1f7ddf23f7..14f746e12b0a 100644 --- a/dev-packages/e2e-tests/package.json +++ b/dev-packages/e2e-tests/package.json @@ -15,8 +15,8 @@ "test:validate": "run-s test:validate-configuration test:validate-test-app-setups", "clean:verdaccio": "sh -c 'pkill -f verdaccio-runner.mjs 2>/dev/null || true'", "clean": "yarn clean:verdaccio && rimraf tmp node_modules verdaccio-config/storage && yarn clean:test-applications && yarn clean:pnpm", - "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts", - "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true", + "ci:build-matrix": "node ./lib/getTestMatrix.mjs", + "ci:build-matrix-optional": "node ./lib/getTestMatrix.mjs --optional=true", "ci:copy-to-temp": "ts-node ./ciCopyToTemp.ts", "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.nuxt,.sveltekit,.react-router,.astro,.output,pnpm-lock.yaml,.last-run.json,test-results,.angular,event-dumps}", "clean:pnpm": "pnpm store prune" diff --git a/nx.json b/nx.json index 7cd807e089fb..08faf7cf2e62 100644 --- a/nx.json +++ b/nx.json @@ -61,7 +61,7 @@ } }, "$schema": "./node_modules/nx/schemas/nx-schema.json", - "cacheDirectory": ".nxcache", + "cacheDirectory": ".nx/cache", "tui": { "autoExit": true }, diff --git a/packages/nestjs/package.json b/packages/nestjs/package.json index 625f8330a65f..024e25cf3cce 100644 --- a/packages/nestjs/package.json +++ b/packages/nestjs/package.json @@ -85,5 +85,24 @@ "volta": { "extends": "../../package.json" }, + "nx": { + "targets": { + "build:types": { + "inputs": [ + "production", + "^production" + ], + "dependsOn": [ + "^build:types" + ], + "outputs": [ + "{projectRoot}/build/types", + "{projectRoot}/build/types-ts3.8", + "{projectRoot}/*.d.ts" + ], + "cache": true + } + } + }, "sideEffects": false }