diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..b2628b2e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,47 @@ +# Build context for tools/e2e/Dockerfile, whose build context is the repository root. +# +# It lives here rather than beside the Dockerfile because BuildKit only reads +# `/.dockerignore` or `.dockerignore`, and this is the location that works on +# every toolchain. A plain `.dockerignore` next to the Dockerfile is read by nothing and fails +# silently, shipping the entire working tree. +# +# Deliberately a denylist. An allowlist omits files silently: forgetting `tools/builders/` breaks +# `yarn install`'s postinstall, and forgetting `packages/e2e/utils/` breaks deep inside `ng serve` +# with an error that points nowhere near the cause. +# +# NOTE: these patterns are not recursive by default. A bare `node_modules` would match only the +# repository root and miss `.opencode/node_modules` (56 MB, self-ignored so it never appears in +# `git status`) — hence the `**/` prefixes. +# +# Verify with: +# docker build --progress=plain --no-cache -f tools/e2e/Dockerfile \ +# --build-arg PLAYWRIGHT_VERSION= . 2>&1 | grep "transferring context" +# Expect roughly 57 MB. Substantially more means one of the patterns below stopped matching. + +**/node_modules +**/dist +**/*.log + +.git +.angular +.nx +.yarn/cache +.yarn/install-state.gz +.yarn/unplugged + +# Playwright outputs. The container produces its own; the committed __screenshots__ baselines it +# compares against are deliberately not excluded. +blob-report +playwright-report +playwright-report-docs +test-results +.playwright-mcp + +# Editor, agent and local tooling state. +.ai +.claude +.idea +.opencode +.vscode +coverage +tmp diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..b0aa6ad40 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# Deliberately narrow. `* text=auto` is tempting but would renormalize every tracked file in a +# single commit, so line endings are left alone except where they are load-bearing. + +# The Docker build inputs are consumed by a Linux shell. A contributor with core.autocrlf=true +# would otherwise commit CRLF into the Dockerfile's `RUN` continuations and the yarn shim it +# writes. +tools/e2e/** text eol=lf + +# Git already detects these as binary; declaring it means no future filter or `text=auto` change +# can start mangling the screenshot baselines, which are compared byte-for-byte at threshold: 0. +*.png binary diff --git a/.github/workflows/e2e-approve-snapshots.yml b/.github/workflows/e2e-approve-snapshots.yml index 75c30959a..48bee98ca 100644 --- a/.github/workflows/e2e-approve-snapshots.yml +++ b/.github/workflows/e2e-approve-snapshots.yml @@ -27,13 +27,23 @@ jobs: - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: message: 🔄 [Updating](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) snapshots. - - uses: ./.github/workflows/actions/setup-node - # Same as e2e.yml: no --with-deps, and a cap so a stalled download fails fast. - - run: yarn run e2e:setup - timeout-minutes: 20 + # Regenerated in the same container that e2e.yml compares against. Doing it on the bare + # runner instead would mean the baselines are written by one renderer and checked by another, + # and this workflow would happily commit screenshots that fail the very next run. + # + # docker compose creates a missing bind-mount source as root; packages/components already + # exists from the checkout, so only the report directories need creating up front. + - run: mkdir -p playwright-report test-results + # npm rather than yarn: no setup-node here, so the repository's Yarn 4 release is not on + # PATH. The container does its own install. e2e:docker:update-snapshots additionally mounts + # packages/components, which is how the rewritten PNGs reach the working tree for the commit + # step below. - id: update-snapshots - run: | - yarn run e2e:components --update-snapshots + run: npm run e2e:docker:update-snapshots + env: + # As in e2e.yml: the compose default is tuned for developer machines, the runner wants + # its own core count. + PLAYWRIGHT_WORKERS: 100% - uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0 id: commit-and-push with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e4abe5485..73e544e61 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -11,30 +11,49 @@ permissions: pull-requests: write jobs: + # Runs in the container built from tools/e2e/, not on the runner directly. The screenshots are + # compared with threshold: 0 against baselines that carry no {platform} suffix, so the thing that + # produces them has to be pinned; a bare runner is only pinned by whatever `ubuntu-latest` happens + # to mean this week. The same image is what `yarn run e2e:docker` gives a developer locally, which + # is the point — a failure here is reproducible off CI. + # + # Regeneration must go through the same image: see .github/workflows/e2e-approve-snapshots.yml. tests: runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: read # for actions/checkout to read the repository + pull-requests: write # for thollander/actions-comment-pull-request to comment on PRs steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/workflows/actions/setup-node - # e2e:setup deliberately omits --with-deps: that shells out to sudo apt-get to install the - # browser's system libraries, and the ubuntu-latest image already ships Chrome, Chromium, - # Edge and Firefox, so they are present before the job starts. Please do not add it back - # without checking the runner image first. + # No setup-node and no browser install: node is already on the runner, tools/e2e/run.js only + # reads package.json, and the browsers come baked into the image. That removes the ~174 MB + # `playwright install` download this job used to nurse through a timeout — but it is not a + # net time saving and should not be read as one. GitHub-hosted runners keep no Docker layer + # cache between runs, so every run rebuilds the image from scratch: the Node install, the + # font layer and `yarn install --immutable` all re-run, and the base image is pulled again + # (872 MB compressed, 3.6 GB on disk). Measured cold on a 64-core machine, the build alone is + # ~157s, and a 4-vCPU runner will be slower. Against the old job this is roughly a wash. # - # Capping the step means a stalled download fails fast instead of consuming the whole job - # budget and taking the test run down with it — which is what happened in run 30790866073, - # where the job hit the wall during setup and produced no report at all. The cap only bites - # while it stays below the job timeout. + # What is bought with that is reproducibility, not speed. If the wall clock ever does become + # the problem, the answer is a prebuilt image pulled from GHCR by tag — not + # `cache-to: type=gha`, which would push well over a gigabyte of layers into the same 10 GB + # Actions cache that every other job's yarn cache is competing for. # - # 8 minutes was too tight: the ~174 MB browser download alone has been overrunning it, so - # the cap was killing otherwise-healthy runs. Keep this comfortably above the observed - # duration — it is a guard against a hang, not a performance budget. - - run: yarn run e2e:setup - timeout-minutes: 20 + # docker compose creates a missing bind-mount source as root. Creating them up front keeps + # the workspace owned by the runner user, which matters on self-hosted runners where the + # next job has to clean it up. + - run: mkdir -p playwright-report test-results + # npm rather than yarn: without setup-node the repository's Yarn 4 release is never put on + # PATH, and the runner's own `yarn` is v1, which cannot read this manifest. Nothing is + # installed here either — the container does its own yarn install. - id: run-e2e-tests - run: | - yarn run e2e:components + run: npm run e2e:docker + env: + # Back to the runner's own setting. The compose file caps workers for developer machines, + # where a container sees far more cores than one dev server can be driven from; a 4-vCPU + # runner has the opposite problem and wants all of them. + PLAYWRIGHT_WORKERS: 100% - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} id: upload-report diff --git a/AGENTS.md b/AGENTS.md index 20134bfab..0db22009a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,8 +91,17 @@ yarn run e2e:setup # Install Playwright browsers (run once) yarn run e2e:components # Run all component E2E tests yarn run e2e:docs # Run the docs site smoke suite (needs `yarn run docs:build` first) npx playwright test # Run specific E2E tests (e.g., npx playwright test packages/components/button/e2e.playwright-spec.ts) + +# Screenshots differ across operating systems — always use Docker for anything visual: +yarn run e2e:docker # Run E2E tests in Docker (matches CI) +yarn run e2e:docker:update-snapshots # Run E2E tests in Docker and update the baselines ``` +The committed baselines under `__screenshots__` are compared with `threshold: 0` and have no +platform suffix, so a native run outside Linux fails on font rasterization alone. `e2e:components` +is still useful for the assertion-based specs; use `e2e:docker` whenever screenshots are involved, +and never regenerate a baseline any other way. + ### Linting ```bash diff --git a/docs/guides/06-testing.md b/docs/guides/06-testing.md index 21c57ed56..522bcb92c 100644 --- a/docs/guides/06-testing.md +++ b/docs/guides/06-testing.md @@ -49,3 +49,57 @@ exist first: yarn run docs:build yarn run e2e:docs ``` + +### Visual regression tests and Docker + +The screenshot baselines committed under `__screenshots__` are compared with `threshold: 0` and carry +no platform suffix, so they are tied to one operating system and one browser build. Running the suite +natively on Windows or macOS compares your machine's font rasterization against Linux bytes and fails +regardless of whether anything actually changed. + +Run anything visual in Docker instead. The image is built from the Playwright release matching +`@playwright/test` in `package.json`, which is what CI runs too: + +```bash +yarn run e2e:docker +``` + +To accept intentional visual changes, regenerate the baselines the same way and commit the result: + +```bash +yarn run e2e:docker:update-snapshots +``` + +Arguments are passed through, replacing the container's command — for example, to run one component: + +```bash +yarn run e2e:docker yarn playwright test packages/components/button +``` + +The container always runs with `CI=true`, so that Playwright behaves the way it does on the runner. +Two consequences matter when debugging inside it: `test.only` is rejected outright rather than +honoured (`forbidOnly`), and a failing test is retried twice before being reported. Narrow a run with +a path and `-g` instead of `test.only`: + +```bash +yarn run e2e:docker yarn playwright test packages/components/select -g "single select" +``` + +Requires Docker with Compose v2. On Windows, Docker Engine installed inside WSL puts no `docker.exe` +on the Windows PATH, so run these commands from inside the WSL distribution rather than from +PowerShell. + +### Worker count + +A container reports every core on the host, and Playwright sizes its worker pool from that. Since all +workers drive one shared Angular dev server, the useful ceiling comes from that server rather than +from the core count — on a 32-core machine `workers: '100%'` means 64 browsers, and the suite +collapses into timeouts that look like failures but are not. The compose file therefore caps workers +at 8. Override it when a machine wants something different: + +```bash +PLAYWRIGHT_WORKERS=16 yarn run e2e:docker +``` + +Baselines can also be regenerated without a local Docker install by commenting `/approve-snapshots` +on a pull request. diff --git a/package.json b/package.json index efd70ea79..6d9a97606 100644 --- a/package.json +++ b/package.json @@ -290,6 +290,8 @@ "dev:e2e": "ng serve dev-e2e", "e2e:setup": "playwright install chromium --with-deps && playwright install webkit --with-deps", "e2e:components": "playwright test packages/components", + "e2e:docker": "node tools/e2e/run.js", + "e2e:docker:update-snapshots": "node tools/e2e/run.js yarn run e2e:components --update-snapshots", "e2e:docs": "playwright test --config playwright.docs.config.ts", "serve:docs": "node tools/serve-docs.mjs", "-----API-----": "--------------------------------------------------------------------------------------------", diff --git a/packages/e2e/README.md b/packages/e2e/README.md index baefd4ef1..077482ce6 100644 --- a/packages/e2e/README.md +++ b/packages/e2e/README.md @@ -25,3 +25,24 @@ yarn run e2e:components # Run a specific E2E test file yarn playwright test packages/components/button/e2e.playwright-spec.ts ``` + +## Screenshots + +The baselines under each component's `__screenshots__` directory are compared with `threshold: 0` and +have no platform suffix, so they belong to one operating system and one browser build. The commands +above only compare them meaningfully on Linux; anywhere else they fail on font rasterization alone. + +Run anything visual in Docker, which uses the Playwright image matching `@playwright/test` and is what +CI runs as well: + +```bash +# Run the suite in Docker +yarn run e2e:docker + +# Accept intentional visual changes and rewrite the baselines +yarn run e2e:docker:update-snapshots +``` + +Requires Docker with Compose v2. On Windows, Docker Engine installed inside WSL puts no `docker.exe` on +the Windows PATH, so run these from inside the WSL distribution. Without a local Docker install, +comment `/approve-snapshots` on a pull request to regenerate the baselines in CI. diff --git a/playwright.config.ts b/playwright.config.ts index 376270c25..71ba09f1e 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -8,6 +8,49 @@ const viewport: ViewportSize = { const baseURL = process.env.BASE_URL || 'http://localhost:4200'; const webServerCommand = process.env.WEB_SERVER_COMMAND || 'yarn run dev:e2e --configuration=production'; +/** + * Every worker drives its own browser against one shared Angular dev server, so the useful ceiling + * comes from that server rather than from the core count. '100%' suits a 4-vCPU CI runner, but not + * Docker: a container reports every core on the host (Playwright reads `os.cpus()`, which no cgroup + * or cpuset limit affects), so on a 32-core machine it means 64 browsers and the suite collapses + * into timeouts. tools/e2e's compose file caps it via PLAYWRIGHT_WORKERS and CI sets it back. + * + * Playwright only accepts a string when it is a percentage, so anything else has to become a number. + * With the variable unset this behaves exactly as it did before. + * + * The value is validated rather than passed through, because Playwright's own guard only rejects + * `workers <= 0` — and `NaN <= 0` is false. A typo like `PLAYWRIGHT_WORKERS=amx` would therefore + * reach the dispatcher's `for (i = 0; i < workers; i++)` loop, spawn zero workers, run zero tests, + * write no report, and still exit 0: a green suite that tested nothing. + */ +const resolveWorkers = () => { + const override = process.env.PLAYWRIGHT_WORKERS?.trim(); + + if (!override) { + return isCI ? '100%' : undefined; + } + + if (override.endsWith('%')) { + const percentage = Number(override.slice(0, -1)); + + if (!Number.isFinite(percentage) || percentage <= 0) { + throw new Error(`PLAYWRIGHT_WORKERS must be a positive percentage, got ${JSON.stringify(override)}.`); + } + + return override; + } + + const workers = Number(override); + + if (!Number.isInteger(workers) || workers <= 0) { + throw new Error( + `PLAYWRIGHT_WORKERS must be a positive integer or a percentage, got ${JSON.stringify(override)}.` + ); + } + + return workers; +}; + /** @see https://playwright.dev/docs/test-configuration */ export default defineConfig({ testDir: __dirname, @@ -17,7 +60,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: isCI, retries: isCI ? 2 : 0, - workers: isCI ? '100%' : undefined, + workers: resolveWorkers(), reporter: [ ['list', { printSteps: true }], ['html', { open: 'never' }] diff --git a/tools/e2e/Dockerfile b/tools/e2e/Dockerfile new file mode 100644 index 000000000..98247bcc6 --- /dev/null +++ b/tools/e2e/Dockerfile @@ -0,0 +1,118 @@ +# syntax=docker/dockerfile:1.7 +# +# Runs the Playwright component suite in the same browser, operating system and font stack as CI, +# so that `expect.toHaveScreenshot` produces identical pixels on any developer machine. +# +# Build it through `yarn run e2e:docker` (tools/e2e/run.js), which derives PLAYWRIGHT_VERSION from +# package.json so the image tag cannot drift from the installed @playwright/test. +ARG PLAYWRIGHT_VERSION + +# Digest-pinned deliberately. The suite compares with `threshold: 0`, which makes the base image +# part of the compiler: were Microsoft to re-push this tag over a newer apt snapshot, every +# committed screenshot would change with no commit to blame it on. +# +# Once a digest is present Docker resolves it and ignores the tag, so the tag is documentation and +# the browser-revision assertion below is what actually keeps the two honest. When bumping +# @playwright/test, bump this digest in the same pull request and regenerate the baselines there: +# +# docker buildx imagetools inspect mcr.microsoft.com/playwright:v-noble --format '{{.Manifest.Digest}}' +# +# noble is Ubuntu 24.04, matching the `ubuntu-latest` runner the current baselines came from. +FROM mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble@sha256:b27e719ecbfef153e13fd24e8341736733bf2658b229677eb21ff57ff5d7fb29 + +WORKDIR /app + +# CI=true selects the same Playwright behavior as the runner: retries: 2, forbidOnly, and +# reuseExistingServer: false. It would also mean workers: '100%', which is wrong in a container — +# see the PLAYWRIGHT_WORKERS note in tools/e2e/docker-compose.yml. +# +# forbidOnly is the one that surprises people: `test.only` is rejected here rather than honoured, so +# a run is narrowed with a path and `-g`. Parity is worth more than the convenience — the whole +# point of this image is that a local run and the runner cannot disagree — but it is documented in +# docs/guides/06-testing.md rather than left to be discovered. +# +# TZ is belt-and-braces rather than a fix: the base image currently resolves to Etc/UTC, matching +# the runner, even though it is built with `ARG TZ=America/Los_Angeles`. Since it is the build +# argument and not the image that says otherwise, that agreement is incidental — and +# playwright.config.ts pins neither `locale` nor `timezoneId`, so the datepicker, timepicker and +# timezone screenshots would silently follow the base image if it ever changed. Stating it costs +# nothing. LANG and LC_ALL are already C.UTF-8 in the base image. +# +# HUSKY=0 because `prepare: husky` runs during `yarn install` and there is no .git in the context. +ENV CI=true \ + TZ=UTC \ + HUSKY=0 \ + YARN_ENABLE_TELEMETRY=0 + +# Body text is bundled woff2 (@fontsource/inter, @fontsource/jetbrains-mono), so system fonts only +# matter for glyphs Inter lacks — but the select and tree-select captions render the macOS modifier +# symbols ⌘ (U+2318) and ⌥ (U+2325). The base image covers neither in a Latin face, so fontconfig +# falls through to IPAGothic and they come out visibly different from the baselines. ubuntu-latest +# ships DejaVu, which is what the committed screenshots were rendered with; without this line six +# of them fail for reasons that have nothing to do with the components. +# +# The two fc-match calls are the assertion, not a demonstration: they are what stops this layer +# from being "cleaned up" later on the reasonable-sounding grounds that the fonts are bundled. A +# regression here is otherwise silent until six screenshots disagree for no visible reason. +RUN apt-get update \ + && apt-get install -y --no-install-recommends fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* \ + && fc-cache -f \ + && for codepoint in 2318 2325; do \ + fc-match ":charset=$codepoint" family | grep -q '^DejaVu Sans$' \ + || { echo "U+$codepoint falls back to '$(fc-match ":charset=$codepoint" family)', expected DejaVu Sans." >&2; exit 1; }; \ + done + +# The base image ships Node 22 from NodeSource. Every other CI job runs the .nvmrc version, so +# install that rather than accept a second Node major for this one job. Its own layer, ahead of the +# manifests, so that editing package.json does not reinstall Node. +# `n` keeps every version it fetches under /usr/local/n, which is 202 MB of no use once the binary +# is in place. Dropped in the same layer, so the bytes are never committed in the first place. +COPY .nvmrc ./ +RUN set -eux; \ + node_version="$(tr -d '\r\n' < .nvmrc)"; \ + npm install --global n; \ + n "$node_version"; \ + npm remove --global n; \ + rm -rf /usr/local/n; \ + npm cache clean --force; \ + test "v$node_version" = "$(node --version)"; \ + node --version + +COPY .yarnrc.yml package.json yarn.lock ./ +COPY .yarn/releases/ .yarn/releases/ + +# The base image has Yarn 1 classic on PATH from its own `npm install -g yarn`. It cannot read a +# Berry lockfile, and playwright.config.ts's webServer command (`yarn run dev:e2e ...`) shells out +# to whatever `yarn` resolves to at test time — so replacing it is required, not a convenience. +# +# A shell script rather than a symlink: the executable bit on the committed .cjs is not guaranteed +# to survive a COPY from a Windows build context. The path is read out of .yarnrc.yml, so this +# cannot drift from `yarnPath`. +RUN set -eux; \ + yarn_path="$(sed -n 's/^yarnPath: *//p' .yarnrc.yml | tr -d '\r')"; \ + test -f "/app/$yarn_path"; \ + printf '#!/bin/sh\nexec node "/app/%s" "$@"\n' "$yarn_path" > /usr/local/bin/yarn; \ + chmod +x /usr/local/bin/yarn; \ + test "4" = "$(yarn --version | cut -d . -f 1)"; \ + yarn --version + +# package.json's postinstall compiles these into node_modules/@koobiq/builders. +COPY tools/builders/ tools/builders/ +COPY tools/e2e/assert-browsers.js tools/e2e/ + +# For whoever debugs a failure here: `xlsx` resolves to https://cdn.sheetjs.com/, not the npm +# registry, so this layer needs that host reachable as well as the registry. +# +# The cache is cleaned in the same layer rather than a later one: node_modules is already +# materialised by then (nodeLinker: node-modules), and Yarn's global cache is another 640 MB that +# would otherwise be committed and then paid for again on every image export and pull. +RUN yarn install --immutable && yarn cache clean --all + +# Fails the build if the image's browsers are not the ones this @playwright/test expects, which is +# otherwise silent and surfaces much later as an unexplained diff in every screenshot. +RUN node tools/e2e/assert-browsers.js + +COPY . . + +CMD ["yarn", "run", "e2e:components"] diff --git a/tools/e2e/assert-browsers.js b/tools/e2e/assert-browsers.js new file mode 100644 index 000000000..6720c3aa3 --- /dev/null +++ b/tools/e2e/assert-browsers.js @@ -0,0 +1,69 @@ +/** + * Asserts that the browsers baked into the Playwright base image are the ones the installed + * playwright-core expects. Run at image build time by tools/e2e/Dockerfile. + * + * The image ships its browsers under /ms-playwright, so `playwright install` never runs in the + * container. That only holds while the image tag matches @playwright/test exactly. When it does + * not, nothing fails loudly: `playwright`'s postinstall quietly downloads a second browser set, + * the tests run against a different build than CI, and the result is an unexplained diff in every + * screenshot — the baselines are compared with `threshold: 0`, so a different browser build + * invalidates all of them at once. + * + * This check turns all of that into one build failure with an actionable message. + */ + +const { existsSync, readFileSync } = require('node:fs'); +const { dirname, join } = require('node:path'); + +// playwright-core does not list browsers.json in its "exports" map, so requiring it by subpath +// throws ERR_PACKAGE_PATH_NOT_EXPORTED. package.json is exported; resolve that and read the file +// next to it, which stays correct wherever the package is installed. +const { browsers } = JSON.parse( + readFileSync(join(dirname(require.resolve('playwright-core/package.json')), 'browsers.json'), 'utf8') +); + +// Everything `playwright install` would fetch, rather than a hand-maintained list. Chromium is the +// obvious one, but it is not the only browser this suite uses: the scrollbar and sidepanel specs +// select WebKit with `test.use({ browserName: 'webkit' })`, which overrides the project's browser +// per file and is easy to miss when reading playwright.config.ts alone. Deriving the list means a +// spec that reaches for Firefox tomorrow is covered without anyone remembering to edit this file. +// +// The excluded entries are the ones the image legitimately lacks: tip-of-tree and beta channels, +// `android`, and `winldd` (Windows-only). +const required = browsers.filter((browser) => browser.installByDefault); +const missing = []; + +if (required.length === 0) { + console.error('No installByDefault browsers in playwright-core/browsers.json — the check cannot be trusted.'); + process.exit(1); +} + +for (const { name, revision } of required) { + // playwright-core stores revisions per browser name; on disk the directories use underscores. + const directory = `/ms-playwright/${name.replace(/-/g, '_')}-${revision}`; + + if (existsSync(directory)) { + console.log(`ok ${directory}`); + } else { + missing.push(directory); + } +} + +if (missing.length > 0) { + console.error( + [ + 'The base image does not ship the browsers this @playwright/test expects.', + `Missing: ${missing.join(', ')}`, + '', + 'The pinned digest in tools/e2e/Dockerfile is stale relative to the @playwright/test', + 'version in package.json. Refresh it with:', + '', + ' docker buildx imagetools inspect mcr.microsoft.com/playwright:v-noble \\', + " --format '{{.Manifest.Digest}}'", + '', + 'and regenerate the screenshot baselines in the same pull request.' + ].join('\n') + ); + + process.exit(1); +} diff --git a/tools/e2e/docker-compose.update.yml b/tools/e2e/docker-compose.update.yml new file mode 100644 index 000000000..360285402 --- /dev/null +++ b/tools/e2e/docker-compose.update.yml @@ -0,0 +1,21 @@ +# Overlay applied only by `yarn run e2e:docker:update-snapshots` — see tools/e2e/run.js, which +# adds this file when it sees --update-snapshots in the arguments. +# +# Every one of the __screenshots__ directories lives under packages/components, so a single mount +# is enough for Playwright's --update-snapshots to write the new baselines back to the working +# tree. +# +# Kept out of the base file on purpose. Two reasons, in order of weight: a plain test run has no +# business being able to write into the working tree at all, and the mount puts the component +# library on the Angular build's hot path, which costs about 5 seconds per run across the host +# filesystem boundary (measured: 27s against 32s for the same spec). Updating snapshots is rare and +# can afford both; a plain run should pay for neither. +services: + e2e: + # The report mounts are repeated from the base file on purpose. Compose has merged sequences + # by mount target in some versions and replaced them wholesale in others; spelling all three + # out is correct either way, and a repeated identical target is a no-op when they are merged. + volumes: + - ../../packages/components:/app/packages/components + - ../../playwright-report:/app/playwright-report + - ../../test-results:/app/test-results diff --git a/tools/e2e/docker-compose.yml b/tools/e2e/docker-compose.yml new file mode 100644 index 000000000..d5468e6a5 --- /dev/null +++ b/tools/e2e/docker-compose.yml @@ -0,0 +1,54 @@ +# Runs the Playwright component suite in the image built from tools/e2e/Dockerfile. +# +# Invoke it through `yarn run e2e:docker` (tools/e2e/run.js), which supplies PLAYWRIGHT_VERSION +# from package.json and checks it is an exact version. Running `docker compose` against this file +# directly leaves that build argument empty, so the tag half of the image reference becomes +# `v-noble`; the pinned digest still resolves the correct image, but the tag no longer records +# which @playwright/test the browsers belong to, and nothing then verifies the two agree. + +# Named explicitly because Compose otherwise derives the project name from this file's directory, +# which is just `e2e` — shared with every other repository that puts its compose file in tools/e2e, +# and enough for two projects to start reusing each other's containers and networks. +name: koobiq-e2e + +services: + e2e: + # The committed baselines are x86-64. Pinning the platform stops an arm64 machine from quietly + # producing renders that differ from CI in a way no diff explains; on Apple Silicon that means + # emulation, which is slow but correct. Override only if you know why. + platform: ${E2E_PLATFORM:-linux/amd64} + build: + context: ../.. + dockerfile: tools/e2e/Dockerfile + args: + - PLAYWRIGHT_VERSION + # Reaps the Chromium processes that would otherwise pile up under PID 1. + init: true + # Recommended upstream for Chromium. Note that the usual justification — Docker's 64 MB + # /dev/shm — does not apply here: Playwright passes --disable-dev-shm-usage unconditionally, so + # Chromium uses /tmp regardless. Kept because upstream still recommends it and it costs nothing. + ipc: host + environment: + # A container resolves `localhost` to both 127.0.0.1 and ::1, and Node >= 17 does not reorder + # them. If the Angular dev server binds v4 only and Playwright's readiness probe tries ::1 + # first, the run dies after the 10-minute webServer timeout with nothing useful in the log. + # Both variables are read by playwright.config.ts. + BASE_URL: http://127.0.0.1:4200 + WEB_SERVER_COMMAND: yarn run dev:e2e --configuration=production --host 127.0.0.1 + # A container reports every core on the host — Playwright reads os.cpus(), which no cgroup or + # cpuset limit affects — so playwright.config.ts's '100%' would mean 64 browsers against one + # dev server on a 32-core machine, and every test times out. Measured on such a machine: 64 + # workers gave 255 failures and 802 timeouts, 8 workers gave zero of either. CI overrides this + # back to '100%', which on a 4-vCPU runner is 4. + # + # 8 rather than a higher guess because it is the value that was actually measured clean; the + # ceiling is the single dev server, not the core count, so raising it buys little. + PLAYWRIGHT_WORKERS: ${PLAYWRIGHT_WORKERS:-8} + # Outputs only — the source tree comes from the image, deliberately, so a plain run cannot write + # anywhere near the working tree. Mounting the sources as well costs about 5 seconds per run + # (measured: 27s against 32s for the same spec), because `ng serve dev-e2e` compiles the whole + # component library and those ~3200 files then cross the host filesystem boundary. Writing + # updated baselines back is docker-compose.update.yml's job instead. + volumes: + - ../../playwright-report:/app/playwright-report + - ../../test-results:/app/test-results diff --git a/tools/e2e/run.js b/tools/e2e/run.js new file mode 100644 index 000000000..51d6c9ff7 --- /dev/null +++ b/tools/e2e/run.js @@ -0,0 +1,118 @@ +/** + * Runs the Playwright component suite inside Docker, so that screenshots are produced by the same + * browser, operating system and font stack as CI. + * + * Screenshots are compared with `threshold: 0` and the baselines carry no `{platform}` suffix, so a + * native run on Windows or macOS compares its own rasterization against Linux bytes and fails on + * font rendering alone. This wrapper is the supported way to run — and the only supported way to + * update — the committed baselines. + * + * The image tag comes from package.json rather than being hardcoded: the browsers baked into the + * image have to match the installed @playwright/test exactly, and a tag that has drifted from the + * manifest is indistinguishable from a genuine visual regression. + * + * node tools/e2e/run.js # run the suite (the image's CMD) + * node tools/e2e/run.js # replace the image's CMD + * + * Anything after the script name replaces the container's command rather than being appended to + * it — that is how `docker compose run` behaves — which is why the update-snapshots script passes + * the whole command and not just the flag. + */ + +const { spawnSync } = require('node:child_process'); +const { existsSync } = require('node:fs'); +const { join } = require('node:path'); +const { devDependencies } = require('../../package.json'); + +const TIME_LABEL = 'Runtime'; +const COMPOSE_FILE = join(__dirname, 'docker-compose.yml'); +const COMPOSE_UPDATE_FILE = join(__dirname, 'docker-compose.update.yml'); + +// Read straight from the manifest rather than from node_modules: CI runs this without an install +// step, so the packages are not on disk. +const version = devDependencies['@playwright/test']; + +// The tag has to be exact. A range would either not resolve to a tag at all or, worse, resolve to +// an image whose browsers differ from the ones the lockfile installs. +if (!/^\d+\.\d+\.\d+$/.test(version)) { + console.error( + `Expected devDependencies["@playwright/test"] in package.json to be an exact version, got ${JSON.stringify(version)}.` + ); + process.exit(1); +} + +// Both prerequisites are checked up front, because neither fails in a way that explains itself. +// A missing `docker` surfaces as a bare ENOENT from spawn; a Docker CLI without the v2 compose +// plugin — a machine carrying only the legacy `docker-compose` binary — spawns fine and exits +// non-zero, which is indistinguishable from a genuine test failure further down. +const compose = spawnSync('docker', ['compose', 'version'], { stdio: 'ignore' }); + +if (compose.error?.code === 'ENOENT') { + console.error( + 'Could not find `docker` on PATH.' + + (process.platform === 'win32' + ? '\nWith Docker Engine installed inside WSL there is no docker.exe on the Windows PATH, ' + + 'so run this from inside the WSL distribution rather than from PowerShell.' + : '') + ); + process.exit(1); +} + +if (compose.error || compose.status !== 0) { + console.error( + '`docker compose` is unavailable. These scripts need Compose v2, which ships as a Docker\n' + + 'CLI plugin; the standalone `docker-compose` v1 binary cannot read this configuration.' + ); + process.exit(1); +} + +const args = process.argv.slice(2); + +// Writing baselines back to the working tree needs the source mounted; a plain run does not, and +// the mount is slow enough to matter. See tools/e2e/docker-compose.update.yml. +const isUpdatingSnapshots = args.some( + (arg) => arg === '-u' || arg === '--update-snapshots' || arg.startsWith('--update-snapshots=') +); + +console.info(`Playwright version: ${version}`); + +if (isUpdatingSnapshots) { + console.info('Mounting packages/components so updated baselines land in the working tree.'); +} + +console.time(TIME_LABEL); + +const result = spawnSync( + 'docker', + [ + 'compose', + '--file', + COMPOSE_FILE, + ...(isUpdatingSnapshots ? ['--file', COMPOSE_UPDATE_FILE] : []), + 'run', + '--rm', + '--build', + 'e2e', + ...args + ], + { + stdio: 'inherit', + env: { ...process.env, PLAYWRIGHT_VERSION: version } + } +); + +console.timeEnd(TIME_LABEL); + +if (result.error) { + console.error(`Failed to run docker: ${result.error.message}`); + process.exit(1); +} + +// Only when there is something to open: the run can also fail before any test executes — a build +// error, or an image that cannot be pulled — and pointing at a report that was never written sends +// whoever is debugging in the wrong direction. +if (result.status !== 0 && existsSync(join(__dirname, '../../playwright-report/index.html'))) { + console.info('To view the test report, run: `npx playwright show-report`'); +} + +process.exit(result.status ?? 1);