diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6fc0885..1bc0308 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,7 +54,7 @@ jobs: repository-cache: true - name: Run E2E and visual tests working-directory: examples/react - run: bazelisk test //:e2e_test //:visual_test //:component_test //:component_visual_test //:remote_integration_test --test_output=errors + run: bazelisk test //:e2e_test //:visual_test //:component_test //:component_visual_test //:remote_integration_test //:native_config_test //:native_visual_test --test_output=errors - name: Upload visual test artifacts if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/README.md b/README.md index dff5dbf..2412933 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ owns typechecking and bundling; the rules own execution and baseline updates. ```starlark load("@rules_web_e2e//e2e:defs.bzl", "web_e2e_test") load("@rules_web_e2e//component:defs.bzl", "browser_shell", "component_browser_test") -load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test") +load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test", "visual_test") web_e2e_test( name = "editor_e2e", tests = ":compiled_e2e_specs", - server = ":editor_test_server", + config = ":compiled_playwright_config", # use.baseURL + optional webServer ) browser_shell( @@ -42,6 +42,7 @@ component_visual_test( | Input | Contract | | --------------------------- | -------------------------------------------------------------------------------------------- | | `tests` | Built ESM specs and dependencies; source files are typechecked/transpiled by the producer | +| `config` | Compiled native Playwright config: base URL, optional webServer, fixtures, reporters, and timeouts | | `server` | Compiled adapter returning a ready URL and cleanup callback | | `shell` | Built HTML/JS/CSS directory plus its entry point; served without a bundler | | `base_url` / `base_url_env` | Existing application endpoint, replacing `server` or `shell` | @@ -57,6 +58,10 @@ const matching: VisualMatching = {threshold: 0.1, maxDiffPixels: 0} export default matching ``` +Use `visual_test(tests = ":compiled_visual_specs", config = ":compiled_playwright_config", ...)` +for page screenshots and interaction-driven VRT with native `toHaveScreenshot`. +It shares `matching`, `baselines`, `baseline_dir`, and `.update` with gallery VRT. + Native component specs mount registered visuals with `mount('moduleId/visualId', props)`. The same gallery supplies generated VRT captures. `ComponentVisualModule` and `installVisualGallery` are exported from diff --git a/docs/api.md b/docs/api.md index 36a5ef4..8e6ab90 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,13 +9,13 @@ browser, server lifecycle, reports, and baseline updates. ```starlark load("@rules_web_e2e//e2e:defs.bzl", "web_e2e_test") load("@rules_web_e2e//component:defs.bzl", "component_browser_test") -load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test") +load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test", "visual_test") ``` | Attribute | Default | Contract | | --------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------ | | `name` | Required | Test target name | -| `tests` | Required for E2E/component | Target containing compiled ESM `*.spec.js` and its dependencies; source JavaScript is rejected | +| `tests` | Required except gallery VRT | Target containing compiled ESM `*.spec.js` and its dependencies; source JavaScript is rejected | | `server` | Unset | Target supplying one compiled default `ServerAdapter` export, with its runtime dependencies/data | | `shell` | Unset | `browser_shell` target supplying built HTML, JavaScript, CSS, and other assets | | `base_url` | Unset | Existing HTTP(S) app URL; caller owns readiness and lifetime | @@ -24,12 +24,16 @@ load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test") | `config` | Generated by the runner | Optional compiled ESM module exporting native Playwright configuration | | `data` | `[]` | Additional declared runtime files | | `env`, `env_inherit` | `{}`, `[]` | Explicit values and inherited variable names | +| `network_origins_env` | `[]` | Explicit variable names containing optional extra HTTP(S) origins; unset values add nothing | +| `args` | `[]` | Default test selection flags; same allowlist as `--test_arg` | | `network_origins` | `[]` | Extra HTTP(S) origins permitted through the browser tunnel | | `execution_timeout_seconds` | `180` | Deadline per Playwright invocation; discovery and capture have separate limits | | `timeout` | `"long"` | Independent Bazel test timeout category | | `tags` | `[]` | Additional tags; local/manual/uncached restrictions remain | -Choose exactly one of `server`, `shell`, `base_url`, or `base_url_env`. +Choose one of `server`, `shell`, `base_url`, or `base_url_env`, or supply only a +`config` with `use.baseURL` and optional native `webServer`. These source attributes +are mutually exclusive; the config-only form avoids repeating server settings. E2E selects compiled `*.spec.js` excluding `*.browser.spec.js` and `*.visual.spec.js`; component tests select `*.browser.spec.js`. Helpers can be included in the compiled target without being treated as specs. The producer @@ -37,7 +41,7 @@ must include typechecking in its build graph; `ts_project` emitted outputs do this, and the rules also request available `transitive_typecheck` outputs. Shell producers must similarly depend on their typecheck action. -Only `component_visual_test` accepts: +Both `component_visual_test` and `visual_test` accept: | Attribute | Default | Contract | | -------------- | ------------------- | ------------------------------------------------------------------ | @@ -45,11 +49,16 @@ Only `component_visual_test` accepts: | `baselines` | `[]` | Existing PNG input labels | | `baseline_dir` | `"__screenshots__"` | Package-relative directory exclusively owned by this visual target | -Visual tests generate captures from the gallery; they do not need `tests`. -An explicit `.update` captures all enabled visuals before replacing PNGs +`component_visual_test` generates captures from the gallery and rejects `tests`. +`visual_test` runs compiled `*.spec.js` containing native `toHaveScreenshot` +assertions, including clicks and page navigation, without a gallery protocol. +An explicit `.update` runs the entire visual target before replacing PNGs and deleting stale PNGs. Other files remain. Missing baselines fail comparison. Visual filters are rejected. E2E/component targets forward `--grep`, -`--grep-invert`, `--project`, and `--shard` via `--test_arg`. +`--grep-invert`, `--project`, and `--shard` via `--test_arg`. They also accept +explicit declared spec paths (source `.spec.ts`/`.spec.tsx` names map to compiled +`.spec.js`) and `--pass-with-no-tests` for tag-filtered CI shards. These options +remain forbidden for visual targets. All targets also create `_sources` and `_inputs`; reserve those names. `visual` and `component` are private mode switches, not caller attributes. @@ -125,14 +134,26 @@ and readiness hooks remain in the visual declarations. ## Optional Playwright configuration -Most call sites need no config. To add fixtures, global setup, timeouts, or E2E +A config-only target sets `use.baseURL`; Playwright owns its optional `webServer` +startup, readiness checks, and teardown. Multiple servers are supported. Relative +server working directories, global setup/teardown, and reporter modules resolve +against the compiled config. Existing servers are never reused. Include server +executables, assets, and setup modules in the config's runfiles or `data`. +The runner inspects the config in a separate process before allocating a browser, +then Playwright loads it normally: keep top-level config evaluation declarative. +Projects must share the same origin; additional service origins still require an +explicit network allowlist. Compiled `.js` and `.mjs` config outputs are accepted. + +Most built-shell and adapter call sites need no config. To add fixtures, global setup, timeouts, or E2E projects, supply a compiled module exporting `PlaywrightTestConfig` from `@playwright/test`. ESM config/spec graphs must include their `package.json` module markers and runtime dependencies. Defaults: Chromium, headless, one worker, no retries, 30-second tests, 1280×720, en-US, UTC, light theme, reduced motion. Component tests block service -workers. VRT disables animations, hides the caret, and captures at CSS scale. +workers. VRT disables animations, hides the caret, and defaults to CSS-scale +screenshots. Set `expect.toHaveScreenshot.scale = 'device'` in the compiled +Playwright config to preserve device-pixel baselines at higher pixel densities. The runner retains ownership of discovery, browser connection, output paths, required list/JUnit reports, and snapshot policy when composing overrides. Visual targets reject Playwright projects; use separate targets and baseline directories instead. diff --git a/docs/architecture.md b/docs/architecture.md index 2edc4aa..cb3e34d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -80,11 +80,14 @@ storage, separate from committed baselines. | ------------------------ | ----------------------------------------------------- | | `web_e2e_test` | Native specs against a managed server or existing URL | | `component_browser_test` | Native mounts through a consumer gallery | +| `visual_test` | Native screenshot specs and baseline updates | | `component_visual_test` | Generated visual captures and baseline updates | See the [API reference](api.md) for attributes and configuration helpers. -Managed-server mode must own startup, readiness, ports, and teardown. Deployed +Server adapters own readiness and teardown; a config-only target delegates +its native `webServer` lifecycle to Playwright. Both paths share the same +runfiles staging, browser container, and exact-origin tunnel. Deployed mode must explicitly opt into network access and consumer-provided auth setup; it must not silently fall back to a local service or ambient credentials. diff --git a/docs/e2e.md b/docs/e2e.md index e028773..36910df 100644 --- a/docs/e2e.md +++ b/docs/e2e.md @@ -30,7 +30,10 @@ web_e2e_test( ) ``` -Alternatively supply a built `shell` or an existing URL. See the +For an existing Playwright setup, pass `config = ":compiled_config"` instead of +`server`. Set `use.baseURL` and optional `webServer` in that config; Playwright +starts and stops the declared server. Declare its executable and assets as data. +A built `shell` or existing URL is also supported. See the [setup guide](getting-started.md) and [all attributes](api.md). The runner selects emitted `*.spec.js`, excluding component/visual specs. It supplies `baseURL` and `VRT_APP_URL`; use `page.goto('./')` to preserve a @@ -117,3 +120,16 @@ options work with VRT, whose `.update` remains explicit. `//:remote_integration_test` in the React example starts an independent fixture on a random port and verifies base paths, interactions, blocked undeclared origins, and caller-owned server lifetime. CI needs no public test site. + +## Interaction-driven visual tests + +Use `visual_test` from `@rules_web_e2e//vrt:defs.bzl` for ordinary Playwright specs +that click around and call `expect(page).toHaveScreenshot('saved.png')`. Pass the +compiled specs, config (or server/shell/URL), matching policy, and baseline inputs. +`bazel run //:visual_test.update` replaces baselines only after the full suite succeeds. +The [native example](../examples/react/native.visual.spec.ts) exercises this path. + +If an extra service endpoint changes between environments, declare its variable +name in `network_origins_env = ["AUTH_ORIGIN"]`. Only named, nonempty variables +are read, and each must be an exact HTTP(S) origin without paths, credentials, +or wildcards. Static endpoints remain in `network_origins`. diff --git a/examples/react/BUILD.bazel b/examples/react/BUILD.bazel index 5f09581..c66d0a1 100644 --- a/examples/react/BUILD.bazel +++ b/examples/react/BUILD.bazel @@ -5,10 +5,46 @@ load("@npm//:defs.bzl", "npm_link_all_packages") load("@rules_web_e2e//component:defs.bzl", "browser_shell", "component_browser_test") load("@rules_web_e2e//e2e:defs.bzl", "web_e2e_test") load("@rules_web_e2e//playwright:defs.bzl", "playwright_runtime") -load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test") +load("@rules_web_e2e//vrt:defs.bzl", "component_visual_test", "visual_test") npm_link_all_packages(name = "node_modules") +js_library( + name = "native_config", + srcs = ["native.config.js"], + data = ["package.json"], + deps = [":typecheck_project"], +) + +js_library( + name = "native_specs", + srcs = ["native.spec.js"], + deps = [":typecheck_project"], +) + +js_library( + name = "native_visual_specs", + srcs = ["native.visual.spec.js"], + deps = [":typecheck_project"], +) + +web_e2e_test( + name = "native_config_test", + config = ":native_config", + tests = ":native_specs", +) + +visual_test( + name = "native_visual_test", + baseline_dir = "__native_screenshots__", + baselines = glob( + ["__native_screenshots__/*.png"], + allow_empty = True, + ), + config = ":native_config", + tests = ":native_visual_specs", +) + npm_link_package( name = "node_modules/@rules-web-e2e/vrt", src = "@rules_web_e2e//runtime:package", diff --git a/examples/react/__native_screenshots__/saved.png b/examples/react/__native_screenshots__/saved.png new file mode 100644 index 0000000..f1be403 Binary files /dev/null and b/examples/react/__native_screenshots__/saved.png differ diff --git a/examples/react/native-server.ts b/examples/react/native-server.ts new file mode 100644 index 0000000..88ce3f3 --- /dev/null +++ b/examples/react/native-server.ts @@ -0,0 +1,8 @@ +import {createServer} from 'node:http' + +createServer((_, response) => { + response.setHeader('content-type', 'text/html') + response.end( + '' + ) +}).listen(Number(process.env.PORT), '127.0.0.1') diff --git a/examples/react/native.config.ts b/examples/react/native.config.ts new file mode 100644 index 0000000..80404f2 --- /dev/null +++ b/examples/react/native.config.ts @@ -0,0 +1,19 @@ +import type {PlaywrightTestConfig} from '@playwright/test' + +const seed = process.env.TEST_TMPDIR ?? 'native-example' +const port = + 20_000 + + [...seed].reduce( + (hash, character) => (hash * 31 + character.charCodeAt(0)) % 20_000, + 0 + ) +const baseURL = `http://127.0.0.1:${port}` +const config: PlaywrightTestConfig = { + use: {baseURL}, + webServer: { + command: `"${process.execPath}" native-server.js`, + url: baseURL, + env: {PORT: String(port)}, + }, +} +export default config diff --git a/examples/react/native.spec.ts b/examples/react/native.spec.ts new file mode 100644 index 0000000..ca1b8a2 --- /dev/null +++ b/examples/react/native.spec.ts @@ -0,0 +1,9 @@ +import {expect, test} from '@playwright/test' + +test('native webServer serves the declared config endpoint', async ({page}) => { + await page.goto('/') + await page.getByRole('button', {name: 'Save', exact: true}).click() + await expect( + page.getByRole('button', {name: 'Saved', exact: true}) + ).toBeVisible() +}) diff --git a/examples/react/native.visual.spec.ts b/examples/react/native.visual.spec.ts new file mode 100644 index 0000000..64ccaeb --- /dev/null +++ b/examples/react/native.visual.spec.ts @@ -0,0 +1,9 @@ +import {expect, test} from '@playwright/test' + +test('native specs own the interaction before capture', async ({page}) => { + await page.goto('/') + await page.getByRole('button', {name: 'Save', exact: true}).click() + await expect( + page.getByRole('button', {name: 'Saved', exact: true}) + ).toHaveScreenshot('saved.png') +}) diff --git a/internal/browser.bzl b/internal/browser.bzl index 8329c49..b8956c0 100644 --- a/internal/browser.bzl +++ b/internal/browser.bzl @@ -28,7 +28,7 @@ browser_shell = rule( def _compiled(target, label): if not target: return None - files = [f for f in target[DefaultInfo].files.to_list() if f.extension == "js"] + files = [f for f in target[DefaultInfo].files.to_list() if f.extension in ["js", "mjs"]] if len(files) != 1 or files[0].is_source: fail(label + " must supply one compiled JavaScript module") return runfile(files[0]) @@ -91,13 +91,15 @@ def browser_test( env = {}, env_inherit = [], network_origins = [], + network_origins_env = [], tags = [], timeout = "long", execution_timeout_seconds = 180, + args = [], visual = False, component = False): """Internal common implementation; public wrappers select the test mode.""" - if any([key.startswith("VRT_") for key in env.keys() + env_inherit]): + if any([key.startswith("VRT_") for key in env.keys() + env_inherit + network_origins_env]): fail("VRT_* environment names are reserved for the browser runtime") if visual and component: fail("Visual and component modes are separate targets") @@ -107,15 +109,17 @@ def browser_test( fail("execution_timeout_seconds must be positive") if not baseline_dir or baseline_dir.startswith("/") or any([p in ["", ".", ".."] for p in baseline_dir.split("/")]): fail("baseline_dir must be a nonempty relative directory without dot segments") - if len([v for v in [server, shell, base_url, base_url_env] if v != None]) != 1: - fail("Supply exactly one of server, shell, base_url, or base_url_env") + sources = len([v for v in [server, shell, base_url, base_url_env] if v != None]) + if sources > 1 or (sources == 0 and not config): + fail("Supply one of server, shell, base_url, base_url_env, or a config with use.baseURL") if base_url_env != None and (not base_url_env or base_url_env.startswith("VRT_")): fail("base_url_env must be a nonempty consumer environment name") if base_url == "": fail("base_url must not be empty") if base_url_env and base_url_env not in env and base_url_env not in env_inherit: env_inherit = env_inherit + [base_url_env] - js_library(name = name + "_sources", srcs = baselines, data = data, deps = [tests] if tests else []) + env_inherit = env_inherit + [key for key in network_origins_env if key not in env and key not in env_inherit] + js_library(name = name + "_sources", srcs = baselines, data = data) _inputs( name = name + "_inputs", tests = tests, @@ -125,25 +129,27 @@ def browser_test( config = config, matching = matching, sources = ":" + name + "_sources", - mode = "visual" if visual else "component" if component else "e2e", + mode = "visual-spec" if visual and tests else "visual" if visual else "component" if component else "e2e", ) common = dict( copy_data_to_bin = False, entry_point = Label("//runtime:runner_entry"), - data = [":" + name + "_inputs", Label("//runtime:files")], + data = [":" + name + "_inputs", Label("//runtime:files")] + data, env = env | { "VRT_DESCRIPTOR": "$(rlocationpath :%s_inputs)" % name, "VRT_BASE_URL": base_url or "", "VRT_BASE_URL_ENV": base_url_env or "", - "VRT_MODE": "visual" if visual else "component" if component else "e2e", + "VRT_MODE": "visual-spec" if visual and tests else "visual" if visual else "component" if component else "e2e", "VRT_BASELINE_RELATIVE": (native.package_name() + "/" if native.package_name() else "") + baseline_dir if visual else "", "VRT_NETWORK_ORIGINS": json.encode(network_origins), + "VRT_NETWORK_ORIGINS_ENV": json.encode(network_origins_env), "VRT_ENV_NAMES": json.encode(env.keys() + env_inherit), "VRT_TIMEOUT_MS": str(execution_timeout_seconds * 1000), }, ) js_test( name = name, + args = args, env_inherit = ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_CONFIG"] + env_inherit, tags = ["manual", "external", "visual_test" if visual else "component_browser_test" if component else "e2e_test", "requires-network", "no-sandbox", "no-remote", "no-cache"] + tags, timeout = timeout, diff --git a/runtime/BUILD.bazel b/runtime/BUILD.bazel index 2f5c05f..51c335f 100644 --- a/runtime/BUILD.bazel +++ b/runtime/BUILD.bazel @@ -41,6 +41,7 @@ js_library( "baselines.js", "capture.js", "config.js", + "config-url.js", "container.js", "isolation.js", "matching.js", diff --git a/runtime/arguments.test.ts b/runtime/arguments.test.ts index 11a0d5c..f29c6e9 100644 --- a/runtime/arguments.test.ts +++ b/runtime/arguments.test.ts @@ -21,3 +21,25 @@ test('visual updates cannot silently select only part of the baseline set', () = assert.deepEqual(testArguments(true, ['--update']), []) assert.throws(() => testArguments(true, ['--update', '--grep=only-one'])) }) + +test('CI file selection maps source names to declared compiled specs only', () => { + const args = testArguments( + false, + ['app/[route].spec.ts', '--pass-with-no-tests'], + ['_main/app/[route].spec.js'] + ) + assert.equal( + new RegExp(args[0]).test('/staged/_main/app/[route].spec.js'), + true + ) + assert.equal( + new RegExp(args[0]).test('/staged/_main/app/[route].spec.ts'), + true + ) + assert.equal(new RegExp(args[0]).test('/staged/_main/app/r.spec.js'), false) + assert.equal(args[1], '--pass-with-no-tests') + assert.throws(() => + testArguments(false, ['other.spec.ts'], ['_main/app/test.spec.js']) + ) + assert.throws(() => testArguments(true, ['--pass-with-no-tests'])) +}) diff --git a/runtime/arguments.ts b/runtime/arguments.ts index 8d9eb36..97c9b53 100644 --- a/runtime/arguments.ts +++ b/runtime/arguments.ts @@ -1,5 +1,9 @@ /** Keep selection flags without allowing CLI overrides of managed paths/reporters. */ -export function testArguments(visual: boolean, args: string[]): string[] { +export function testArguments( + visual: boolean, + args: string[], + files: string[] = [] +): string[] { if (visual) { if (args.some(arg => arg !== '--update')) throw new Error( @@ -10,6 +14,26 @@ export function testArguments(visual: boolean, args: string[]): string[] { const result: string[] = [] for (let index = 0; index < args.length; index++) { const arg = args[index] + if (arg === '--pass-with-no-tests') { + result.push(arg) + continue + } + if (!arg.startsWith('-')) { + const file = arg.replace(/\.spec\.tsx?$/, '.spec.js') + if ( + !files.some( + declared => declared === file || declared.endsWith('/' + file) + ) + ) + throw new Error(`Spec selector is not a declared test input: ${arg}`) + // Playwright filters both loaded modules and source-mapped test locations. + result.push( + file + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + .replace(/\\\.js$/, '\\.(?:js|ts|tsx)') + '$' + ) + continue + } const flag = arg.split('=')[0] if (!['--grep', '--grep-invert', '--project', '--shard'].includes(flag)) throw new Error( diff --git a/runtime/config-url.ts b/runtime/config-url.ts new file mode 100644 index 0000000..1cccb8b --- /dev/null +++ b/runtime/config-url.ts @@ -0,0 +1,21 @@ +import {pathToFileURL} from 'node:url' +import type {PlaywrightTestConfig} from '@playwright/test' +import {remoteAppUrl} from './network.js' + +// Inspect in the same staged environment used by Playwright. Native webServer +// startup and teardown remain owned by Playwright, including multi-server setups. +const config = ( + await import(pathToFileURL(process.env.VRT_CONFIG_OVERRIDE!).href) +).default as PlaywrightTestConfig +const url = remoteAppUrl({VRT_BASE_URL: config.use?.baseURL}) +if (!url) throw new Error('A config-only target must set use.baseURL') +for (const project of config.projects ?? []) { + if ( + project.use?.baseURL && + new URL(project.use.baseURL).origin !== new URL(url).origin + ) + throw new Error( + 'Projects with different origins need separate browser targets' + ) +} +process.send?.({url}) diff --git a/runtime/config.test.ts b/runtime/config.test.ts index c7f8c19..66a5eab 100644 --- a/runtime/config.test.ts +++ b/runtime/config.test.ts @@ -39,7 +39,7 @@ test('component gallery resolves base paths without broadening browser access', }) test('compiled suite config preserves runner paths and accepts a pixel-count budget', async () => { - const {mkdtempSync, mkdirSync, writeFileSync, rmSync} = + const {mkdtempSync, mkdirSync, writeFileSync, rmSync, realpathSync} = await import('node:fs') const {tmpdir} = await import('node:os') const {join} = await import('node:path') @@ -48,10 +48,18 @@ test('compiled suite config preserves runner paths and accepts a pixel-count bud writeFileSync(join(temp, 'package.json'), '{"type":"module"}') writeFileSync(join(temp, 'matching.js'), 'export default {maxDiffPixels: 7}') writeFileSync(join(temp, 'reporter.js'), 'export default class Reporter {}') + writeFileSync( + join(temp, 'setup.js'), + 'export default async function setup() {}' + ) writeFileSync( join(temp, 'custom.js'), `export default { testDir: '/wrong', outputDir: '/wrong', updateSnapshots: 'all', + globalSetup: './setup.js', globalTeardown: ['./setup.js'], + webServer: [{command: 'node app.js', port: 8080, reuseExistingServer: true}, + {command: 'node api.js', port: 8081, cwd: './backend'}], + expect: {toHaveScreenshot: {scale: 'device', maxDiffPixels: 999}}, reporter: [['./reporter.js', {project: 'example'}], ['json', {outputFile: 'extra.json'}]], use: {connectOptions: {wsEndpoint: 'ws://wrong'}, viewport: {width: 500, height: 300}} }` @@ -73,15 +81,34 @@ test('compiled suite config preserves runner paths and accepts a pixel-count bud assert.equal(config.testDir, temp) assert.equal(config.outputDir, join(temp, 'artifacts')) assert.equal(config.updateSnapshots, 'none') + assert.deepEqual(config.webServer, [ + { + command: 'node app.js', + port: 8080, + cwd: temp, + reuseExistingServer: false, + }, + { + command: 'node api.js', + port: 8081, + cwd: join(temp, 'backend'), + reuseExistingServer: false, + }, + ]) + assert.deepEqual(config.globalSetup, [realpathSync(join(temp, 'setup.js'))]) + assert.deepEqual(config.globalTeardown, [ + realpathSync(join(temp, 'setup.js')), + ]) assert.deepEqual(config.reporter, [ ['list'], ['junit', {outputFile: join(temp, 'junit.xml')}], - [join(temp, 'reporter.js'), {project: 'example'}], + [realpathSync(join(temp, 'reporter.js')), {project: 'example'}], ['json', {outputFile: 'extra.json'}], ]) assert.equal(config.use?.connectOptions?.wsEndpoint, 'ws://127.0.0.1:5678') assert.deepEqual(config.use?.viewport, {width: 500, height: 300}) assert.equal(config.expect?.toHaveScreenshot?.maxDiffPixels, 7) + assert.equal(config.expect?.toHaveScreenshot?.scale, 'device') assert.equal(config.expect?.toHaveScreenshot?.maxDiffPixelRatio, undefined) const reporterPackage = join(temp, 'node_modules', 'consumer-reporter') mkdirSync(reporterPackage, {recursive: true}) @@ -103,10 +130,11 @@ test('compiled suite config preserves runner paths and accepts a pixel-count bud new URL('./suite-config.js?reporter-package', import.meta.url).href ) ).default + assert.equal(packageConfig.expect?.toHaveScreenshot?.scale, 'css') assert.deepEqual(packageConfig.reporter, [ ['list'], ['junit', {outputFile: join(temp, 'junit.xml')}], - [join(reporterPackage, 'index.js')], + [realpathSync(join(reporterPackage, 'index.js'))], ]) } finally { for (const key of Object.keys(process.env)) diff --git a/runtime/network.test.ts b/runtime/network.test.ts index b5046d7..3e7eb3b 100644 --- a/runtime/network.test.ts +++ b/runtime/network.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import {test} from 'node:test' -import {networkTargets, remoteAppUrl} from './network.js' +import {networkTargets, remoteAppUrl, environmentOrigins} from './network.js' test('remote URL selection preserves paths and permits only explicit destinations', () => { const url = remoteAppUrl({ @@ -42,3 +42,23 @@ test('remote mode fails closed for missing or malformed endpoints', () => { ]) assert.throws(() => networkTargets('https://app.example', [origin])) }) + +test('only declared environment endpoints join the exact origin allowlist', () => { + const env = { + BASELINE_URL: 'https://baseline.example', + UNDECLARED: 'https://other.example', + } + assert.equal( + networkTargets( + 'http://localhost:8080', + environmentOrigins(['BASELINE_URL', 'UNSET'], env) + ), + 'localhost:8080,baseline.example:443' + ) + assert.throws(() => + networkTargets( + 'http://localhost:8080', + environmentOrigins(['BASELINE_URL'], {BASELINE_URL: 'https://*.example'}) + ) + ) +}) diff --git a/runtime/network.ts b/runtime/network.ts index ac2460f..9375ed2 100644 --- a/runtime/network.ts +++ b/runtime/network.ts @@ -1,4 +1,11 @@ /** Turn explicit HTTP origins into exact Playwright tunnel host:port entries. */ +export function environmentOrigins( + names: string[], + env: NodeJS.ProcessEnv +): string[] { + return names.flatMap(name => (env[name] ? [env[name]!] : [])) +} + export function networkTargets(fixture: string, origins: string[]): string { return [new URL(fixture).origin, ...origins] .map(origin => { diff --git a/runtime/runner.ts b/runtime/runner.ts index 5133394..0518430 100644 --- a/runtime/runner.ts +++ b/runtime/runner.ts @@ -5,7 +5,7 @@ import {fileURLToPath, pathToFileURL} from 'node:url' import {createRequire} from 'node:module' import {validatePlaywrightVersions} from './versions.js' import {spawn, type ChildProcess} from 'node:child_process' -import {remoteAppUrl, networkTargets} from './network.js' +import {remoteAppUrl, networkTargets, environmentOrigins} from './network.js' import {testArguments} from './arguments.js' import {baselineDestination, updateBaselines} from './baselines.js' import {stageRunfiles, testEnvironment} from './isolation.js' @@ -17,17 +17,21 @@ function required(name: string) { } async function main() { - const visual = required('VRT_MODE') === 'visual' + const gallery = required('VRT_MODE') === 'visual' + const visual = gallery || required('VRT_MODE') === 'visual-spec' const remote = remoteAppUrl(process.env) + const origins = [ + ...(JSON.parse(required('VRT_NETWORK_ORIGINS')) as string[]), + ...environmentOrigins( + JSON.parse(required('VRT_NETWORK_ORIGINS_ENV')) as string[], + process.env + ), + ] // Validate explicit tunnel destinations before allocating resources. - networkTargets( - remote || 'http://127.0.0.1', - JSON.parse(required('VRT_NETWORK_ORIGINS')) as string[] - ) + networkTargets(remote || 'http://127.0.0.1', origins) const args = process.argv.slice(2) const update = args.includes('--update') if (!visual && update) throw new Error('E2E tests do not update baselines') - const selectors = testArguments(visual, args) const destination = update ? baselineDestination( required('BUILD_WORKSPACE_DIRECTORY'), @@ -56,6 +60,7 @@ async function main() { shell: {directory: string; entryPoint: string} | null playwright: {test: string; core: string; version: string; image: string} } + const selectors = testArguments(visual, args, descriptor.tests) const node = fs.realpathSync(required('JS_BINARY__NODE_BINARY')) const testRoot = path.dirname(descriptorPath) const generated = path.join(testRoot, '.rules-browser') @@ -112,7 +117,7 @@ async function main() { testPackage, path.join(generated, 'node_modules', '@playwright', 'test') ) - if (visual) + if (gallery) fs.copyFileSync( fileURLToPath(new URL('./capture.js', import.meta.url)), path.join(generated, '.rules-visual.spec.js') @@ -134,7 +139,7 @@ async function main() { JSON.parse(required('VRT_ENV_NAMES')) as string[], temp ), - VRT_NETWORK_ORIGINS: required('VRT_NETWORK_ORIGINS'), + VRT_NETWORK_ORIGINS: JSON.stringify(origins), VRT_INPUTS: inputs, VRT_MODE: required('VRT_MODE'), VRT_TEST_ROOT: visual ? testRoot : inputs, @@ -155,6 +160,15 @@ async function main() { VRT_OUTPUTS: outputs, VRT_VISUAL_CATALOG: path.join(temp, 'visual-catalog.json'), VRT_CACHE: path.join(temp, 'server-cache'), + RUNFILES_DIR: inputs, + RUNFILES: inputs, + RUNFILES_MANIFEST_FILE: '', + TEST_WORKSPACE: required('VRT_DESCRIPTOR').split('/')[0], + BAZEL_WORKSPACE: required('VRT_DESCRIPTOR').split('/')[0], + BAZEL_BINDIR: '.', + TEST_TMPDIR: temp, + TEST_UNDECLARED_OUTPUTS_DIR: outputs, + PATH: `${path.dirname(node)}:/usr/bin:/bin`, } // Docker discovery is deliberately separate from the fixture environment. // Set before importing Testcontainers, which reads helper settings at import time. @@ -195,7 +209,16 @@ async function main() { if (!appUrl) { const server = spawn( node, - [fileURLToPath(new URL('./server.js', import.meta.url))], + [ + fileURLToPath( + new URL( + descriptor.server || descriptor.shell + ? './server.js' + : './config-url.js', + import.meta.url + ) + ), + ], { cwd: testRoot, env, @@ -223,6 +246,7 @@ async function main() { }) }) } + networkTargets(appUrl!, origins) browser = await startBrowser(descriptor.playwright.image, core) if (interrupted) throw new Error('VRT interrupted') const run = (discover: boolean) => @@ -265,7 +289,7 @@ async function main() { resolve(code ?? 1) }) }) - const discoveryCode = visual ? await run(true) : 0 + const discoveryCode = gallery ? await run(true) : 0 const code = discoveryCode === 0 ? await run(false) : discoveryCode if (code !== 0) { if (visual) diff --git a/runtime/suite-config.ts b/runtime/suite-config.ts index 15488ef..a2f1d28 100644 --- a/runtime/suite-config.ts +++ b/runtime/suite-config.ts @@ -6,22 +6,23 @@ import { } from '@playwright/test' import {pathToFileURL} from 'node:url' import {createRequire} from 'node:module' +import path from 'node:path' import {e2eConfig, componentBrowserConfig, visualConfig} from './config.js' import {screenshotMatching, type VisualMatching} from './matching.js' const mode = process.env.VRT_MODE! const root = process.env.VRT_TEST_ROOT! -const defaults = - mode === 'visual' - ? visualConfig({root}) - : mode === 'component' - ? componentBrowserConfig({root, gallery: process.env.VRT_APP_URL!}) - : e2eConfig({root}) +const visual = mode === 'visual' || mode === 'visual-spec' +const defaults = visual + ? visualConfig({root}) + : mode === 'component' + ? componentBrowserConfig({root, gallery: process.env.VRT_APP_URL!}) + : e2eConfig({root}) const custom = process.env.VRT_CONFIG_OVERRIDE ? ((await import(pathToFileURL(process.env.VRT_CONFIG_OVERRIDE).href)) .default as PlaywrightTestConfig) : {} -if (mode === 'visual' && custom.projects) +if (visual && custom.projects) throw new Error( 'Use separate visual targets and baseline directories instead of Playwright projects' ) @@ -58,6 +59,27 @@ const additionalReporters: ReporterDescription[] = reporters.map( ) // Keep browser connections, baseline updates, and required reports managed. const merged = defineConfig(defaults, custom) +const configDirectory = process.env.VRT_CONFIG_OVERRIDE + ? path.dirname(process.env.VRT_CONFIG_OVERRIDE) + : root +const lifecycleModules = (value: string | string[] | undefined) => + value === undefined + ? undefined + : (Array.isArray(value) ? value : [value]).map(file => + createRequire(path.join(configDirectory, 'package.json')).resolve(file) + ) +const webServer = custom.webServer + ? (Array.isArray(custom.webServer) + ? custom.webServer + : [custom.webServer] + ).map(server => ({ + ...server, + cwd: server.cwd + ? path.resolve(configDirectory, server.cwd) + : configDirectory, + reuseExistingServer: false, + })) + : undefined const testMatch = mode === 'visual' ? '**/.rules-visual.spec.js' @@ -70,47 +92,58 @@ const managedUse = { connectOptions: defaults.use!.connectOptions, browserName: 'chromium' as const, } -export default defineConfig(merged, { - testDir: root, - testMatch, - testIgnore: [], - outputDir: defaults.outputDir, - reporter: [ - ...(defaults.reporter as ReporterDescription[]), - ...additionalReporters, - ], - updateSnapshots: defaults.updateSnapshots, - snapshotPathTemplate: defaults.snapshotPathTemplate, - use: managedUse, - ...(merged.projects - ? { - projects: merged.projects.map(project => ({ - ...project, - testDir: root, - testMatch, - testIgnore: [], - outputDir: defaults.outputDir, - snapshotPathTemplate: defaults.snapshotPathTemplate, - use: { - ...managedUse, - ...project.use, - connectOptions: defaults.use!.connectOptions, - browserName: 'chromium' as const, +export default defineConfig( + {...merged, webServer: undefined}, + { + testDir: root, + testMatch, + testIgnore: [], + outputDir: defaults.outputDir, + reporter: [ + ...(defaults.reporter as ReporterDescription[]), + ...additionalReporters.filter( + ([name, options]) => name !== 'list' || options !== undefined + ), + ], + updateSnapshots: defaults.updateSnapshots, + snapshotPathTemplate: visual + ? defaults.snapshotPathTemplate + : custom.snapshotPathTemplate, + webServer, + globalSetup: lifecycleModules(custom.globalSetup), + globalTeardown: lifecycleModules(custom.globalTeardown), + use: managedUse, + ...(merged.projects + ? { + projects: merged.projects.map(project => ({ + ...project, + testDir: root, + testMatch, + testIgnore: [], + outputDir: defaults.outputDir, + snapshotPathTemplate: + project.snapshotPathTemplate ?? custom.snapshotPathTemplate, + use: { + ...managedUse, + ...project.use, + connectOptions: defaults.use!.connectOptions, + browserName: 'chromium' as const, + }, + })), + } + : {}), + ...(visual + ? { + expect: { + ...merged.expect, + toHaveScreenshot: { + animations: 'disabled', + caret: 'hide', + scale: custom.expect?.toHaveScreenshot?.scale ?? 'css', + ...screenshotMatching(matching), + }, }, - })), - } - : {}), - ...(mode === 'visual' - ? { - expect: { - ...merged.expect, - toHaveScreenshot: { - animations: 'disabled', - caret: 'hide', - scale: 'css', - ...screenshotMatching(matching), - }, - }, - } - : {}), -}) + } + : {}), + } +) diff --git a/vrt/defs.bzl b/vrt/defs.bzl index 743d216..104f8be 100644 --- a/vrt/defs.bzl +++ b/vrt/defs.bzl @@ -4,6 +4,13 @@ load("//internal:browser.bzl", "browser_test", _PLAYWRIGHT_IMAGE = "PLAYWRIGHT_I PLAYWRIGHT_IMAGE = _PLAYWRIGHT_IMAGE +def visual_test(name, tests, **kwargs): + """Run compiled native screenshot specs with matching and owned baseline updates.""" + for key in ["visual", "component"]: + if key in kwargs: + fail("%s is not a visual test option" % key) + browser_test(name = name, tests = tests, visual = True, **kwargs) + def component_visual_test(name, **kwargs): """See docs/component-vrt.md for built shell, matching, and baseline arguments.""" for key in ["visual", "component", "tests"]: