diff --git a/example/package.json b/example/package.json index 8d32e2a364..b24663d85e 100644 --- a/example/package.json +++ b/example/package.json @@ -8,7 +8,9 @@ "eas-build-pre-install": "cd .. && yarn", "start": "EXPO_NO_TYPESCRIPT_SETUP=1 expo start --dev-client", "android": "EXPO_NO_TYPESCRIPT_SETUP=1 expo run:android", - "ios": "EXPO_NO_TYPESCRIPT_SETUP=1 expo run:ios" + "ios": "EXPO_NO_TYPESCRIPT_SETUP=1 expo run:ios", + "visual": "node visual/run.mjs", + "test:visual": "node --test visual/run.test.mjs" }, "dependencies": { "@expo/metro-config": "~56.0.0", diff --git a/example/src/Examples/SurfaceExample.tsx b/example/src/Examples/SurfaceExample.tsx index e941c7c843..4a745e341e 100644 --- a/example/src/Examples/SurfaceExample.tsx +++ b/example/src/Examples/SurfaceExample.tsx @@ -53,7 +53,7 @@ const SurfaceExample = () => { return ( - + { - + -realistic-1-t0.02.png`). To a human the change is borderline: noticeable only when flipping between the two images. + +## Web + +Tried once on 2026-09-14 against `expo export --platform web` of the example app, served locally, with agent-device's managed browser (`agent-device web setup`, agent-browser 0.27.1). `open` works and a full-page `screenshot` works (1280x577). `screenshot --crop-on 'id="surface-example-elevated"'` is refused: `UNSUPPORTED_OPERATION`, `CROP_TARGET_NOT_ACCEPTED`, `PENDING_PIXEL_IDENTITY_EVIDENCE`. Transcript in `evidence/web-excerpt.json`, the full-page capture in `evidence/diff-images/web-full-page-screenshot.png`. A web leg would therefore diff full viewports rather than per-section crops. Not pursued for a Surface-only PoC. + +## Caveats + +- The crop is not perfectly isolated. Under the gross break a full-width 33 px band at the top of the crop changed: the Appbar is itself a `Surface`, and the break changed every Surface. The realistic break showed no bleed. An Appbar-only change could register against this crop; that is inherent in screenshotting real screens rather than isolated components. +- Dev-client chrome can land in the crop. The Expo dev-client's floating "Tools" button, switched on by a stray dev-menu press, sits inside the elevated section on both platforms and produced deterministic false FAILs with `src/` clean: 2,822 px on Android (`evidence/diff-images/android-devclient-tools-button.png`, `evidence/devclient-excerpt.json`) and 3,916 px on iOS. Turning it off in the dev menu ("Tools button") restored 0 on both. On iOS the dev-client's first-run onboarding sheet also dims the whole app, and `wait stable` reports that as settled. A release build would remove the whole class of dev menu, dev launcher, onboarding sheet and floating button. +- Not measured: swiftshader (what `ubuntu-latest` renders with), cold simulator/emulator boot, another host or day, runtime or image updates, other components, text-heavy crops, dark theme. + +## Reproducing by hand + +Prerequisites: the example app built and installed on a device matching the profile in `env.json`, Metro running (`yarn example start`). Each platform gets its own named session, `paper-visual-ios` and `paper-visual-android`, the same names the runner uses. An agent-device session is bound to one device, so if another session on the machine still claims the device, `open` is refused; close that session (`npx agent-device@0.21.0 close --session `) and try again. Start by listing the devices to get the UDID or serial; the loop is the same on both platforms and only these values differ: + +| value | iOS | Android | +| -------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `TARGET` | `--platform ios --session paper-visual-ios --udid ` | `--platform android --session paper-visual-android --serial ` | +| `DENSITY` | `--pixel-density 3` | (none; Android returns native pixels) | +| overlay after `--relaunch` | dev menu, a few seconds after launch: `press 'label="Close"'` | dev launcher: `press 'label="http://:8081"'`, then wait for the bundle | +| `BASELINE` | `example/visual/__baselines__/ios` | `example/visual/__baselines__/android` | + +```bash +AD="npx agent-device@0.21.0" +$AD devices --platform ios --json # or --platform android; yields the UDID or serial +$AD open com.callstack.reactnativepaperexample --relaunch $TARGET --json +$AD wait stable 500 10000 $TARGET --json +# dismiss the overlay for this platform (table above); the app restores its last screen, +# so if it is not the Surface example, press Back to the list and press the "Surface" row +$AD wait stable 500 10000 $TARGET --json && sleep 2 +$AD screenshot current.png --crop-on 'id="surface-example-elevated"' $DENSITY $TARGET --json +$AD diff screenshot --baseline $BASELINE/surface-example-elevated.png current.png --out diff.png --threshold 0.02 --json +``` + +Expect `differentPixels: 0`. To see a failure, change `shadow(elevation, …)` to `shadow(elevation === 1 ? 2 : elevation, …)` in `src/components/Surface.tsx` (iOS) or `androidElevationLevels[elevation]` to `androidElevationLevels[elevation === 1 ? 2 : elevation]` (Android), relaunch so the app fetches the bundle, capture, diff: about 10,179 px (iOS) or 9,336 px (Android) at 0.02, and `match: true` at 0.1. `git checkout -- src/components/Surface.tsx` afterwards. + +## Running it + +`example/visual/run.mjs` runs the loop above on either platform: plain ESM on Node 20 or newer, no new dependencies, spawning `npx agent-device@0.21.0 … --json` and parsing the output. + +```bash +yarn example visual --platform ios +yarn example visual --platform android +``` + +It resolves and boots the device from the profile in `env.json`, relaunches the app so the bundle is fresh, waits for the app to be ready, dismisses the dev menu, dev launcher and floating Tools button if present, goes Back to the example list root and presses the Surface row if the app restored another screen, then captures and diffs each story and prints one line per story: + +``` +ios surface-example-elevated changed=0 (0%) regions=0 threshold=0.02 → PASS +``` + +Exit codes: 1 if any story fails the diff, 2 for a setup error (no simulator with that name on that runtime, no AVD with that name, an emulator whose API level or density is not the one the baselines were captured on, a failed boot, or an `open` that agent-device refused because another session claims the device), 3 if a capture's size does not match its baseline PNG. `--update` writes the captures as baselines, creating missing ones. `--out ` sets where captures, diff images and `summary.json` go (default `example/visual/artifacts/run/`, gitignored). `--threshold <0-1>` overrides the 0.02 default and `--story a,b` limits the run to some stories; `--help` lists them all. `summary.json` is written on every exit except an argument error, which exits 2 before an output directory is known. + +Prerequisites are the same as for the hand-run loop: the app built and installed on a device matching the profile, an AVD named exactly `Pixel_10_Pro` on Android with `adb` and `emulator` reachable, and Metro running. After an Android boot the runner sets `adb reverse tcp:8081 tcp:8081` itself, since a cold emulator has no route to Metro. A simulator that has never booted runs Apple's first-boot migration, which can take longer than agent-device's 120 s boot cap; boot it once from Xcode before the first run. + +The pure helpers have ten `node:test` cases, no device needed (boot and the `open` retry are not covered): + +```bash +yarn example test:visual +``` + +Left as follow-ups: creating the simulator or AVD when it is missing, per-profile baseline directories, `open --metro-port` in place of dismissing the dev launcher, a release-build variant of the example app, and CI. Before anything runs this in CI, `npx agent-device@0.21.0` has to become a locked dependency: today the runner executes a package fetched from the network on every run with no lockfile entry. + +Not run by Jest, the pre-commit hook or CI. `results.csv` was transcribed from the raw `diff screenshot` JSON; its last column is the path of that JSON on branch `poc/agent-device-visual-runner`, not in this PR, except one iOS dev-client row from an uncommitted 2026-09-16 re-verification. + +## What a real runner has to handle + +Things `run.mjs` had to deal with. None is an agent-device bug: + +- Relaunch before every capture (`open --relaunch` on iOS, `am force-stop` plus `open` on Android) so the app fetches the current bundle. Fast Refresh silently stopped reaching the Android app; without a fresh bundle a stale screen reads as PASS. `agent-device metro reload` exists for the same purpose and works, but the runner relaunches anyway because the app persists its navigation state. +- The example app persists navigation state. A relaunch lands on the last screen, so the runner has to press Back to the list root and pick the Surface row, not the header title. +- Dev-client chrome: on Android a relaunch lands in the dev launcher; on iOS the first-run onboarding sheet dims the whole app and `wait stable` reports it as settled; on both, the floating Tools button can sit inside the crop. Dismiss all of it before capturing, or capture from a release build. The runner keeps the dismissal code because the example app is only built as a dev client here; a release-build variant is a follow-up. +- Set `--threshold` explicitly (0.02 for soft shadows) and verify noise at it; the default 0.1 is documented as a 44-unit RGB tolerance, far looser than a one-step shadow change. +- Use `find … list` to locate without tapping, `--first` or a `role=` qualifier where Android exposes the same label on a row and its text child, and assert on node count after `snapshot --scope`, which returns success with empty nodes when nothing matches. +- Pin the device profile, resolve the instance through `simctl` and `agent-device devices` and boot it, and refuse to compare or re-baseline on a different image; derive expected capture size from the baseline PNG, per story. +- A plain `.mjs` entry guarded by `import.meta.main` does nothing on Node 20/22; guard with an `argv[1]` comparison. + +## Evidence + +- `evidence/results.csv`: one row per `diff screenshot` run (74 rows, including the two dev-client false FAILs): platform, capture, threshold, total and changed pixels, mismatch %, regions, match, and the path of the raw JSON it came from on the runner branch (one iOS dev-client row comes from an uncommitted re-verification). +- `evidence/diff-images/`: one diff image per platform for the realistic break (ring on the Elevation 1 card) and the gross break, both at 0.02, the dev-client Tools-button false FAIL, and the full-page web capture. There is no 0.1 image of the realistic break because it matches at 0.1 and `diff screenshot --out` writes nothing on a match, deleting any stale file at that path; the gross break did produce 0.1 images and they were not kept. +- `evidence/a11y-excerpt.json`, `evidence/devclient-excerpt.json`, `evidence/web-excerpt.json`: the nodes and responses that matter; full trees on the runner branch. +- `env.json`: the pinned device profiles, versions and thresholds. + +The first-run rows predate the test-id rename and their source files say `surface-elevated` / `surface-flat`; renaming an id changes no pixel. diff --git a/example/visual/__baselines__/android/surface-example-elevated.png b/example/visual/__baselines__/android/surface-example-elevated.png new file mode 100644 index 0000000000..c94e2bc1b1 Binary files /dev/null and b/example/visual/__baselines__/android/surface-example-elevated.png differ diff --git a/example/visual/__baselines__/android/surface-example-flat.png b/example/visual/__baselines__/android/surface-example-flat.png new file mode 100644 index 0000000000..4f688b4e61 Binary files /dev/null and b/example/visual/__baselines__/android/surface-example-flat.png differ diff --git a/example/visual/__baselines__/ios/surface-example-elevated.png b/example/visual/__baselines__/ios/surface-example-elevated.png new file mode 100644 index 0000000000..5eb2aa499f Binary files /dev/null and b/example/visual/__baselines__/ios/surface-example-elevated.png differ diff --git a/example/visual/__baselines__/ios/surface-example-flat.png b/example/visual/__baselines__/ios/surface-example-flat.png new file mode 100644 index 0000000000..b909a990ea Binary files /dev/null and b/example/visual/__baselines__/ios/surface-example-flat.png differ diff --git a/example/visual/env.json b/example/visual/env.json new file mode 100644 index 0000000000..c99db567ff --- /dev/null +++ b/example/visual/env.json @@ -0,0 +1,34 @@ +{ + "platform": "ios", + "device": "iPhone 17 Pro", + "iosVersion": "26.5", + "runtime": "com.apple.CoreSimulator.SimRuntime.iOS-26-5", + "scale": 3, + "pixelDensity": 3, + "agentDeviceVersion": "0.21.0", + "reactNativeVersion": "0.85.3", + "branch": "poc/agent-device-visual", + "date": "2026-09-10T09:30:58Z", + "baselines": { + "surface-example-elevated": "__baselines__/ios/surface-example-elevated.png", + "surface-example-flat": "__baselines__/ios/surface-example-flat.png" + }, + "android": { + "avd": "Pixel_10_Pro", + "device": "Pixel 10 Pro", + "apiLevel": 37, + "androidRelease": "17", + "systemImage": "system-images/android-37.0/google_apis_playstore_ps16k/arm64-v8a/", + "density": 480, + "screenPx": "1280x2856", + "emulatorVersion": "36.6.11.0 (build_id 15507667)", + "gpuMode": "auto (hw.gpu.enabled=yes, hw.gpu.mode=auto; swiftshader NOT measured)", + "agentDeviceVersion": "0.21.0", + "reactNativeVersion": "0.85.3", + "date": "2026-09-10T09:45:00Z", + "baselines": { + "surface-example-elevated": "__baselines__/android/surface-example-elevated.png", + "surface-example-flat": "__baselines__/android/surface-example-flat.png" + } + } +} diff --git a/example/visual/evidence/a11y-excerpt.json b/example/visual/evidence/a11y-excerpt.json new file mode 100644 index 0000000000..7e6bd24500 --- /dev/null +++ b/example/visual/evidence/a11y-excerpt.json @@ -0,0 +1,74 @@ +{ + "about": "Excerpts from the XCUITest / UiAutomator trees captured 2026-09-10 with `npx agent-device@0.21.0 snapshot --raw --force-full --json`. The full trees (13,500 lines) are on branch poc/agent-device-visual-runner under example/visual/evidence/a11y/. Test id at the time: surface-elevated (renamed surface-example-elevated later; the rename changes no pixel).", + "ios": { + "with_accessible": { + "total_nodes": 115, + "elevation_labelled_nodes": 2, + "surface_section_node": [ + { + "type": "Other", + "role": "RCTViewComponentView", + "identifier": "surface-elevated", + "label": "Elevated surface, Elevation 0, Elevation 1, Elevation 2, Elevation 3, Elevation 4, Elevation 5, Vertical scroll bar, 3 pages", + "rect": { + "x": 0, + "y": 134, + "width": 402, + "height": 214 + } + } + ] + }, + "testID_only": { + "total_nodes": 177, + "elevation_labelled_nodes": 26, + "surface_section_node": [ + { + "type": "Other", + "role": "RCTViewComponentView", + "identifier": "surface-elevated", + "rect": { + "x": 0, + "y": 134, + "width": 402, + "height": 214 + } + } + ] + } + }, + "android": { + "with_accessible": { + "total_nodes": 217, + "elevation_labelled_nodes": 13, + "surface_section_node": [ + { + "type": "android.view.ViewGroup", + "identifier": "surface-elevated", + "rect": { + "x": 0, + "y": 372, + "width": 1280, + "height": 642 + } + } + ] + }, + "testID_only": { + "total_nodes": 217, + "elevation_labelled_nodes": 13, + "surface_section_node": [ + { + "type": "android.view.ViewGroup", + "identifier": "surface-elevated", + "rect": { + "x": 0, + "y": 372, + "width": 1280, + "height": 642 + } + } + ] + } + } +} diff --git a/example/visual/evidence/devclient-excerpt.json b/example/visual/evidence/devclient-excerpt.json new file mode 100644 index 0000000000..d9f3f47a87 --- /dev/null +++ b/example/visual/evidence/devclient-excerpt.json @@ -0,0 +1,38 @@ +{ + "about": "Excerpts captured 2026-09-11 with `snapshot --force-full --json`; full files on branch poc/agent-device-visual-runner under example/visual/evidence/runner/.", + "ios_after_open_relaunch": { + "immediately": { + "total_nodes": 3, + "labels": ["React Native Paper Example", null, "Downloading 100%\u2026"] + }, + "six_seconds_later": { + "total_nodes": 77, + "dev_menu_close_node": [ + { + "type": "Button", + "role": "SwiftUI.AccessibilityNode", + "identifier": "xmark", + "label": "Close", + "rect": { + "x": 344.06965174129357, + "y": 401.5837479270315, + "width": 34.56716417910445, + "height": 34.56716417910445 + } + } + ] + } + }, + "android_devclient_floating_tools_button": [ + { + "type": "android.widget.ImageView", + "label": "Tools", + "rect": { + "x": 1115, + "y": 243, + "width": 78, + "height": 78 + } + } + ] +} diff --git a/example/visual/evidence/diff-images/android-devclient-tools-button.png b/example/visual/evidence/diff-images/android-devclient-tools-button.png new file mode 100644 index 0000000000..856b41c00b Binary files /dev/null and b/example/visual/evidence/diff-images/android-devclient-tools-button.png differ diff --git a/example/visual/evidence/diff-images/android-gross-t0.02.png b/example/visual/evidence/diff-images/android-gross-t0.02.png new file mode 100644 index 0000000000..a6f157ad1a Binary files /dev/null and b/example/visual/evidence/diff-images/android-gross-t0.02.png differ diff --git a/example/visual/evidence/diff-images/android-realistic-1-t0.02.png b/example/visual/evidence/diff-images/android-realistic-1-t0.02.png new file mode 100644 index 0000000000..5e6290efe8 Binary files /dev/null and b/example/visual/evidence/diff-images/android-realistic-1-t0.02.png differ diff --git a/example/visual/evidence/diff-images/ios-gross-t0.02.png b/example/visual/evidence/diff-images/ios-gross-t0.02.png new file mode 100644 index 0000000000..79e8aa8658 Binary files /dev/null and b/example/visual/evidence/diff-images/ios-gross-t0.02.png differ diff --git a/example/visual/evidence/diff-images/ios-realistic-1-t0.02.png b/example/visual/evidence/diff-images/ios-realistic-1-t0.02.png new file mode 100644 index 0000000000..02219e89f4 Binary files /dev/null and b/example/visual/evidence/diff-images/ios-realistic-1-t0.02.png differ diff --git a/example/visual/evidence/diff-images/web-full-page-screenshot.png b/example/visual/evidence/diff-images/web-full-page-screenshot.png new file mode 100644 index 0000000000..496bc7a782 Binary files /dev/null and b/example/visual/evidence/diff-images/web-full-page-screenshot.png differ diff --git a/example/visual/evidence/results.csv b/example/visual/evidence/results.csv new file mode 100644 index 0000000000..c022cf582a --- /dev/null +++ b/example/visual/evidence/results.csv @@ -0,0 +1,75 @@ +platform,group,capture,threshold,totalPixels,differentPixels,mismatchPercent,regions,match,sourceOnRunnerBranch +ios,first-run,a-vs-b (density 1),0.1,86028,0,0,0,True,evidence/ios/26-diff.json +ios,first-run,flat a-vs-b (density 1),0.1,86028,0,0,0,True,evidence/ios/30-diff.json +ios,first-run,warm-1,0.1,774252,0,0,0,True,evidence/ios/37-diff.json +ios,first-run,warm-2,0.1,774252,0,0,0,True,evidence/ios/40-diff.json +ios,first-run,warm-3,0.1,774252,0,0,0,True,evidence/ios/43-diff.json +ios,first-run,relaunch-1,0.1,774252,0,0,0,True,evidence/ios/53-diff.json +ios,first-run,gross,0.1,774252,4277,0.55,1,False,evidence/ios/59-diff.json +ios,first-run,gross,0.02,774252,116292,15.02,3,False,evidence/ios/60-diff.json +ios,first-run,revert-1,0.1,774252,0,0,0,True,evidence/ios/63-diff.json +ios,first-run,realistic,0.02,774252,10179,1.31,3,False,evidence/ios/66-diff-t0.02.json +ios,first-run,realistic,0.1,774252,0,0,0,True,evidence/ios/66-diff-t0.1.json +ios,first-run,revert-2,0.1,774252,0,0,0,True,evidence/ios/69-diff.json +ios,first-run,relaunch-1,0.02,774252,0,0,0,True,evidence/ios/6x-diff-relaunch-1-t002.json +ios,first-run,warm-1,0.02,774252,0,0,0,True,evidence/ios/6x-diff-warm-1-t002.json +ios,first-run,warm-2,0.02,774252,0,0,0,True,evidence/ios/6x-diff-warm-2-t002.json +ios,first-run,warm-3,0.02,774252,0,0,0,True,evidence/ios/6x-diff-warm-3-t002.json +android,first-run,relaunch-1,0.01,821760,0,0,0,True,evidence/android/103-diff-relaunch-1-t0.01.json +android,first-run,relaunch-1,0.02,821760,0,0,0,True,evidence/android/103-diff-relaunch-1-t0.02.json +android,first-run,relaunch-1,0.05,821760,0,0,0,True,evidence/android/103-diff-relaunch-1-t0.05.json +android,first-run,relaunch-1,0.1,821760,0,0,0,True,evidence/android/103-diff-relaunch-1-t0.1.json +android,first-run,gross,0.01,821760,265916,32.36,2,False,evidence/android/108-diff-gross-t0.01.json +android,first-run,gross,0.02,821760,208505,25.37,4,False,evidence/android/108-diff-gross-t0.02.json +android,first-run,gross,0.05,821760,118661,14.44,4,False,evidence/android/108-diff-gross-t0.05.json +android,first-run,gross,0.1,821760,65051,7.92,2,False,evidence/android/108-diff-gross-t0.1.json +android,first-run,revert1,0.01,821760,0,0,0,True,evidence/android/111-diff-revert1-t0.01.json +android,first-run,revert1,0.02,821760,0,0,0,True,evidence/android/111-diff-revert1-t0.02.json +android,first-run,revert1,0.05,821760,0,0,0,True,evidence/android/111-diff-revert1-t0.05.json +android,first-run,revert1,0.1,821760,0,0,0,True,evidence/android/111-diff-revert1-t0.1.json +android,first-run,realistic,0.01,821760,14686,1.79,1,False,evidence/android/114-diff-realistic-t0.01.json +android,first-run,realistic,0.02,821760,9336,1.14,1,False,evidence/android/114-diff-realistic-t0.02.json +android,first-run,realistic,0.05,821760,1830,0.22,1,False,evidence/android/114-diff-realistic-t0.05.json +android,first-run,realistic,0.1,821760,0,0,0,True,evidence/android/114-diff-realistic-t0.1.json +android,first-run,revert2,0.01,821760,0,0,0,True,evidence/android/117-diff-revert2-t0.01.json +android,first-run,revert2,0.02,821760,0,0,0,True,evidence/android/117-diff-revert2-t0.02.json +android,first-run,revert2,0.05,821760,0,0,0,True,evidence/android/117-diff-revert2-t0.05.json +android,first-run,revert2,0.1,821760,0,0,0,True,evidence/android/117-diff-revert2-t0.1.json +android,first-run,warm-1,0.01,821760,0,0,0,True,evidence/android/94-diff-warm-1-t0.01.json +android,first-run,warm-1,0.02,821760,0,0,0,True,evidence/android/94-diff-warm-1-t0.02.json +android,first-run,warm-1,0.05,821760,0,0,0,True,evidence/android/94-diff-warm-1-t0.05.json +android,first-run,warm-1,0.1,821760,0,0,0,True,evidence/android/94-diff-warm-1-t0.1.json +android,first-run,warm-2,0.01,821760,0,0,0,True,evidence/android/94-diff-warm-2-t0.01.json +android,first-run,warm-2,0.02,821760,0,0,0,True,evidence/android/94-diff-warm-2-t0.02.json +android,first-run,warm-2,0.05,821760,0,0,0,True,evidence/android/94-diff-warm-2-t0.05.json +android,first-run,warm-2,0.1,821760,0,0,0,True,evidence/android/94-diff-warm-2-t0.1.json +android,first-run,warm-3,0.01,821760,0,0,0,True,evidence/android/94-diff-warm-3-t0.01.json +android,first-run,warm-3,0.02,821760,0,0,0,True,evidence/android/94-diff-warm-3-t0.02.json +android,first-run,warm-3,0.05,821760,0,0,0,True,evidence/android/94-diff-warm-3-t0.05.json +android,first-run,warm-3,0.1,821760,0,0,0,True,evidence/android/94-diff-warm-3-t0.1.json +ios,re-measured,gross,0.02,774252,116292,15.02,3,False,evidence/sensitivity/ios/gross-diff-t0.02.json +ios,re-measured,gross,0.1,774252,4277,0.55,1,False,evidence/sensitivity/ios/gross-diff-t0.1.json +ios,re-measured,realistic-1,0.02,774252,10179,1.31,3,False,evidence/sensitivity/ios/realistic-1-diff-t0.02.json +ios,re-measured,realistic-1,0.1,774252,0,0,0,True,evidence/sensitivity/ios/realistic-1-diff-t0.1.json +ios,re-measured,realistic-2,0.02,774252,10179,1.31,3,False,evidence/sensitivity/ios/realistic-2-diff-t0.02.json +ios,re-measured,realistic-2,0.1,774252,0,0,0,True,evidence/sensitivity/ios/realistic-2-diff-t0.1.json +ios,re-measured,realistic-3,0.02,774252,10179,1.31,3,False,evidence/sensitivity/ios/realistic-3-diff-t0.02.json +ios,re-measured,realistic-3,0.1,774252,0,0,0,True,evidence/sensitivity/ios/realistic-3-diff-t0.1.json +ios,re-measured,reverted,0.02,774252,0,0,0,True,evidence/sensitivity/ios/reverted-diff-t0.02.json +ios,re-measured,reverted,0.1,774252,0,0,0,True,evidence/sensitivity/ios/reverted-diff-t0.1.json +android,re-measured,gross,0.02,821760,208505,25.37,4,False,evidence/sensitivity/android/gross-diff-t0.02.json +android,re-measured,gross,0.1,821760,65051,7.92,2,False,evidence/sensitivity/android/gross-diff-t0.1.json +android,re-measured,gross-probe,0.02,821760,0,0,0,True,evidence/sensitivity/android/gross-probe-diff-t0.02.json +android,re-measured,gross-probe,0.1,821760,0,0,0,True,evidence/sensitivity/android/gross-probe-diff-t0.1.json +android,re-measured,realistic-1,0.02,821760,9336,1.14,1,False,evidence/sensitivity/android/realistic-1-diff-t0.02.json +android,re-measured,realistic-1,0.1,821760,0,0,0,True,evidence/sensitivity/android/realistic-1-diff-t0.1.json +android,re-measured,realistic-2,0.02,821760,9336,1.14,1,False,evidence/sensitivity/android/realistic-2-diff-t0.02.json +android,re-measured,realistic-2,0.1,821760,0,0,0,True,evidence/sensitivity/android/realistic-2-diff-t0.1.json +android,re-measured,realistic-3,0.02,821760,9336,1.14,1,False,evidence/sensitivity/android/realistic-3-diff-t0.02.json +android,re-measured,realistic-3,0.1,821760,0,0,0,True,evidence/sensitivity/android/realistic-3-diff-t0.1.json +android,re-measured,reverted,0.02,821760,0,0,0,True,evidence/sensitivity/android/reverted-diff-t0.02.json +android,re-measured,reverted,0.1,821760,0,0,0,True,evidence/sensitivity/android/reverted-diff-t0.1.json +android,re-measured,sanity-before-break,0.02,821760,0,0,0,True,evidence/sensitivity/android/sanity-before-break-diff-t0.02.json +android,re-measured,sanity-before-break,0.1,821760,0,0,0,True,evidence/sensitivity/android/sanity-before-break-diff-t0.1.json +android,dev-client-chrome,tools-button-on,0.02,821760,2822,0.34,1,False,runner-branch:evidence/runs/android/round3-pass/summary.json +ios,dev-client-chrome,tools-button-on,0.02,774252,3916,0.51,1,False,"verify/ios/a11-diff.json (2026-09-16 re-verification, not committed)" diff --git a/example/visual/evidence/web-excerpt.json b/example/visual/evidence/web-excerpt.json new file mode 100644 index 0000000000..35f8593766 --- /dev/null +++ b/example/visual/evidence/web-excerpt.json @@ -0,0 +1,33 @@ +{ + "about": "agent-device 0.21.0, --platform web (managed agent-browser 0.27.1, Node 24.18), against `expo export --platform web` of the example app served on 127.0.0.1:4322, 2026-09-14.", + "open": { + "success": true, + "device": "Agent Browser Chrome", + "startupMs": 1576 + }, + "screenshot_crop_on": { + "command": "screenshot web-crop.png --crop-on 'id=\"surface-example-elevated\"' --platform web --session web --json", + "success": false, + "error": { + "code": "UNSUPPORTED_OPERATION", + "message": "screenshot --crop-on is not accepted on web targets", + "hint": "This command is not available for the selected platform/device.", + "diagnosticId": "mu0zgqxl-62c0defe", + "logPath": "/Users/juliankobrynski/.agent-device/sessions/web/requests/4f9d41985f52c500.ndjson", + "diagnosticsRecord": { + "session": "web", + "requestId": "4f9d41985f52c500" + }, + "details": { + "reason": "CROP_TARGET_NOT_ACCEPTED", + "rejectionReason": "PENDING_PIXEL_IDENTITY_EVIDENCE" + } + } + }, + "screenshot_plain": { + "command": "screenshot web-full.png --platform web --session web --json", + "success": true, + "width": 1280, + "height": 577 + } +} diff --git a/example/visual/run.mjs b/example/visual/run.mjs new file mode 100644 index 0000000000..67cf876950 --- /dev/null +++ b/example/visual/run.mjs @@ -0,0 +1,1334 @@ +#!/usr/bin/env node +/** + * Visual regression loop for the Surface example (PoC). + * + * node example/visual/run.mjs --platform ios|android [--update] + * [--threshold 0.02] [--story surface-example-elevated,surface-example-flat] + * [--out ] + * + * Drives agent-device 0.21.0 through `npx` and parses `--json` stdout. Its + * Node client was not used because it requires adding agent-device as a + * dependency, which this PoC avoided; a runner that adopts the tool should + * switch to the client. + * + * The device is resolved from the profile in example/visual/env.json (simulator + * name plus runtime on iOS, AVD name on Android) and booted if needed, so no + * UDID or serial is pinned anywhere. + * + * Prerequisites (documented, not automated): the example app is already built + * and installed on a device matching example/visual/env.json, Metro is running, + * and the baselines in example/visual/__baselines__// were captured + * on that same device profile. + * + * Every run relaunches the app: the example app persists its navigation state + * (PERSISTENCE_KEY in example/src/index.tsx), and only a relaunch makes the app + * fetch a fresh JS bundle from Metro, Fast Refresh alone was observed not to + * reach the Android app, which made captures silently stale. + * + * Exit codes: 0 pass, 1 a story FAILed the diff, 2 setup/environment error, + * 3 a capture came back with the wrong dimensions. `summary.json` is written in + * the output directory in every one of those cases, except an argument error, + * which prints the problem and exits 2 before there are any options to summarise. + * + * Unit tests: node --test example/visual/run.test.mjs + */ + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const VISUAL_DIR = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_DIR = path.join(VISUAL_DIR, '__baselines__'); +const ENV_FILE = path.join(VISUAL_DIR, 'env.json'); +const AGENT_DEVICE = 'agent-device@0.21.0'; +const BUNDLE_ID = 'com.callstack.reactnativepaperexample'; +const DEFAULT_STORIES = ['surface-example-elevated', 'surface-example-flat']; +const DEFAULT_THRESHOLD = 0.02; // PoC finding: the CLI default of 0.1 misses soft-shadow regressions. +const SCROLL_STEP = 6; // rows per `scroll down` in the example list +const MAX_SCROLL_STEPS = 6; // bound the search so a wrong screen fails instead of looping +const MAX_BACK_STEPS = 8; // deepest example nesting is 2; 8 leaves room and still terminates +const MAX_OVERLAY_STEPS = 5; // dev menu + dev launcher, each possibly twice + +// App-ready polling. Right after `open` the tree is just the splash screen +// (iOS: 3 nodes, Application, SplashScreenLogo, "Downloading 100%…"), and +// after leaving the Android dev launcher it is an 11-node "Connecting to the +// development server…" screen, so any check that snapshots immediately reads +// the wrong screen. Node count alone is not enough: the loading screens are +// matched by label too. +const READY_TIMEOUT_MS = 30000; +const READY_POLL_MS = 1000; +const READY_MIN_NODES = 10; +const SETTLE_TIMEOUT_MS = 5000; // adb keyevents have no --settle; poll this long +const NOT_READY_LABEL = + /^(Downloading|Connecting to|Loading|Building JavaScript bundle)/i; +const NOT_READY_IDENTIFIER = /SplashScreen/i; + +// Overlay detection. The dev-menu labels are generic, so they only count when +// a dev-menu-only marker is on screen too; the dev launcher is Expo's +// "DEVELOPMENT SERVERS / RECENTLY OPENED" screen, which no dev-menu label hits. +const DEV_MENU_LABELS = ['Close', 'Continue']; +const DEV_MENU_MARKERS = [ + 'Reload', + 'Go home', + 'Fast refresh', + 'Fast Refresh', + 'TOOLS', + 'Toggle element inspector', + 'Open DevTools', +]; +const DEV_LAUNCHER_MARKERS = ['DEVELOPMENT SERVERS', 'RECENTLY OPENED']; +const METRO_PORT = '8081'; + +// The Expo dev client's floating "Tools" button: a small image control labelled +// exactly "Tools", drawn on top of the app. On Android at 480 dpi it sits in the +// top-right corner of the surface-example-elevated crop, where it diffs as a +// deterministic ~2,800-pixel "regression" with the library untouched. The dev +// menu has a "TOOLS" section of its own, so a tree showing the dev menu is +// excluded outright; its rows are text nodes, which is what the type check +// rules out (android.widget.TextView on Android, StaticText on iOS). +const FLOATING_TOOLS_LABEL = 'Tools'; +const TOOLS_TOGGLE_LABEL = 'Tools button'; // the dev menu row that toggles it +const DEV_MENU_CLOSE_LABEL = 'Close'; +const ANDROID_KEYCODE_MENU = '82'; // opens the RN dev menu via adb + +const LIST_ROOT_TITLE = 'Examples'; // Appbar title of the example-list root +const SURFACE_ROW_LABEL = 'Surface'; +const BACK_LABEL = 'Back'; + +// One fixed session name per platform, so a run is reproducible from any +// directory and the README's hand-run commands address the same session. +const sessionName = (platform) => `paper-visual-${platform}`; + +let commandSeq = 0; +let cmdDir = VISUAL_DIR; + +/** + * A failure with an exit code attached. Thrown rather than exiting on the spot + * so that main() can still write summary.json before the process ends. + */ +class RunFailure extends Error { + constructor(message, exitCode) { + super(message); + this.name = 'RunFailure'; + this.exitCode = exitCode; + } +} + +function fail(message, code = 2) { + throw new RunFailure(message, code); +} + +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function parseArgs(argv) { + const out = { + update: false, + threshold: DEFAULT_THRESHOLD, + stories: DEFAULT_STORIES, + outDir: null, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value == null) fail(`${arg} needs a value`); + return value; + }; + + switch (arg) { + case '--platform': + out.platform = next(); + break; + case '--update': + out.update = true; + break; + case '--threshold': + out.threshold = Number(next()); + break; + case '--out': + out.outDir = path.resolve(next()); + break; + case '--story': + case '--stories': + out.stories = next() + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + if (out.stories.length === 0) + fail(`${arg} needs at least one story id`); + break; + case '--help': + case '-h': + console.log( + 'usage: node example/visual/run.mjs --platform ios|android [--update]\n' + + ' [--threshold <0-1>] [--story|--stories a,b] [--out ]\n' + + '\n' + + ' --update write the captures to __baselines__// instead of diffing\n' + + ' them (the capture-size check is skipped; the captured dimensions\n' + + ' are printed instead)' + ); + process.exit(0); + break; + default: + fail(`unknown argument ${arg}`); + } + } + + if (out.platform !== 'ios' && out.platform !== 'android') { + fail('--platform must be ios or android'); + } + if ( + !Number.isFinite(out.threshold) || + out.threshold < 0 || + out.threshold > 1 + ) { + fail('--threshold must be a number between 0 and 1'); + } + if (out.outDir == null) { + out.outDir = path.join(VISUAL_DIR, 'artifacts', 'run', out.platform); + } + + return out; +} + +/** + * Environment for the nested `npx agent-device` call. When this script itself + * runs under `npx -p node@20 node run.mjs`, npx exports `npm_config_package` + * (and friends) into the child; the nested npx then reads that, installs + * node@20 and treats `agent-device@0.21.0` as a command name inside it. Strip + * every npm_config_* variable so the nested npx resolves its own package. + */ +function spawnEnv() { + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (!key.startsWith('npm_config_')) env[key] = value; + } + // A documented lookup path for the Android SDK: agent-device needs the + // `emulator` binary to list stopped AVDs at all, and looks for the SDK under + // ANDROID_SDK_ROOT, ANDROID_HOME and ~/Android/Sdk. The standard macOS + // install is at neither, so point it there when the user has set nothing. + const sdk = path.join(process.env.HOME || '', 'Library/Android/sdk'); + if (!env.ANDROID_HOME && !env.ANDROID_SDK_ROOT && fs.existsSync(sdk)) { + env.ANDROID_HOME = sdk; + } + return env; +} + +/** Runs one agent-device command, records its JSON, returns the parsed payload. */ +function ad(label, args, { allowFail = false } = {}) { + const argv = [AGENT_DEVICE, ...args, '--json']; + const startedAt = Date.now(); + const proc = spawnSync('npx', argv, { + cwd: VISUAL_DIR, + encoding: 'utf8', + env: spawnEnv(), + maxBuffer: 256 * 1024 * 1024, + }); + const elapsedMs = Date.now() - startedAt; + const stdout = proc.stdout || ''; + + let parsed = null; + const first = stdout.indexOf('{'); + const last = stdout.lastIndexOf('}'); + if (first !== -1 && last > first) { + try { + parsed = JSON.parse(stdout.slice(first, last + 1)); + } catch { + parsed = null; + } + } + + const file = path.join( + cmdDir, + `${String(++commandSeq).padStart(3, '0')}-${label}.json` + ); + fs.writeFileSync( + file, + JSON.stringify( + { + command: ['npx', ...argv], + cwd: VISUAL_DIR, + exitCode: proc.status, + elapsedMs, + response: parsed, + stdout: parsed ? undefined : stdout, + stderr: proc.stderr || undefined, + }, + null, + 2 + ) + ); + + const ok = proc.status === 0 && parsed && parsed.success !== false; + if (!ok && !allowFail) { + const reason = + parsed?.error?.details?.reason || + parsed?.error?.message || + proc.stderr?.trim() || + (proc.status === 0 + ? 'exit 0 but stdout was not parseable JSON' + : `exit ${proc.status}`); + fail(`agent-device ${label} failed: ${reason} (see ${file})`); + } + + return { + ok, + data: parsed?.data ?? null, + error: parsed?.error ?? null, + elapsedMs, + }; +} + +/** + * Snapshot helper. `--force-full` asks for the complete tree rather than a + * delta. A `--scope` that matches nothing returns success with empty nodes, so + * callers assert on node count rather than on success. + */ +function snapshot(label, globals, extra = []) { + const snap = ad(label, ['snapshot', ...extra, '--force-full', ...globals]); + return snap.data?.nodes || []; +} + +function sh(command, args) { + const proc = spawnSync(command, args, { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + return { + status: proc.status, + stdout: proc.stdout || '', + stderr: proc.stderr || '', + }; +} + +function adbPath() { + const home = process.env.HOME || ''; + const sdk = + process.env.ANDROID_HOME || + process.env.ANDROID_SDK_ROOT || + path.join(home, 'Library/Android/sdk'); + const candidate = path.join(sdk, 'platform-tools/adb'); + return fs.existsSync(candidate) ? candidate : 'adb'; +} + +/** adb addressed at the resolved device, so a second emulator cannot take the call. */ +const adb = (device, args) => sh(adbPath(), ['-s', device.id, ...args]); + +/** + * Picks the simulator the baselines belong to out of a parsed + * `xcrun simctl list -j devices`: the pinned name on the pinned runtime, a + * booted one first so a second clone is not started for nothing. Pure, so it + * can be tested against a captured listing. `elsewhere` collects the runtimes + * that do carry that name, which is the useful half of a "not found" message. + */ +function pickIosCandidate(listing, env) { + const candidates = []; + const elsewhere = new Set(); + + for (const [runtime, list] of Object.entries(listing?.devices || {})) { + for (const device of list) { + if (device.name !== env.device || device.isAvailable === false) continue; + if (runtime === env.runtime) { + candidates.push({ + id: device.udid, + name: device.name, + runtime, + state: device.state, + booted: device.state === 'Booted', + }); + } else { + elsewhere.add(runtime); + } + } + } + + return { + device: candidates.find((c) => c.booted) || candidates[0] || null, + elsewhere: [...elsewhere], + }; +} + +/** Resolves the iOS simulator from the profile. The runtime check is part of it. */ +function resolveIos(env) { + const res = sh('xcrun', ['simctl', 'list', '-j', 'devices']); + if (res.status !== 0) { + fail(`xcrun simctl list failed: exit ${res.status} ${res.stderr.trim()}`); + } + + let listing; + try { + listing = JSON.parse(res.stdout); + } catch { + fail('xcrun simctl list did not return parseable JSON'); + } + + const { device, elsewhere } = pickIosCandidate(listing, env); + if (!device) { + fail( + `no simulator named ${env.device} on ${env.runtime}` + + (elsewhere.length + ? `; that name exists on ${elsewhere.join(', ')}` + : '') + + '; create one in Xcode (Devices and Simulators)' + ); + } + return device; +} + +/** + * Picks the emulator or stopped AVD matching the profile. agent-device reports + * a running emulator under its serial and the AVD name as `name`, and a stopped + * one under the AVD name twice, so both handles are accepted; a physical device + * with the same marketing name is not an AVD and is ignored. Pure, exported for + * the tests. + */ +function pickAndroid(devices, env) { + const candidates = (devices || []).filter( + (d) => d.kind === 'emulator' && (d.id === env.avd || d.name === env.device) + ); + return candidates.find((d) => d.booted) || candidates[0] || null; +} + +/** Resolves the Android emulator from the profile, booting the AVD if it is stopped. */ +function resolveAndroid(env, adFn = ad) { + const list = adFn('devices-android', ['devices', '--platform', 'android']); + const found = pickAndroid(list.data?.devices, env); + if (!found) { + fail( + `no AVD named ${env.avd} (agent-device lists stopped AVDs only when the ` + + 'emulator binary is on PATH or ANDROID_HOME is set); create one in ' + + `Android Studio with ${env.systemImage}` + ); + } + if (found.booted) return { id: found.id, name: found.name }; + + // No --session on boot: the session may still be bound to a serial from an + // earlier emulator, and a selector naming another identity is INVALID_ARGS. + const boot = adFn( + 'boot-android', + ['boot', '--platform', 'android', '--device', found.id], + { allowFail: true } + ); + if (!boot.ok) failAd(`boot ${found.id}`, boot); + return { id: boot.data?.id || found.id, name: found.name }; +} + +/** Reports an agent-device failure with its own message and hint, verbatim. */ +function failAd(what, res) { + fail( + `agent-device ${what} failed: ${res.error?.message || res.error?.code || 'no error object in the response'}` + + (res.error?.hint ? `\nhint: ${res.error.hint}` : '') + ); +} + +/** Reads API level, release and density off the resolved device via adb. */ +function observeAndroid(serial) { + const adbBin = adbPath(); + const prop = (name) => + sh(adbBin, ['-s', serial, 'shell', 'getprop', name]).stdout.trim(); + const sdkLevel = prop('ro.build.version.sdk'); + if (!sdkLevel) { + return { + error: { + what: 'device inspection', + expected: `Android properties readable via ${adbBin} -s ${serial}`, + actual: 'no response (is the emulator still up?)', + }, + }; + } + + // An override set with `wm density` is what the screen actually renders at, + // so it wins over the physical density of the image. + const densityOut = sh(adbBin, [ + '-s', + serial, + 'shell', + 'wm', + 'density', + ]).stdout; + const density = + densityOut.match(/Override density:\s*(\d+)/) || + densityOut.match(/Physical density:\s*(\d+)/); + + return { + observed: { + apiLevel: Number(sdkLevel), + androidRelease: prop('ro.build.version.release'), + density: density ? Number(density[1]) : null, + }, + }; +} + +const DEFAULT_OBSERVERS = { observeAndroid }; + +/** + * The one check left after resolution, and Android only: an AVD name says + * nothing about the system image behind it, so the API level and the density + * the baselines were captured on are still compared. iOS needs nothing here, + * because resolving the simulator already matched the name and the runtime. + * The observer is injected so this can be tested without adb. + */ +function enforceEnvironment( + platform, + env, + device, + observers = DEFAULT_OBSERVERS +) { + if (platform !== 'android') return {}; + + const read = observers.observeAndroid(device.id); + if (read.error) { + fail( + `${read.error.what}: expected ${read.error.expected}, ` + + `observed ${read.error.actual}` + ); + } + const observed = read.observed; + + if (observed.apiLevel !== env.apiLevel || observed.density !== env.density) { + fail( + `AVD ${env.avd} reports API ${observed.apiLevel} / ` + + `${observed.density ?? 'unreadable'} dpi, baselines were captured on ` + + `API ${env.apiLevel} / ${env.density} dpi. To re-baseline on this ` + + 'image, set apiLevel and density in env.json and run with --update' + ); + } + return observed; +} + +/** Canonical baseline path: __baselines__//.png */ +function baselinePath(platform, story) { + return path.join(BASELINE_DIR, platform, `${story}.png`); +} + +/** Normal mode: a missing baseline is a hard error naming the exact path. */ +function requireBaseline(platform, story) { + const baseline = baselinePath(platform, story); + if (!fs.existsSync(baseline)) { + fail( + `no baseline for ${story}: expected ${baseline}, ` + + 'run with --update to create it' + ); + } + return baseline; +} + +/** + * `--update` mode: write the capture to the baseline, creating it if new. The + * captured dimensions go in the log line because the size check is skipped in + * this mode (see checkCaptureSize), this is the only place a human sees what + * the new baseline actually measures. + */ +function writeBaseline(platform, story, current, shotData) { + const baseline = baselinePath(platform, story); + const created = !fs.existsSync(baseline); + fs.mkdirSync(path.dirname(baseline), { recursive: true }); + fs.copyFileSync(current, baseline); + const size = `${shotData?.width ?? '?'}x${shotData?.height ?? '?'}`; + console.log( + `${platform} ${story} baseline ${created ? 'created' : 'updated'} (${size}) → ${path.relative(VISUAL_DIR, baseline)}` + ); + return { baseline, created }; +} + +/** + * Width and height out of a PNG's IHDR: the signature is 8 bytes, the IHDR + * length+type another 8, so the two big-endian uint32s live at bytes 16–24. No + * decoding and no dependency, the header is all this needs. + */ +function readPngSize(file) { + const header = Buffer.alloc(24); + let read = 0; + let fd; + try { + fd = fs.openSync(file, 'r'); + read = fs.readSync(fd, header, 0, 24, 0); + } catch { + return null; + } finally { + if (fd != null) fs.closeSync(fd); + } + const signature = '89504e470d0a1a0a'; + if ( + read < 24 || + header.toString('hex', 0, 8) !== signature || + header.toString('ascii', 12, 16) !== 'IHDR' + ) { + return null; + } + return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; +} + +/** + * Fails unless the capture has the same dimensions as the committed baseline it + * is about to be compared with. A wrong-sized capture would otherwise diff + * clean and read as PASS, and so would a capture whose size agent-device did + * not report at all, which is why a missing value is a failure rather than a + * warning. + * + * The expected size comes from the baseline PNG itself rather than from + * env.json: that makes it per story by construction, so a story that was just + * added and `--update`d passes, and it cannot drift from the file the diff + * actually uses. + * + * Skipped entirely in `--update` mode: the whole point of that mode is to + * record what the device produces now, and a new story has no baseline to check + * against. In diff mode there is no override: a size mismatch on a device that + * matches env.json means something is wrong that no flag should paper over. + */ +function checkCaptureSize( + env, + story, + shotData, + { update = false, baselineFile = null } = {} +) { + if (update) return; + + const expected = baselineFile ? readPngSize(baselineFile) : null; + const problems = []; + + if (!expected) { + fail( + `${story}: could not read the dimensions of the baseline PNG ` + + `(${baselineFile ?? 'none given'}), so the capture cannot be ` + + 'size-checked, re-record the baseline with --update', + 3 + ); + } + + const compare = (what, want, got) => { + if (got == null) { + problems.push( + `${what}: expected ${want}, agent-device reported no ${what}` + ); + return; + } + if (Number(got) !== Number(want)) { + problems.push(`${what}: expected ${want}, actual ${got}`); + } + }; + + compare('width', expected.width, shotData?.width); + compare('height', expected.height, shotData?.height); + // pixelDensity is reported (and pinnable) on iOS only. Android screenshots + // are native device pixels and env.json pins no density there, so the check + // is skipped rather than warned about. + if (env.pixelDensity != null) { + compare('pixelDensity', env.pixelDensity, shotData?.pixelDensity); + } + + if (problems.length === 0) return; + + const detail = problems.map((p) => ` - ${p}`).join('\n'); + fail( + `${story}: capture dimensions do not match the baseline ` + + `${path.relative(VISUAL_DIR, baselineFile)}:\n${detail}\n` + + 'a wrong-sized capture cannot be compared with the baseline. Check ' + + '--crop-on/--pixel-density and the device; if the new size is the intended ' + + 'one, re-record the baselines with --update.', + 3 + ); +} + +const labelOf = (node) => (node.label || '').trim(); +const hasLabel = (nodes, label) => nodes.some((n) => labelOf(n) === label); +const signatureOf = (nodes) => nodes.map(labelOf).join('|'); + +/** + * Nodes that belong to the app. On Android a snapshot also carries the system + * UI status bar (~10 nodes), which is present even while the app is still + * starting, so it must not count towards readiness. + */ +function appNodes(nodes) { + return nodes.filter((n) => !n.bundleId || n.bundleId === BUNDLE_ID); +} + +function looksReady(nodes) { + const own = appNodes(nodes); + if (own.length <= READY_MIN_NODES) return false; + return !own.some( + (n) => + NOT_READY_LABEL.test(labelOf(n)) || + NOT_READY_IDENTIFIER.test(n.identifier || '') + ); +} + +/** + * Polls the accessibility tree until the app has rendered something real: + * more than READY_MIN_NODES app nodes and no splash/"Downloading" node. Bounded + * so a stuck launch fails instead of hanging. + */ +function waitForAppReady(globals, timeoutMs = READY_TIMEOUT_MS) { + const deadline = Date.now() + timeoutMs; + let nodes = snapshot('snapshot-ready', globals); + while (!looksReady(nodes) && Date.now() < deadline) { + sleepSync(READY_POLL_MS); + nodes = snapshot('snapshot-ready', globals); + } + if (!looksReady(nodes)) { + fail( + `the app did not become ready within ${timeoutMs}ms, the last snapshot ` + + `had ${appNodes(nodes).length} app nodes ` + + `(${appNodes(nodes) + .slice(0, 3) + .map((n) => labelOf(n) || n.identifier || n.type) + .join(' / ')})` + ); + } + console.log(` app ready (${appNodes(nodes).length} app nodes)`); + return nodes; +} + +/** The RN dev menu: a Close/Continue control with a dev-menu-only marker next to it. */ +/** + * Polls the tree until `predicate(nodes)` holds or the timeout passes, and + * returns the last snapshot either way. Used after an adb keyevent, which has + * no `--settle`: the first snapshot can still show the screen from before the + * key was injected. + */ +function waitUntil(globals, predicate, timeoutMs = SETTLE_TIMEOUT_MS) { + const deadline = Date.now() + timeoutMs; + let nodes = snapshot('snapshot-settle', globals); + while (!predicate(nodes) && Date.now() < deadline) { + sleepSync(READY_POLL_MS); + nodes = snapshot('snapshot-settle', globals); + } + return nodes; +} + +function findDevMenuNode(nodes) { + const marker = nodes.some((n) => DEV_MENU_MARKERS.includes(labelOf(n))); + if (!marker) return null; + return nodes.find((n) => DEV_MENU_LABELS.includes(labelOf(n))) || null; +} + +/** The floating dev-client "Tools" button, or null when it is not on screen. */ +function findFloatingToolsNode(nodes) { + if (nodes.some((n) => DEV_MENU_MARKERS.includes(labelOf(n)))) return null; + return ( + nodes.find( + (n) => labelOf(n) === FLOATING_TOOLS_LABEL && !/text/i.test(n.type || '') + ) || null + ); +} + +function describeNode(node) { + const rect = node.rect || {}; + const round = (value) => Math.round(value ?? -1); + return ( + `${node.type || 'node'} label="${labelOf(node)}" at ` + + `{x:${round(rect.x)}, y:${round(rect.y)}, ` + + `w:${round(rect.width)}, h:${round(rect.height)}}` + ); +} + +/** + * Turns the floating dev-client "Tools" button off through the dev menu, once, + * and returns the settled tree. A dev overlay inside the crop must never be + * reported as a visual regression, so anything that leaves it on screen, a + * platform with no way to open the dev menu, or a toggle that did not take, + * is a setup error naming the node and the manual fix. + * + * Android opens the dev menu with KEYCODE_MENU. agent-device 0.21.0 has no + * dev-menu/shake command (checked in `help commands`), so on iOS there is + * nothing to drive and the run stops instead. + */ +function disableFloatingTools(platform, globals, device, nodes) { + const tools = findFloatingToolsNode(nodes); + if (!tools) return nodes; + + const giveUp = (why) => + fail( + `the dev-client floating Tools button is inside the capture area ` + + `(${describeNode(tools)}) and ${why}. Turn it off on the device: ` + + 'dev menu → Tools button.' + ); + + if (platform !== 'android') giveUp('cannot be toggled automatically on iOS'); + + const menu = adb(device, [ + 'shell', + 'input', + 'keyevent', + ANDROID_KEYCODE_MENU, + ]); + if (menu.status !== 0) { + fail( + `adb input keyevent ${ANDROID_KEYCODE_MENU} failed: ${menu.stderr.trim()}` + ); + } + + let current = waitUntil(globals, (nodes) => + hasLabel(nodes, TOOLS_TOGGLE_LABEL) + ); + for (const label of [TOOLS_TOGGLE_LABEL, DEV_MENU_CLOSE_LABEL]) { + if (!current.some((n) => labelOf(n) === label)) { + giveUp(`the dev menu has no "${label}" entry`); + } + console.log(` dev menu: pressing "${label}"`); + // By selector, not by ref: the dev menu's "Close" is a container whose + // tappable area belongs entirely to an unlabelled child, so pressing its + // own ref is rejected (covered_by_interactive_descendants). The selector + // lets agent-device resolve down to that child. + ad(`press-${label.replace(/\s+/g, '-').toLowerCase()}`, [ + 'press', + `label="${label}"`, + '--settle', + ...globals, + ]); + current = waitForAppReady(globals); + } + + if (findFloatingToolsNode(current)) + giveUp('is still on screen after one toggle'); + + console.log( + ' dev-client floating Tools button was inside the capture area, ' + + 'disabled it via the dev menu' + ); + return current; +} + +/** + * The Expo dev launcher ("DEVELOPMENT SERVERS" / "RECENTLY OPENED"). It is left + * behind by an Android relaunch and none of the dev-menu labels match it. The + * way out is the recently-opened row carrying the Metro URL; if the launcher is + * up but that row is not there, the run fails rather than pressing whatever row + * happens to be nearby, which could open a different app. + */ +function findDevLauncherNode(nodes) { + if (!nodes.some((n) => DEV_LAUNCHER_MARKERS.includes(labelOf(n)))) { + return null; + } + + const metroRow = nodes.find( + (n) => labelOf(n).startsWith('http://') && labelOf(n).includes(METRO_PORT) + ); + if (metroRow) return metroRow; + + fail( + 'the Expo dev launcher is on screen but has no recently-opened row for ' + + `http://…:${METRO_PORT}, start Metro and open the app from the launcher once` + ); + return null; // unreachable +} + +/** + * Clears the two dev overlays a relaunch can land in, handling them distinctly + * and re-checking after every press. Returns the settled tree. + */ +function dismissOverlays(globals) { + let nodes = waitForAppReady(globals); + + for (let step = 0; step < MAX_OVERLAY_STEPS; step++) { + const devMenu = findDevMenuNode(nodes); + if (devMenu) { + console.log( + ` dev menu: pressing "${labelOf(devMenu)}" (@${devMenu.ref})` + ); + ad('press-devmenu', ['press', `@${devMenu.ref}`, '--settle', ...globals]); + nodes = waitForAppReady(globals); + continue; + } + + const launcher = findDevLauncherNode(nodes); + if (launcher) { + console.log( + ` dev launcher: pressing "${labelOf(launcher)}" (@${launcher.ref})` + ); + ad('press-launcher', [ + 'press', + `@${launcher.ref}`, + '--settle', + ...globals, + ]); + nodes = waitForAppReady(globals); + continue; + } + + return nodes; + } + + fail( + `a dev overlay was still on screen after ${MAX_OVERLAY_STEPS} dismissal ` + + 'attempts (dev menu and/or Expo dev launcher)' + ); + return nodes; // unreachable +} + +function onSurfaceScreen(globals, stories) { + const nodes = snapshot('snapshot-raw', globals, ['--raw']); + return stories.every((story) => nodes.some((n) => n.identifier === story)); +} + +/** + * The example-list root is the only screen whose Appbar title is "Examples", + * and it shows the drawer button where every example screen shows "Back". Both + * signals are required: "no Back element" on its own is also true of a screen + * with no header at all, which would make any such screen read as the list. + */ +function atListRoot(nodes) { + return hasLabel(nodes, LIST_ROOT_TITLE) && !hasLabel(nodes, BACK_LABEL); +} + +/** + * Pops the navigation stack until the example list root is on screen. Uses the + * Appbar "Back" element when there is one (present on both platforms), and + * falls back to platform back navigation for a screen with no header. + */ +function goBackToListRoot(platform, globals, device, nodes) { + let current = nodes; + + for (let step = 0; step < MAX_BACK_STEPS; step++) { + if (atListRoot(current)) return current; + + const back = current.find((n) => labelOf(n) === BACK_LABEL); + if (back) { + console.log(` pressing "Back" (@${back.ref})`); + ad('press-back', ['press', `@${back.ref}`, '--settle', ...globals]); + } else if (platform === 'android') { + console.log(' no "Back" element, sending Android KEYCODE_BACK'); + const res = adb(device, ['shell', 'input', 'keyevent', '4']); + if (res.status !== 0) { + fail(`adb input keyevent 4 failed: ${res.stderr.trim()}`); + } + // No --settle on a keyevent: wait for the screen to actually change. + const before = signatureOf(current); + current = waitUntil( + globals, + (nodes) => atListRoot(nodes) || signatureOf(nodes) !== before + ); + continue; + } else { + console.log(' no "Back" element, using system back'); + ad('back', ['back', '--system', '--settle', ...globals]); + } + + current = waitForAppReady(globals); + } + + fail( + `could not reach the example list: after ${MAX_BACK_STEPS} back steps the ` + + `"${LIST_ROOT_TITLE}" title was still not on screen (last screen: ` + + `${current + .map((n) => labelOf(n)) + .filter(Boolean) + .slice(0, 5) + .join(' / ')})` + ); + return current; // unreachable +} + +/** + * Picks the "Surface" LIST ROW out of a snapshot of the example list. + * + * Several nodes can carry that label, and pressing the wrong one is a no-op + * that reads as a navigation failure: the Appbar title of the Surface screen + * itself, and on Android both the row container and its inset text child. The + * row is the candidate that + * - sits below the Appbar, rect.y at or past the bottom of the + * "Examples" title node (iOS 108pt, Android 294px), which is what excludes + * any header/title node, and + * - spans the list, width at least 90% of the screen, which excludes the + * Android Appbar title (1100 of 1280px) a second way, and then the widest + * of what is left, i.e. the row container (iOS 402 of 402pt, Android 1280 + * of 1280px) rather than its inset text child (Android 1160px). + * + * With no "Examples" title in the tree this is not the list root, so there is + * no row to press and the function reports nothing rather than guessing. + */ +function findSurfaceRow(nodes) { + const titles = nodes.filter((n) => labelOf(n) === LIST_ROOT_TITLE); + if (titles.length === 0) return null; + const headerBottom = Math.max( + ...titles.map((n) => (n.rect?.y ?? 0) + (n.rect?.height ?? 0)) + ); + const screenWidth = Math.max(0, ...nodes.map((n) => n.rect?.width ?? 0)); + + const candidates = nodes.filter( + (n) => + labelOf(n) === SURFACE_ROW_LABEL && + (n.rect?.y ?? -1) >= headerBottom && + (n.rect?.width ?? 0) >= 0.9 * screenWidth + ); + if (candidates.length === 0) return null; + + return candidates.sort( + (a, b) => (b.rect?.width || 0) - (a.rect?.width || 0) + )[0]; +} + +/** + * Scrolls down the example list until the "Surface" row shows up, then opens + * it. The caller must have put the app on the list root first, since scrolling + * only ever goes down. + */ +function navigateToSurface(globals, stories) { + for (let step = 0; step <= MAX_SCROLL_STEPS; step++) { + const nodes = snapshot('snapshot-list', globals); + const target = findSurfaceRow(nodes); + + if (target) { + // Press by ref off a fresh snapshot: bare `find` presses its match + // (`find ... list` is the read-only form), and on Android the row and + // its text child share the label, which is AMBIGUOUS_MATCH. + console.log( + ` pressing the "Surface" row (@${target.ref}, ${target.type}, ` + + `y=${Math.round(target.rect?.y ?? -1)}, w=${Math.round(target.rect?.width ?? -1)})` + ); + ad('press-surface', ['press', `@${target.ref}`, '--settle', ...globals]); + + if (onSurfaceScreen(globals, stories)) return; + fail( + `pressed the "Surface" row but the example screen did not appear ` + + `(expected ids: ${stories.join(', ')})` + ); + } + + if (step < MAX_SCROLL_STEPS) { + ad('scroll', [ + 'scroll', + 'down', + String(SCROLL_STEP), + '--settle', + ...globals, + ]); + } + } + + fail( + `could not find the "Surface" row after ${MAX_SCROLL_STEPS} scrolls of ` + + `${SCROLL_STEP} rows. Either the example list is not on screen or the ` + + 'list grew past the scroll budget (MAX_SCROLL_STEPS)' + ); +} + +/** + * Opens the app and binds the session to the resolved device in the same call. + * A session left over from an earlier run can still be bound to a device that + * is gone (a re-created emulator gets a new serial), and agent-device rejects + * the selector rather than rebinding, so that one case is recovered by closing + * the session and opening again. Everything else is a setup error carrying + * agent-device's own message and hint, a foreign claim on the device included. + */ +function openApp(label, extra, globals, bind) { + const args = ['open', BUNDLE_ID, ...extra, ...globals, ...bind]; + const first = ad(label, args, { allowFail: true }); + if (first.ok) return; + + const code = first.error?.code; + if (code !== 'INVALID_ARGS' && code !== 'DEVICE_NOT_FOUND') + failAd(label, first); + + console.log(` ${code} from open, closing the session and retrying once`); + ad('close-session', ['close', ...globals], { allowFail: true }); + const retry = ad(`${label}-retry`, args, { allowFail: true }); + if (!retry.ok) failAd(`${label} (after close)`, retry); +} + +/** + * Relaunches the app and leaves it on the Surface example screen. + * + * The relaunch is unconditional. The app persists its navigation state + * (PERSISTENCE_KEY in example/src/index.tsx), so "already on the Surface + * screen" says nothing about which screen a plain `open` will land on, and, + * more importantly, only a relaunch makes the app fetch the current JS bundle + * from Metro. Skipping it on the strength of the screen that happens to be up + * captured a stale bundle on Android. + */ +function openOnSurfaceScreen(platform, globals, bind, device, stories) { + console.log(' relaunching the app (fresh bundle from Metro)'); + if (platform === 'ios') { + openApp('open-relaunch', ['--relaunch'], globals, bind); + } else { + const stop = adb(device, ['shell', 'am', 'force-stop', BUNDLE_ID]); + if (stop.status !== 0) { + fail(`adb force-stop ${BUNDLE_ID} failed: ${stop.stderr.trim()}`); + } + openApp('open-after-force-stop', [], globals, bind); + } + + const nodes = disableFloatingTools( + platform, + globals, + device, + dismissOverlays(globals) + ); + + if (onSurfaceScreen(globals, stories)) { + console.log(' restored onto the Surface screen'); + return; + } + + console.log(' not on the Surface screen, going back to the example list'); + const rootNodes = goBackToListRoot(platform, globals, device, nodes); + console.log(` at the example list root (${rootNodes.length} nodes)`); + navigateToSurface(globals, stories); +} + +/** + * The whole device pass. Mutates `state` as it goes so main() can write a + * summary whether this returns or throws. Returns the exit code (0 or 1). + */ +function runPass(state) { + const opts = state.opts; + const { platform } = opts; + + if (!fs.existsSync(ENV_FILE)) fail(`missing ${ENV_FILE}`); + const envFile = JSON.parse(fs.readFileSync(ENV_FILE, 'utf8')); + const env = + platform === 'ios' + ? { ...envFile, android: undefined } + : { ...envFile.android }; + state.env = env; + + // Resolve the instance from the profile, boot it if it is down, and only + // then bind a session to it: `boot` takes a selector and no session, because + // a session left bound to another identity would reject the selector. + const device = platform === 'ios' ? resolveIos(env) : resolveAndroid(env); + if (platform === 'ios' && !device.booted) { + const boot = ad( + 'boot-ios', + ['boot', '--platform', 'ios', '--udid', device.id], + { allowFail: true } + ); + if (!boot.ok) failAd(`boot ${device.id}`, boot); + } + state.device = { id: device.id, name: device.name }; + console.log(`# ${platform} ${device.name} (${device.id})`); + if (platform === 'android') { + // A cold-booted emulator has no tunnel to Metro; the dev launcher's + // http://localhost:8081 row loads nothing without it. + const reverse = adb(device, ['reverse', 'tcp:8081', 'tcp:8081']); + if (reverse.status !== 0) { + console.log(` adb reverse failed: ${reverse.stderr.trim()}`); + } + } + console.log( + `# threshold ${opts.threshold}${opts.update ? ' (update)' : ''}, output: ${opts.outDir}` + ); + + // `open` binds the session to the device; every later command addresses the + // session alone, so no selector can conflict with that binding mid-run. + const globals = ['--platform', platform, '--session', sessionName(platform)]; + const bind = [platform === 'ios' ? '--udid' : '--serial', device.id]; + + state.observedEnv = enforceEnvironment(platform, env, device); + + openOnSurfaceScreen(platform, globals, bind, device, opts.stories); + ad('wait-stable', ['wait', 'stable', '500', '10000', ...globals]); + sleepSync(2000); // covers the customFontLoaded theme swap, which has no node change + + let failed = false; + + for (const story of opts.stories) { + const current = path.join(opts.outDir, `${story}.png`); + const shotArgs = [ + 'screenshot', + current, + '--crop-on', + `id="${story}"`, + ...globals, + ]; + // --pixel-density is iOS-only (UNSUPPORTED_OPERATION on Android). + if (platform === 'ios') + shotArgs.push('--pixel-density', String(env.pixelDensity || 3)); + const shot = ad(`screenshot-${story}`, shotArgs); + + if (opts.update) { + checkCaptureSize(env, story, shot.data, { update: true }); + const { baseline, created } = writeBaseline( + platform, + story, + current, + shot.data + ); + state.results.push({ + story, + updated: !created, + created, + baseline, + current, + width: shot.data?.width ?? null, + height: shot.data?.height ?? null, + }); + continue; + } + + const baseline = requireBaseline(platform, story); + checkCaptureSize(env, story, shot.data, { baselineFile: baseline }); + + const diffOut = path.join(opts.outDir, `${story}-diff.png`); + const diff = ad(`diff-${story}`, [ + 'diff', + 'screenshot', + '--baseline', + baseline, + current, + '--out', + diffOut, + '--threshold', + String(opts.threshold), + ...globals, + ]); + + const changed = diff.data?.differentPixels ?? -1; + const pct = diff.data?.mismatchPercentage ?? -1; + const regions = diff.data?.regions?.length ?? 0; + if (typeof diff.data?.match !== 'boolean') { + fail( + `diff screenshot for ${story} returned no boolean "match" ` + + `(got ${JSON.stringify(diff.data)}); is agent-device still ${AGENT_DEVICE}?` + ); + } + const pass = diff.data.match; + if (!pass) failed = true; + + console.log( + `${platform} ${story} changed=${changed} (${pct}%) regions=${regions} threshold=${opts.threshold} → ${pass ? 'PASS' : 'FAIL'}` + ); + + state.results.push({ + story, + baseline, + current, + diff: diffOut, + totalPixels: diff.data?.totalPixels ?? null, + changedPixels: changed, + mismatchPercentage: pct, + regions: diff.data?.regions ?? [], + threshold: opts.threshold, + pass, + }); + } + + return failed ? 1 : 0; +} + +function writeSummary(state, exitCode, error) { + const { opts } = state; + const summary = { + platform: opts.platform, + threshold: opts.threshold, + update: opts.update, + stories: opts.stories, + outDir: opts.outDir, + agentDeviceVersion: AGENT_DEVICE, + session: sessionName(opts.platform), + device: state.device, + expectedEnv: state.env, + observedEnv: state.observedEnv, + startedAt: new Date(state.startedAt).toISOString(), + wallClockMs: Date.now() - state.startedAt, + results: state.results, + status: exitCode === 0 ? 'pass' : exitCode === 1 ? 'fail' : 'error', + exitCode, + error, + pass: exitCode === 0, + }; + const file = path.join(opts.outDir, 'summary.json'); + fs.mkdirSync(opts.outDir, { recursive: true }); + fs.writeFileSync(file, JSON.stringify(summary, null, 2)); + return file; +} + +function main() { + const argv = process.argv.slice(2); + + // An argument error happens before there are options to summarise (not even + // an output directory), so it just prints and exits; every later failure + // goes through writeSummary below. + let opts; + try { + opts = parseArgs(argv); + } catch (thrown) { + if (!(thrown instanceof RunFailure)) throw thrown; + console.error(`error: ${thrown.message}`); + process.exit(thrown.exitCode); + } + + const state = { + opts, + env: null, + device: null, + observedEnv: {}, + results: [], + startedAt: Date.now(), + }; + + let exitCode = 0; + let error = null; + try { + cmdDir = path.join(opts.outDir, 'cmds'); + try { + fs.mkdirSync(cmdDir, { recursive: true }); + } catch (thrown) { + fail(`cannot create ${cmdDir}: ${thrown.message}`); + } + exitCode = runPass(state); + } catch (thrown) { + if (thrown instanceof RunFailure) { + console.error(`error: ${thrown.message}`); + exitCode = thrown.exitCode; + error = { message: thrown.message, exitCode: thrown.exitCode }; + } else { + console.error(thrown?.stack || String(thrown)); + exitCode = 2; + error = { + message: thrown?.message || String(thrown), + stack: thrown?.stack, + exitCode: 2, + }; + } + } + + try { + const summaryFile = writeSummary(state, exitCode, error); + console.log(`# summary: ${summaryFile} (exit ${exitCode})`); + } catch (thrown) { + // The output directory itself is unusable; the error above already says so. + console.error(`no summary written: ${thrown.message}`); + } + process.exit(exitCode); +} + +/** + * True when this module is the process entry point. `import.meta.main` would + * say the same thing but only exists on Node >= 24.2, and this script has to + * run on the Node 20 floor in example/package.json. Both sides are realpath'd + * so a symlinked invocation still matches. + */ +function isMainModule(argvPath, moduleUrl) { + if (!argvPath) return false; + const real = (p) => { + try { + return fs.realpathSync(p); + } catch { + return path.resolve(p); + } + }; + return real(argvPath) === real(fileURLToPath(moduleUrl)); +} + +if (isMainModule(process.argv[1], import.meta.url)) { + main(); +} + +// Exported for example/visual/run.test.mjs only. +export { + RunFailure, + parseArgs, + isMainModule, + checkCaptureSize, + enforceEnvironment, + pickAndroid, + pickIosCandidate, + resolveAndroid, + atListRoot, + findFloatingToolsNode, + findSurfaceRow, +}; diff --git a/example/visual/run.test.mjs b/example/visual/run.test.mjs new file mode 100644 index 0000000000..6b5b0db4cc --- /dev/null +++ b/example/visual/run.test.mjs @@ -0,0 +1,308 @@ +/** + * Unit tests for example/visual/run.mjs. No device, no adb/xcrun, no + * agent-device, no new dependencies, only the pure decision helpers are + * exercised, with the device observers faked. + * + * node --test example/visual/run.test.mjs + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + RunFailure, + atListRoot, + checkCaptureSize, + enforceEnvironment, + findFloatingToolsNode, + findSurfaceRow, + isMainModule, + parseArgs, + pickAndroid, + pickIosCandidate, + resolveAndroid, +} from './run.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const RUN_URL = new URL('./run.mjs', import.meta.url); +const RUN_PATH = fileURLToPath(RUN_URL); + +const env = {}; +const wrongSize = { width: 1280, height: 700 }; + +/** + * Writes a file carrying a PNG signature and an IHDR of the given size. Only + * those first 24 bytes are what run.mjs reads, so no encoder is needed. + */ +function writePngHeader(dir, name, width, height) { + const header = Buffer.alloc(24); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(header, 0); + header.writeUInt32BE(13, 8); + header.write('IHDR', 12, 'ascii'); + header.writeUInt32BE(width, 16); + header.writeUInt32BE(height, 20); + const file = path.join(dir, name); + fs.writeFileSync(file, header); + return file; +} + +const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'paper-visual-')); +process.on('exit', () => fs.rmSync(TMP_DIR, { recursive: true, force: true })); + +test('isMainModule is false for another argv[1] and true for the script itself', () => { + assert.equal(isMainModule(path.join(HERE, 'other.mjs'), RUN_URL), false); + assert.equal(isMainModule(undefined, RUN_URL), false); + assert.equal(isMainModule(RUN_PATH, RUN_URL), true); +}); + +test('checkCaptureSize skips the check in update mode', () => { + // Regression: this is the gate that used to abort `--update` before + // writeBaseline ever ran, so a story with no baseline could never get one. + assert.doesNotThrow(() => + checkCaptureSize(env, 'new-story', wrongSize, { update: true }) + ); +}); + +test('parseArgs rejects an empty story list and unknown flags', () => { + const exit2 = (thrown) => + thrown instanceof RunFailure && thrown.exitCode === 2; + // A typo here used to yield zero stories, zero captures and a green summary. + assert.throws(() => parseArgs(['--platform', 'ios', '--story', '']), exit2); + assert.throws(() => parseArgs(['--platform', 'ios', '--story', ',,']), exit2); + assert.throws(() => parseArgs(['--platform', 'ios', '--force']), exit2); + assert.throws(() => parseArgs(['--platform', 'web']), exit2); + assert.deepEqual( + parseArgs(['--platform', 'android', '--story', ' a, b ']).stories, + ['a', 'b'] + ); +}); + +test('checkCaptureSize fails with exit code 3 in diff mode', () => { + const baselineFile = writePngHeader(TMP_DIR, 'story.png', 1206, 642); + + assert.throws( + () => checkCaptureSize(env, 'story', wrongSize, { baselineFile }), + (thrown) => { + assert.ok(thrown instanceof RunFailure); + assert.equal(thrown.exitCode, 3); + assert.match(thrown.message, /capture dimensions do not match/); + return true; + } + ); + + assert.doesNotThrow(() => + checkCaptureSize( + env, + 'story', + { width: 1206, height: 642 }, + { + baselineFile, + } + ) + ); + + // A story added today: whatever size its freshly written baseline has is the + // size the check expects, so no env.json edit is needed to make it pass. + const fresh = writePngHeader(TMP_DIR, 'new-story.png', 837, 219); + assert.doesNotThrow(() => + checkCaptureSize( + env, + 'new-story', + { width: 837, height: 219 }, + { + baselineFile: fresh, + } + ) + ); +}); + +test('enforceEnvironment compares the image of the device it resolved', () => { + // Fake observer: the check compares env.json with this, so the test needs no + // adb on PATH. It also records the serial it was addressed with. + const seen = []; + const observers = { + observeAndroid: (serial) => { + seen.push(serial); + return { observed: { apiLevel: 37, androidRelease: '17', density: 480 } }; + }, + }; + const device = { id: 'emulator-5556', name: 'Pixel 10 Pro' }; + const check = (apiLevel, density) => + enforceEnvironment( + 'android', + { avd: 'Pixel_10_Pro', apiLevel, density }, + device, + observers + ); + + assert.throws( + () => check(36, 420), + (thrown) => { + assert.ok(thrown instanceof RunFailure); + assert.equal(thrown.exitCode, 2); + // Both the observed and the expected image are named. + assert.match(thrown.message, /API 37 \/ 480 dpi/); + assert.match(thrown.message, /API 36 \/ 420 dpi/); + return true; + } + ); + assert.doesNotThrow(() => check(37, 480)); + // adb was addressed at the resolved emulator, not at adb's default device. + assert.deepEqual(seen, ['emulator-5556', 'emulator-5556']); + // iOS is fully resolved by then, so there is nothing left to observe. + assert.deepEqual(enforceEnvironment('ios', {}, device, observers), {}); + assert.equal(seen.length, 2); +}); + +// Shapes captured from `agent-device devices --platform android --json`: a +// running emulator is listed under its serial, a stopped AVD under its name. +const emu = (id, name, booted, kind = 'emulator') => ({ + id, + name, + kind, + booted, +}); +const COLD = emu('Pixel_10_Pro', 'Pixel_10_Pro', false); +const WARM = emu('emulator-5554', 'Pixel 10 Pro', true); +const PROFILE = { avd: 'Pixel_10_Pro', device: 'Pixel 10 Pro' }; + +test('pickAndroid finds the AVD warm or cold and ignores a real device', () => { + assert.equal(pickAndroid([COLD], PROFILE)?.id, 'Pixel_10_Pro'); + assert.equal(pickAndroid([WARM], PROFILE)?.id, 'emulator-5554'); + // A booted emulator wins over the stopped AVD entry for the same profile. + assert.equal(pickAndroid([COLD, WARM], PROFILE)?.id, 'emulator-5554'); + // A physical phone with the same marketing name is not the AVD. + assert.equal( + pickAndroid([emu('R5CT80', 'Pixel 10 Pro', true, 'device')], PROFILE), + null + ); + assert.equal(pickAndroid([], PROFILE), null); +}); + +test('pickIosCandidate matches the name on the pinned runtime only', () => { + // Shape of `xcrun simctl list -j devices`, trimmed to what the picker reads. + const sim = (udid, name, state, isAvailable = true) => ({ + udid, + name, + state, + isAvailable, + }); + const PINNED = 'com.apple.CoreSimulator.SimRuntime.iOS-26-5'; + const OTHER = 'com.apple.CoreSimulator.SimRuntime.iOS-26-4'; + const profile = { device: 'iPhone 17 Pro', runtime: PINNED }; + const listing = (proState) => ({ + devices: { + [PINNED]: [ + sim('U-PRO', 'iPhone 17 Pro', proState), + sim('U-MAX', 'iPhone 17 Pro Max', 'Shutdown'), + sim('U-17', 'iPhone 17', 'Shutdown'), + ], + [OTHER]: [sim('U-OLD', 'iPhone 17 Pro', 'Booted')], + }, + }); + + const picked = pickIosCandidate(listing('Booted'), profile); + assert.equal(picked.device.id, 'U-PRO'); + assert.equal(picked.device.booted, true); + // The other runtime's simulator is not chosen even when it is the only + // Booted one, and neither is the Pro Max on the pinned runtime. + assert.equal( + pickIosCandidate(listing('Shutdown'), profile).device.id, + 'U-PRO' + ); + + // Nothing on the pinned runtime: report the runtimes that do have the name. + const missing = pickIosCandidate( + { devices: { [OTHER]: [sim('U-OLD', 'iPhone 17 Pro', 'Booted')] } }, + profile + ); + assert.equal(missing.device, null); + assert.deepEqual(missing.elsewhere, [OTHER]); +}); + +test('resolveAndroid boots a stopped AVD by name and takes the serial back', () => { + const calls = []; + const fakeAd = (label, args) => { + calls.push(args); + return label === 'devices-android' + ? { ok: true, data: { devices: [COLD] } } + : { ok: true, data: { id: 'emulator-5554', booted: true } }; + }; + + assert.equal(resolveAndroid(PROFILE, fakeAd).id, 'emulator-5554'); + // A session on boot would collide with a binding left from an earlier run. + assert.deepEqual(calls[1], [ + 'boot', + '--platform', + 'android', + '--device', + 'Pixel_10_Pro', + ]); +}); + +test('atListRoot needs the "Examples" title and no Back element', () => { + const title = { label: 'Examples' }; + const back = { label: 'Back' }; + + assert.equal(atListRoot([title]), true); + assert.equal(atListRoot([title, back]), false); + // A screen with no header at all is not the list root. + assert.equal(atListRoot([{ label: 'Elevated surface' }]), false); + assert.equal(atListRoot([]), false); +}); + +test('findFloatingToolsNode finds the dev-client button, not the dev menu', () => { + // Shape taken from a real Android snapshot (480 dpi, top-right of the crop). + const floating = { + ref: 'e18', + type: 'android.widget.ImageView', + label: 'Tools', + rect: { x: 1115, y: 243, width: 78, height: 78 }, + }; + + assert.equal( + findFloatingToolsNode([{ label: 'Elevated surface' }, floating])?.ref, + 'e18' + ); + // The dev menu has a "TOOLS" section of its own: not the floating button. + assert.equal( + findFloatingToolsNode([ + { type: 'android.widget.TextView', label: 'TOOLS' }, + { type: 'android.widget.TextView', label: 'Reload' }, + { type: 'android.widget.TextView', label: 'Tools' }, + ]), + null + ); + assert.equal(findFloatingToolsNode([]), null); +}); + +test('findSurfaceRow picks the list row, not the Appbar title', () => { + const nodes = [ + { + ref: 'title', + label: 'Examples', + rect: { y: 60, height: 48, width: 402 }, + }, + // Appbar title of the Surface screen: above the list, must not win. + { + ref: 'header', + label: 'Surface', + rect: { y: 60, height: 48, width: 402 }, + }, + // Inset text child: below the header but too narrow to be the row. + { + ref: 'child', + label: 'Surface', + rect: { y: 300, height: 48, width: 360 }, + }, + { ref: 'row', label: 'Surface', rect: { y: 300, height: 48, width: 402 } }, + ]; + + assert.equal(findSurfaceRow(nodes)?.ref, 'row'); + // No "Examples" title means this is not the list: report nothing. + assert.equal(findSurfaceRow(nodes.filter((n) => n.ref !== 'title')), null); +});