From 54eb9325913d2794feab16166f4fe2810bc85c21 Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 13:37:29 +0200 Subject: [PATCH 1/6] feat(example): add a frame-time benchmark harness Measures main-thread frame intervals, JS-thread stalls and memory while the map is driven through fixed scenarios, so the marker pipeline can be judged on numbers instead of estimates. - example/modules/frame-stats: a local Expo module. CADisplayLink on iOS and Choreographer.FrameCallback on Android record every main-thread frame interval together with the interval the display was running at, so jank is judged against the display's own budget and ProMotion rate changes do not count as jank. Also exposes phys_footprint / PSS, the display refresh rate, and a system-log writer so release builds can be harvested without Metro. - example/benchmark: nearest-rank percentiles, jank and dropped-frame counts, pass/fail rules scaled to the frame budget (unit tested), a JS-thread lag sampler, scenarios A to L driven by animated camera moves and prop updates, a runner, and a BenchmarkApp screen with Run all, per-scenario runs, a manual recorder for real gestures and JSON export. - example/index.js picks the harness when EXPO_PUBLIC_BENCHMARK=1; the demo bundle is unchanged otherwise. app.json sets CADisableMinimumFrameDurationOnPhone so a ProMotion iPhone is measured at 120 Hz. - Maestro flows for the scripted run and a real-gesture pan, a script that turns captured [benchmark] lines into a Markdown table, and docs/benchmarks.md with the method, thresholds, scenarios, how to run and collect, and two labeled smoke runs (iPhone 17 Pro simulator, Android API 35 emulator). --- docs/benchmarks.md | 183 ++++++++ eslint.config.mjs | 2 +- example/app.json | 5 +- example/benchmark/BenchmarkApp.tsx | 440 ++++++++++++++++++ .../benchmark/__tests__/frameStats.test.ts | 73 +++ .../benchmark/__tests__/thresholds.test.ts | 87 ++++ example/benchmark/datasets.ts | 105 +++++ example/benchmark/frameStats.ts | 78 ++++ example/benchmark/jsLagSampler.ts | 50 ++ example/benchmark/runner.ts | 108 +++++ example/benchmark/scenarios.ts | 245 ++++++++++ example/benchmark/thresholds.ts | 69 +++ example/examples/advancedFeatures.ts | 2 +- example/index.js | 7 +- example/maestro/benchmark-pan.yaml | 34 ++ example/maestro/benchmark-run-all.yaml | 18 + .../modules/frame-stats/android/build.gradle | 15 + .../android/src/main/AndroidManifest.xml | 1 + .../expo/modules/framestats/FrameRecorder.kt | 61 +++ .../modules/framestats/FrameStatsModule.kt | 56 +++ .../frame-stats/expo-module.config.json | 9 + example/modules/frame-stats/index.ts | 1 + .../frame-stats/ios/FrameRecorder.swift | 79 ++++ .../frame-stats/ios/FrameStats.podspec | 25 + .../frame-stats/ios/FrameStatsModule.swift | 39 ++ example/modules/frame-stats/package.json | 10 + example/modules/frame-stats/src/FrameStats.ts | 56 +++ example/scripts/benchmark-table.mjs | 68 +++ example/tsconfig.json | 9 +- 29 files changed, 1930 insertions(+), 5 deletions(-) create mode 100644 docs/benchmarks.md create mode 100644 example/benchmark/BenchmarkApp.tsx create mode 100644 example/benchmark/__tests__/frameStats.test.ts create mode 100644 example/benchmark/__tests__/thresholds.test.ts create mode 100644 example/benchmark/datasets.ts create mode 100644 example/benchmark/frameStats.ts create mode 100644 example/benchmark/jsLagSampler.ts create mode 100644 example/benchmark/runner.ts create mode 100644 example/benchmark/scenarios.ts create mode 100644 example/benchmark/thresholds.ts create mode 100644 example/maestro/benchmark-pan.yaml create mode 100644 example/maestro/benchmark-run-all.yaml create mode 100644 example/modules/frame-stats/android/build.gradle create mode 100644 example/modules/frame-stats/android/src/main/AndroidManifest.xml create mode 100644 example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameRecorder.kt create mode 100644 example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameStatsModule.kt create mode 100644 example/modules/frame-stats/expo-module.config.json create mode 100644 example/modules/frame-stats/index.ts create mode 100644 example/modules/frame-stats/ios/FrameRecorder.swift create mode 100644 example/modules/frame-stats/ios/FrameStats.podspec create mode 100644 example/modules/frame-stats/ios/FrameStatsModule.swift create mode 100644 example/modules/frame-stats/package.json create mode 100644 example/modules/frame-stats/src/FrameStats.ts create mode 100644 example/scripts/benchmark-table.mjs diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..9ae8f1d --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,183 @@ +# Benchmarks + +The example app ships a benchmark harness that measures what the performance +audit could only estimate: main-thread frame intervals, JS-thread stalls and +memory while the map is driven through fixed scenarios. It does not ship in the +library; it lives in `example/benchmark` and the local Expo module +`example/modules/frame-stats`. + +## What is measured + +| Metric | How | Where | +| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | +| Frame intervals | `CADisplayLink` on iOS, `Choreographer.FrameCallback` on Android, both on the main thread. The gap between two callbacks is the frame the user saw; a blocked main thread is one long gap. The interval the display was running at is recorded per frame, so jank is judged against 8.33 ms on a 120 Hz display and against 16.67 ms on a 60 Hz one, and ProMotion rate changes do not count as jank. | `modules/frame-stats` | +| JS-thread lag | A timer re-armed every 16 ms; how late it fires is how long the JS thread was busy, for example serializing a marker array during a commit. | `benchmark/jsLagSampler.ts` | +| Memory | `phys_footprint` on iOS, PSS on Android, before and after each scenario. | `modules/frame-stats` | + +Percentiles use the nearest-rank method. A frame is jank when it is longer than +1.5× the interval the display asked for. Dropped frames are the refresh slots +that passed with nothing drawn. + +## Pass / fail + +Thresholds scale with the display's refresh rate (`budget = 1000 / Hz`): + +| Metric | Limit | +| ------------------------------------------ | ---------------------------------------------------------------- | +| p50, p95 | ≤ budget + 5 % (display-link jitter around the nominal interval) | +| p99 | ≤ 1.5 × budget | +| worst frame | ≤ 3 × budget (25 ms at 120 Hz, 50 ms at 60 Hz) | +| jank frames | ≤ 1 % | +| JS lag p95 (animated-marker scenario only) | ≤ budget | + +They are implemented in `benchmark/thresholds.ts` and unit-tested with +`cd example && bun test`. + +## Scenarios + +| ID | Setup | Script | +| --- | ---------------------------------- | ------------------------------------------------------------------------- | +| A | empty map | 3 s idle, short pan | +| B | 100 markers | pan | +| C | 1,000 markers | pan | +| D | 10,000 markers | pan | +| E | 10,000 markers, clustering on | zoom sweep across five levels, then pan | +| F | 10,000 markers | ten-leg pan | +| G | 10,000 markers | zoom sweep | +| H | 10,000 markers | four heading changes | +| I | 1,000 markers | 100 of them move at 10 Hz for 5 s through prop updates; JS lag is checked | +| K | 5,000-point route and 200 polygons | five style changes, then pan | +| L | 10,000 markers | three pan legs, then 5 s idle | + +Scenario J (live location) is not scripted: it needs location permission and a +GPS feed. Use the simulator's location menu with the manual recorder. + +The scripted scenarios move the camera with `animateCamera`. That exercises the +same native camera path as a gesture on MapKit and Android, but on the iOS +Google provider the live marker refresh during movement only runs for real +gestures, so use the manual recorder or the Maestro flow there. + +## Running + +```bash +EXPO_PUBLIC_BENCHMARK=1 bun example ios --port 8082 +EXPO_PUBLIC_BENCHMARK=1 bun example android --port 8082 +``` + +`EXPO_PUBLIC_BENCHMARK` is inlined at bundle time; the demo app is unchanged +without it. Use a release build and a physical device for numbers you intend to +keep. Simulators and emulators run at 60 Hz with a different GPU and CPU and +only prove that the harness works. + +In the app, "Run all" runs every scenario in order, "Run X" runs the selected +one, "Record" starts a manual recording for real gestures. Each result is +printed as one JSON line: + +```text +[benchmark] {"id":"D-markers-10k","frames":{"p95":8.4,...},...} +``` + +In a debug build the line shows in Metro's terminal. Every build also writes it +to the system log, which is how release builds are harvested: + +```bash +xcrun simctl spawn booted log stream --predicate 'eventMessage contains "[benchmark]"' +adb logcat -s NitroMapsBenchmark +``` + +"Share JSON" exports the whole run through the system share sheet, and +`node example/scripts/benchmark-table.mjs ` turns captured lines into +the Markdown table used below. + +### Maestro + +```bash +maestro test example/maestro/benchmark-run-all.yaml # every scripted scenario +maestro test example/maestro/benchmark-pan.yaml # real-gesture pan on scenario D +``` + +The flow selects scenario D, starts the manual recorder, performs four swipes +and stops. Maestro has no pinch gesture, so zoom runs stay manual. + +### 120 Hz on iPhone + +`CADisplayLink` is capped at 60 Hz on iPhone unless the app opts in, so +`example/app.json` sets `CADisableMinimumFrameDurationOnPhone`. Without it a +ProMotion device reports a 60 Hz budget and hides half the frames. + +## Baselines + +No device numbers are recorded yet. The first accepted run on a 120 Hz iPhone +and a 120 Hz Android device becomes the baseline table here; until then the +audit's estimates stand and every pass/fail line the harness prints is +informational. + +### Harness smoke run (not a device baseline) + +iPhone 17 Pro simulator, iOS 26.5, release build, MapKit provider, 60 Hz, on an +Apple Silicon Mac. Recorded 2026-09-08 with the harness from #64, evaluated with the thresholds above. The point +of this table is that the harness produces the numbers; a simulator says nothing +about a phone's GPU or CPU. The failures it does show are the ones the audit +predicted: p99 climbs to two frames on the clustered zoom sweep and on rotation, +and the worst frame is 80 ms during rotation. + +| Scenario | Result | FPS | p50 | p95 | p99 | Worst | Jank | JS lag p95 | RSS Δ | +| ------------------ | -------- | --- | ------- | ------- | ------- | ----- | ----- | ---------- | ------- | +| A-empty-idle | fail (1) | 59 | 16.7 ms | 16.7 ms | 16.7 ms | 56 ms | 0.9 % | 1.1 ms | +111 MB | +| B-markers-100 | pass | 59 | 16.7 ms | 16.7 ms | 16.7 ms | 47 ms | 0.7 % | 1.1 ms | +66 MB | +| C-markers-1k | pass | 60 | 16.7 ms | 16.7 ms | 16.7 ms | 42 ms | 0.3 % | 1.1 ms | +66 MB | +| D-markers-10k | pass | 59 | 16.7 ms | 16.7 ms | 16.7 ms | 44 ms | 0.7 % | 1.1 ms | +66 MB | +| E-clustered-10k | fail (2) | 59 | 16.7 ms | 16.7 ms | 33.3 ms | 34 ms | 1.7 % | 1.2 ms | +107 MB | +| F-pan-10k | pass | 59 | 16.7 ms | 16.7 ms | 24.6 ms | 42 ms | 1.0 % | 1.2 ms | +70 MB | +| G-zoom-10k | fail (2) | 58 | 16.7 ms | 16.7 ms | 33.4 ms | 35 ms | 3.7 % | 1.2 ms | +83 MB | +| H-rotate-10k | fail (3) | 58 | 16.7 ms | 16.7 ms | 40.0 ms | 80 ms | 2.1 % | 1.2 ms | +74 MB | +| I-animated-markers | pass | 60 | 16.7 ms | 16.7 ms | 16.7 ms | 17 ms | 0.0 % | 1.1 ms | -6 MB | +| K-shapes | pass | 59 | 16.7 ms | 16.7 ms | 19.4 ms | 47 ms | 0.9 % | 1.3 ms | +88 MB | +| L-idle-after-pan | pass | 59 | 16.7 ms | 16.7 ms | 16.7 ms | 45 ms | 0.6 % | 1.1 ms | +41 MB | + +- A-empty-idle: worst frame 55.59 ms > 50.00 ms +- E-clustered-10k: p99 33.33 ms > 25.00 ms; jank 1.73% > 1% +- G-zoom-10k: p99 33.35 ms > 25.00 ms; jank 3.69% > 1% +- H-rotate-10k: p99 40.05 ms > 25.00 ms; worst frame 80.45 ms > 50.00 ms; jank 2.08% > 1% + +### Harness smoke run, Android emulator (not a device baseline) + +Android emulator, API 35, arm64, Google Maps provider, 60 Hz, on the same Mac. +Debug build with the JS bundle served by Metro, so JS-thread numbers include +dev-mode overhead and are not comparable with the iOS table; frame intervals are +measured natively and are unaffected. Recorded 2026-09-08. An emulated GPU +exaggerates the marker add/remove churn the audit described: the worst frames on +the 10k scenarios are the diff applies after each camera move. + +| Scenario | Result | FPS | p50 | p95 | p99 | Worst | Jank | JS lag p95 | RSS Δ | +| ------------------ | -------- | --- | ------- | ------- | -------- | ------ | ------ | ---------- | ------ | +| A-empty-idle | fail (3) | 59 | 16.7 ms | 16.7 ms | 33.3 ms | 67 ms | 1.4 % | 22.7 ms | -21 MB | +| B-markers-100 | pass | 60 | 16.7 ms | 16.7 ms | 16.7 ms | 33 ms | 1.0 % | 26.5 ms | -31 MB | +| C-markers-1k | fail (4) | 47 | 16.7 ms | 50.0 ms | 100.0 ms | 133 ms | 12.1 % | 69.2 ms | -98 MB | +| D-markers-10k | fail (4) | 34 | 16.7 ms | 66.7 ms | 233.3 ms | 850 ms | 12.3 % | 36.7 ms | +38 MB | +| E-clustered-10k | fail (4) | 29 | 16.7 ms | 83.3 ms | 500.0 ms | 717 ms | 10.6 % | 179.8 ms | -83 MB | +| F-pan-10k | fail (3) | 54 | 16.7 ms | 16.7 ms | 66.7 ms | 250 ms | 3.6 % | 75.2 ms | +66 MB | +| G-zoom-10k | fail (4) | 45 | 16.7 ms | 50.0 ms | 133.3 ms | 150 ms | 11.1 % | 96.9 ms | -39 MB | +| H-rotate-10k | fail (3) | 56 | 16.7 ms | 16.7 ms | 33.3 ms | 117 ms | 3.6 % | 28.6 ms | -35 MB | +| I-animated-markers | fail (3) | 59 | 16.7 ms | 16.7 ms | 33.3 ms | 33 ms | 2.0 % | 33.7 ms | -46 MB | +| K-shapes | fail (3) | 59 | 16.7 ms | 16.7 ms | 33.3 ms | 50 ms | 1.3 % | 46.3 ms | -64 MB | +| L-idle-after-pan | fail (4) | 44 | 16.7 ms | 50.0 ms | 166.7 ms | 300 ms | 9.0 % | 125.8 ms | +59 MB | + +- A-empty-idle: p99 33.33 ms > 25.00 ms; worst frame 66.67 ms > 50.00 ms; jank 1.44% > 1% +- C-markers-1k: p95 50.00 ms > budget 17.50 ms; p99 100.00 ms > 25.00 ms; worst frame 133.33 ms > 50.00 ms; jank 12.15% > 1% +- D-markers-10k: p95 66.67 ms > budget 17.50 ms; p99 233.33 ms > 25.00 ms; worst frame 850.00 ms > 50.00 ms; jank 12.30% > 1% +- E-clustered-10k: p95 83.33 ms > budget 17.50 ms; p99 500.00 ms > 25.00 ms; worst frame 716.67 ms > 50.00 ms; jank 10.62% > 1% +- F-pan-10k: p99 66.67 ms > 25.00 ms; worst frame 250.00 ms > 50.00 ms; jank 3.63% > 1% +- G-zoom-10k: p95 50.00 ms > budget 17.50 ms; p99 133.33 ms > 25.00 ms; worst frame 150.00 ms > 50.00 ms; jank 11.11% > 1% +- H-rotate-10k: p99 33.33 ms > 25.00 ms; worst frame 116.67 ms > 50.00 ms; jank 3.65% > 1% +- I-animated-markers: p99 33.33 ms > 25.00 ms; jank 2.01% > 1%; JS lag p95 33.68 ms > budget 17.50 ms +- K-shapes: p99 33.33 ms > 25.00 ms; worst frame 50.00 ms > 50.00 ms; jank 1.33% > 1% +- L-idle-after-pan: p95 50.00 ms > budget 17.50 ms; p99 166.67 ms > 25.00 ms; worst frame 300.00 ms > 50.00 ms; jank 9.01% > 1% + +## Profiling markers + +The library emits `os_signpost` intervals (iOS, subsystem `com.nitromaps`, +category `MarkerPipeline`) and `android.os.Trace` sections (Android, prefix +`NitroMaps.`) around the marker fingerprint, the spatial index build, the +viewport compute and the diff apply. They show up in Instruments' Points of +Interest track and in Perfetto, and cost nothing when no tracer is attached. diff --git a/eslint.config.mjs b/eslint.config.mjs index ffc2974..e8ee0a0 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -23,7 +23,7 @@ export default tseslint.config( eslint.configs.recommended, ...tseslint.configs.recommended, { - files: ['**/*.config.js', '**/app.plugin.js'], + files: ['**/*.config.js', '**/app.plugin.js', 'example/scripts/*.mjs'], languageOptions: { globals: { ...globals.node, diff --git a/example/app.json b/example/app.json index 36d27c1..a8bf074 100644 --- a/example/app.json +++ b/example/app.json @@ -7,7 +7,10 @@ "userInterfaceStyle": "light", "ios": { "supportsTablet": true, - "bundleIdentifier": "com.nitromaps.example" + "bundleIdentifier": "com.nitromaps.example", + "infoPlist": { + "CADisableMinimumFrameDurationOnPhone": true + } }, "android": { "package": "com.nitromaps.example" diff --git a/example/benchmark/BenchmarkApp.tsx b/example/benchmark/BenchmarkApp.tsx new file mode 100644 index 0000000..0c8295c --- /dev/null +++ b/example/benchmark/BenchmarkApp.tsx @@ -0,0 +1,440 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + Platform, + Pressable, + ScrollView, + Share, + StyleSheet, + Text, + View, +} from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { + MapView, + type MapProvider, + type MapViewRef, +} from 'react-native-better-maps'; +import { + displayRefreshRateHz, + memoryFootprintBytes, + startFrameRecording, + stopFrameRecording, +} from '../modules/frame-stats'; +import { computeFrameStats } from './frameStats'; +import { startJsLagSampler, type LagSampler } from './jsLagSampler'; +import { + formatResultLine, + publishResult, + runScenario, + type ScenarioResult, +} from './runner'; +import { + SCENARIOS, + SKIPPED_SCENARIOS, + type BenchmarkMapProps, + type BenchmarkScenario, + type ScenarioContext, +} from './scenarios'; +import { evaluateFrameStats } from './thresholds'; + +type BenchmarkProvider = Extract; + +const PROVIDERS: BenchmarkProvider[] = + Platform.OS === 'ios' ? ['apple', 'google'] : ['google']; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Waits for React to commit pending state and for the next frame to start. */ +function nextCommit(): Promise { + return new Promise((resolve) => { + setTimeout(() => requestAnimationFrame(() => resolve()), 0); + }); +} + +/** + * Benchmark harness screen. Enabled with `EXPO_PUBLIC_BENCHMARK=1`; the demo + * app is untouched otherwise. + * + * "Run all" mounts each scenario in turn, drives it with animated camera moves + * and prop updates, and records main-thread frame intervals, JS-thread lag and + * memory. "Record" is the manual mode for real gestures (Maestro flows use it). + */ +export default function BenchmarkApp() { + const insets = useSafeAreaInsets(); + const mapRef = useRef(null); + const [provider, setProvider] = useState(PROVIDERS[0]); + const [manualActive, setManualActive] = useState(false); + const [scenarioIndex, setScenarioIndex] = useState(0); + const [mapProps, setMapProps] = useState( + SCENARIOS[0].props, + ); + const [mapKey, setMapKey] = useState(0); + const [results, setResults] = useState([]); + const [status, setStatus] = useState('Idle'); + const [running, setRunning] = useState(false); + const [refreshRate, setRefreshRate] = useState(null); + const readyResolver = useRef<(() => void) | null>(null); + const manualRecording = useRef<{ + lag: LagSampler; + beforeBytes: number; + } | null>(null); + + const scenario = SCENARIOS[scenarioIndex]; + + useEffect(() => { + displayRefreshRateHz() + .then(setRefreshRate) + .catch(() => setRefreshRate(null)); + }, []); + + const handleMapReady = useCallback(() => { + readyResolver.current?.(); + readyResolver.current = null; + }, []); + + const mount = useCallback((next: BenchmarkScenario) => { + return new Promise((resolve) => { + const timeout = setTimeout(resolve, 10_000); + readyResolver.current = () => { + clearTimeout(timeout); + resolve(); + }; + setMapProps(next.props); + setMapKey((key) => key + 1); + }); + }, []); + + const context = useMemo( + () => ({ + map: () => mapRef.current, + async setProps(patch) { + setMapProps((current) => ({ ...current, ...patch })); + await nextCommit(); + }, + sleep, + }), + [], + ); + + const appendResult = useCallback((result: ScenarioResult) => { + setResults((current) => [...current, result]); + }, []); + + const runAll = useCallback(async () => { + if (running) { + return; + } + setRunning(true); + setResults([]); + try { + for (let index = 0; index < SCENARIOS.length; index += 1) { + setScenarioIndex(index); + const result = await runScenario(SCENARIOS[index], { + provider, + mount, + context, + onStatus: setStatus, + }); + appendResult(result); + } + setStatus('Done'); + } catch (error) { + setStatus(`Failed: ${String(error)}`); + } finally { + setRunning(false); + } + }, [appendResult, context, mount, provider, running]); + + const runOne = useCallback(async () => { + if (running) { + return; + } + setRunning(true); + try { + appendResult( + await runScenario(scenario, { + provider, + mount, + context, + onStatus: setStatus, + }), + ); + setStatus('Done'); + } catch (error) { + setStatus(`Failed: ${String(error)}`); + } finally { + setRunning(false); + } + }, [appendResult, context, mount, provider, running, scenario]); + + const toggleManualRecording = useCallback(async () => { + const active = manualRecording.current; + if (active == null) { + const beforeBytes = await memoryFootprintBytes(); + manualRecording.current = { lag: startJsLagSampler(), beforeBytes }; + await startFrameRecording(); + setManualActive(true); + setStatus('Recording: gesture now, then tap Stop'); + return; + } + + manualRecording.current = null; + setManualActive(false); + const recording = await stopFrameRecording(); + const jsLag = active.lag.stop(); + const afterBytes = await memoryFootprintBytes(); + const frames = computeFrameStats(recording); + const MB = 1024 * 1024; + const result: ScenarioResult = { + id: `manual-${scenario.id}`, + name: `Manual · ${scenario.name}`, + platform: Platform.OS, + provider, + recordedAt: new Date().toISOString(), + frames, + jsLag, + memory: { + beforeMB: active.beforeBytes / MB, + afterMB: afterBytes / MB, + deltaMB: (afterBytes - active.beforeBytes) / MB, + }, + evaluation: evaluateFrameStats(frames, jsLag), + }; + await publishResult(result); + appendResult(result); + setStatus('Done'); + }, [appendResult, provider, scenario]); + + const selectScenario = useCallback( + (index: number) => { + if (running) { + return; + } + setScenarioIndex(index); + setMapProps(SCENARIOS[index].props); + setMapKey((key) => key + 1); + }, + [running], + ); + + const shareResults = useCallback(() => { + const payload = { + platform: Platform.OS, + provider, + refreshRateHz: refreshRate, + recordedAt: new Date().toISOString(), + results, + }; + Share.share({ message: JSON.stringify(payload, null, 2) }).catch(() => {}); + }, [provider, refreshRate, results]); + + const passed = results.filter((result) => result.evaluation.passed).length; + + // `MapView` props are a discriminated union on `provider`, so each provider + // gets its own element with a literal prop, as the demo app does. + const commonMapProps = { + style: styles.map, + region: mapProps.region, + markers: mapProps.markers, + polylines: mapProps.polylines, + polygons: mapProps.polygons, + clusteringEnabled: mapProps.clusteringEnabled, + markerEnteringAnimation: false as const, + clusterEnteringAnimation: false as const, + onMapReady: handleMapReady, + }; + + return ( + + {provider === 'google' ? ( + + ) : ( + + )} + + + + + Benchmark · {Platform.OS} · {provider} + {refreshRate != null ? ` · ${refreshRate.toFixed(0)} Hz` : ''} + + + {status} + + + + {SCENARIOS.map((item, index) => ( + selectScenario(index)} + style={[ + styles.chip, + index === scenarioIndex && styles.chipActive, + ]} + > + {item.id} + + ))} + + + + + Run all + + + + Run {scenario.id.split('-')[0]} + + + + + {manualActive ? 'Stop' : 'Record'} + + + {PROVIDERS.length > 1 ? ( + + setProvider((current) => + current === 'apple' ? 'google' : 'apple', + ) + } + disabled={running} + style={styles.button} + > + {provider} + + ) : null} + + + + + + + + {results.length > 0 + ? `${passed}/${results.length} passed` + : `${SCENARIOS.length} scenarios · ${SKIPPED_SCENARIOS.length} skipped`} + + + Share JSON + + + + {results.map((result) => ( + + {formatResultLine(result)} + {result.evaluation.failures.length > 0 + ? `\n ${result.evaluation.failures.join('; ')}` + : ''} + + ))} + {results.length === 0 ? ( + {scenario.description} + ) : null} + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#0A0A0B' }, + map: { flex: 1 }, + panel: { position: 'absolute', left: 8, right: 8 }, + panelInner: { + backgroundColor: 'rgba(18, 18, 20, 0.92)', + borderRadius: 12, + padding: 10, + gap: 8, + }, + title: { color: '#FFFFFF', fontSize: 13, fontWeight: '700' }, + status: { color: 'rgba(255,255,255,0.72)', fontSize: 12 }, + row: { flexDirection: 'row', gap: 6, flexWrap: 'wrap' }, + chip: { + paddingHorizontal: 8, + paddingVertical: 5, + borderRadius: 999, + backgroundColor: 'rgba(255,255,255,0.08)', + }, + chipActive: { backgroundColor: 'rgba(59, 130, 246, 0.45)' }, + chipText: { color: '#FFFFFF', fontSize: 11, fontVariant: ['tabular-nums'] }, + button: { + paddingHorizontal: 12, + paddingVertical: 8, + borderRadius: 8, + backgroundColor: 'rgba(59, 130, 246, 0.35)', + }, + buttonRecord: { backgroundColor: 'rgba(255, 59, 48, 0.4)' }, + buttonDisabled: { opacity: 0.4 }, + buttonText: { color: '#FFFFFF', fontSize: 12, fontWeight: '600' }, + results: { + position: 'absolute', + left: 8, + right: 8, + maxHeight: 260, + backgroundColor: 'rgba(18, 18, 20, 0.92)', + borderRadius: 12, + padding: 10, + }, + resultsHeader: { + flexDirection: 'row', + justifyContent: 'space-between', + marginBottom: 6, + }, + resultsTitle: { color: '#FFFFFF', fontSize: 12, fontWeight: '700' }, + link: { paddingHorizontal: 4 }, + linkText: { color: '#7EB5DF', fontSize: 12, fontWeight: '600' }, + resultsList: { maxHeight: 220 }, + resultLine: { + color: 'rgba(255,255,255,0.85)', + fontSize: 11, + lineHeight: 15, + fontVariant: ['tabular-nums'], + fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace', + marginBottom: 4, + }, + resultFail: { color: '#FFB4AE' }, +}); diff --git a/example/benchmark/__tests__/frameStats.test.ts b/example/benchmark/__tests__/frameStats.test.ts new file mode 100644 index 0000000..837f830 --- /dev/null +++ b/example/benchmark/__tests__/frameStats.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from 'bun:test'; +import { computeFrameStats, percentile } from '../frameStats'; + +function recording(intervalsMs: number[], expectedMs = 16.667) { + return { + intervalsMs, + expectedMs: intervalsMs.map(() => expectedMs), + durationMs: intervalsMs.reduce((sum, value) => sum + value, 0), + refreshRateHz: 1000 / expectedMs, + }; +} + +describe('percentile', () => { + test('uses nearest rank', () => { + const sorted = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + expect(percentile(sorted, 50)).toBe(5); + expect(percentile(sorted, 90)).toBe(9); + expect(percentile(sorted, 99)).toBe(10); + }); + + test('returns 0 for an empty sample', () => { + expect(percentile([], 50)).toBe(0); + }); +}); + +describe('computeFrameStats', () => { + test('a steady recording has no jank', () => { + const stats = computeFrameStats(recording(Array(120).fill(16.667))); + expect(stats.frames).toBe(120); + expect(stats.jankFrames).toBe(0); + expect(stats.droppedFrames).toBe(0); + expect(stats.averageFps).toBeCloseTo(60, 0); + expect(stats.p99).toBeCloseTo(16.667, 3); + }); + + test('a long frame counts as jank and as dropped refresh slots', () => { + const intervals = [...Array(99).fill(16.667), 50]; + const stats = computeFrameStats(recording(intervals)); + expect(stats.jankFrames).toBe(1); + expect(stats.jankRatio).toBeCloseTo(0.01, 5); + expect(stats.droppedFrames).toBe(2); + expect(stats.max).toBe(50); + // Nearest rank: over 100 samples p99 is the 99th value, the outlier is only in max. + expect(stats.p99).toBeCloseTo(16.667, 3); + expect(stats.p95).toBeCloseTo(16.667, 3); + }); + + test('p99 catches an outlier in a short recording', () => { + const stats = computeFrameStats(recording([...Array(9).fill(16.667), 50])); + expect(stats.p99).toBe(50); + expect(stats.p95).toBe(50); + expect(stats.p90).toBeCloseTo(16.667, 3); + expect(stats.p50).toBeCloseTo(16.667, 3); + }); + + test('jank is judged against the interval the display ran at', () => { + const stats = computeFrameStats({ + intervalsMs: [8.33, 8.33, 16.67, 33.3], + expectedMs: [8.33, 8.33, 16.67, 16.67], + durationMs: 67, + refreshRateHz: 120, + }); + expect(stats.jankFrames).toBe(1); + expect(stats.droppedFrames).toBe(1); + }); + + test('handles an empty recording', () => { + const stats = computeFrameStats(recording([])); + expect(stats.frames).toBe(0); + expect(stats.averageFps).toBe(0); + expect(stats.expectedMs).toBeCloseTo(16.667, 3); + }); +}); diff --git a/example/benchmark/__tests__/thresholds.test.ts b/example/benchmark/__tests__/thresholds.test.ts new file mode 100644 index 0000000..371066e --- /dev/null +++ b/example/benchmark/__tests__/thresholds.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from 'bun:test'; +import type { FrameStatsSummary } from '../frameStats'; +import { evaluateFrameStats } from '../thresholds'; + +function summary( + overrides: Partial = {}, +): FrameStatsSummary { + return { + frames: 600, + durationMs: 5000, + refreshRateHz: 120, + expectedMs: 8.333, + averageFps: 120, + p50: 8.3, + p90: 8.3, + p95: 8.3, + p99: 8.4, + max: 12, + jankFrames: 0, + jankRatio: 0, + droppedFrames: 0, + ...overrides, + }; +} + +describe('evaluateFrameStats', () => { + test('passes a clean 120 Hz recording', () => { + const result = evaluateFrameStats(summary(), null); + expect(result.passed).toBe(true); + expect(result.budgetMs).toBeCloseTo(8.333, 3); + }); + + test('scales the budget with the refresh rate', () => { + const result = evaluateFrameStats( + summary({ refreshRateHz: 60, p50: 16, p95: 16.5, p99: 20, max: 40 }), + null, + ); + expect(result.passed).toBe(true); + }); + + test('tolerates display-link jitter around the budget', () => { + const result = evaluateFrameStats( + summary({ + refreshRateHz: 60, + expectedMs: 16.667, + p50: 16.7, + p95: 16.9, + p99: 17, + max: 30, + }), + null, + ); + expect(result.passed).toBe(true); + }); + + test('fails on p95 above the budget', () => { + const result = evaluateFrameStats(summary({ p95: 9 }), null); + expect(result.passed).toBe(false); + expect(result.failures[0]).toContain('p95'); + }); + + test('fails on a worst frame above three budgets', () => { + const result = evaluateFrameStats(summary({ max: 26 }), null); + expect(result.failures.some((f) => f.includes('worst frame'))).toBe(true); + }); + + test('fails on more than one percent jank', () => { + const result = evaluateFrameStats( + summary({ jankFrames: 12, jankRatio: 0.02 }), + null, + ); + expect(result.failures.some((f) => f.includes('jank'))).toBe(true); + }); + + test('only checks JS lag when asked', () => { + const lag = { samples: 100, p50: 1, p95: 20, p99: 30, max: 40 }; + expect(evaluateFrameStats(summary(), lag).passed).toBe(true); + expect(evaluateFrameStats(summary(), lag, { jsLag: true }).passed).toBe( + false, + ); + }); + + test('fails an empty recording', () => { + const result = evaluateFrameStats(summary({ frames: 0 }), null); + expect(result.failures).toContain('no frames recorded'); + }); +}); diff --git a/example/benchmark/datasets.ts b/example/benchmark/datasets.ts new file mode 100644 index 0000000..f47db2a --- /dev/null +++ b/example/benchmark/datasets.ts @@ -0,0 +1,105 @@ +import type { + MapViewProps, + MarkerDescriptor, + Region, +} from 'react-native-better-maps'; +import { generatePolandMarkers } from '../examples/advancedFeatures'; + +export type PolylineDescriptor = NonNullable[number]; +export type PolygonDescriptor = NonNullable[number]; + +/** Warsaw at city scale: the 10k dataset has its densest hotspot here. */ +export const WARSAW_REGION: Region = { + latitude: 52.2297, + longitude: 21.0122, + latitudeDelta: 0.12, + longitudeDelta: 0.12, +}; + +/** Whole country: clusters form and re-form as the zoom sweep crosses octaves. */ +export const POLAND_REGION: Region = { + latitude: 51.92, + longitude: 19.13, + latitudeDelta: 6.2, + longitudeDelta: 10.5, +}; + +const markerCache = new Map(); + +/** Deterministic Poland dataset, memoized so scenarios share one array identity. */ +export function markers(count: number): MarkerDescriptor[] { + let cached = markerCache.get(count); + if (cached == null) { + cached = generatePolandMarkers(count); + markerCache.set(count, cached); + } + return cached; +} + +/** Moves `movingCount` markers by a small deterministic step; returns a new array. */ +export function stepMarkers( + current: MarkerDescriptor[], + movingCount: number, + tick: number, +): MarkerDescriptor[] { + const angle = tick * 0.35; + const dLat = Math.sin(angle) * 0.0006; + const dLon = Math.cos(angle) * 0.0009; + return current.map((marker, index) => + index < movingCount + ? { + ...marker, + coordinate: { + latitude: marker.coordinate.latitude + dLat, + longitude: marker.coordinate.longitude + dLon, + }, + } + : marker, + ); +} + +/** A sinuous 5,000-point route from Gdańsk down to Kraków. */ +export function longRoute( + points = 5_000, + strokeColor = '#FF3B30', +): PolylineDescriptor { + const coordinates = []; + const startLat = 54.35; + const endLat = 50.06; + for (let index = 0; index < points; index += 1) { + const t = index / (points - 1); + coordinates.push({ + latitude: startLat + (endLat - startLat) * t, + longitude: + 19.0 + Math.sin(t * Math.PI * 6) * 0.6 + Math.sin(t * 90) * 0.02, + }); + } + return { id: 'route', coordinates, strokeColor, strokeWidth: 4 }; +} + +/** A grid of 200 small squares around Warsaw. */ +export function polygonGrid(count = 200): PolygonDescriptor[] { + const columns = 20; + const size = 0.006; + const gap = 0.009; + const polygons: PolygonDescriptor[] = []; + for (let index = 0; index < count; index += 1) { + const row = Math.floor(index / columns); + const column = index % columns; + const lat = WARSAW_REGION.latitude - 0.05 + row * gap; + const lon = WARSAW_REGION.longitude - 0.09 + column * gap; + polygons.push({ + id: `cell-${index}`, + coordinates: [ + { latitude: lat, longitude: lon }, + { latitude: lat + size, longitude: lon }, + { latitude: lat + size, longitude: lon + size }, + { latitude: lat, longitude: lon + size }, + ], + fillColor: '#007AFF33', + strokeColor: '#007AFF', + strokeWidth: 1, + }); + } + return polygons; +} diff --git a/example/benchmark/frameStats.ts b/example/benchmark/frameStats.ts new file mode 100644 index 0000000..3da6264 --- /dev/null +++ b/example/benchmark/frameStats.ts @@ -0,0 +1,78 @@ +import type { FrameRecording } from '../modules/frame-stats'; + +/** Percentiles and jank counts derived from one recording. */ +export interface FrameStatsSummary { + frames: number; + durationMs: number; + refreshRateHz: number; + /** Median refresh interval the display ran at during the recording. */ + expectedMs: number; + averageFps: number; + p50: number; + p90: number; + p95: number; + p99: number; + max: number; + /** Frames longer than 1.5× the interval the display was running at. */ + jankFrames: number; + jankRatio: number; + /** Refresh slots that passed without a frame, summed over the recording. */ + droppedFrames: number; +} + +/** Nearest-rank percentile over a sorted ascending sample. */ +export function percentile(sortedAscending: number[], p: number): number { + if (sortedAscending.length === 0) { + return 0; + } + const rank = Math.ceil((p / 100) * sortedAscending.length); + const index = Math.min(sortedAscending.length - 1, Math.max(0, rank - 1)); + return sortedAscending[index]; +} + +export const JANK_THRESHOLD_FACTOR = 1.5; + +export function computeFrameStats( + recording: FrameRecording, +): FrameStatsSummary { + const { intervalsMs, expectedMs, refreshRateHz } = recording; + const frames = intervalsMs.length; + const sortedIntervals = [...intervalsMs].sort((a, b) => a - b); + const sortedExpected = [...expectedMs].sort((a, b) => a - b); + const expected = + sortedExpected.length > 0 + ? percentile(sortedExpected, 50) + : 1000 / Math.max(1, refreshRateHz); + + let jankFrames = 0; + let droppedFrames = 0; + let totalMs = 0; + for (let index = 0; index < frames; index += 1) { + const interval = intervalsMs[index]; + const frameExpected = expectedMs[index] ?? expected; + totalMs += interval; + if (interval > frameExpected * JANK_THRESHOLD_FACTOR) { + jankFrames += 1; + } + droppedFrames += Math.max(0, Math.round(interval / frameExpected) - 1); + } + + return { + frames, + durationMs: recording.durationMs, + refreshRateHz, + expectedMs: expected, + averageFps: totalMs > 0 ? (frames * 1000) / totalMs : 0, + p50: percentile(sortedIntervals, 50), + p90: percentile(sortedIntervals, 90), + p95: percentile(sortedIntervals, 95), + p99: percentile(sortedIntervals, 99), + max: + sortedIntervals.length > 0 + ? sortedIntervals[sortedIntervals.length - 1] + : 0, + jankFrames, + jankRatio: frames > 0 ? jankFrames / frames : 0, + droppedFrames, + }; +} diff --git a/example/benchmark/jsLagSampler.ts b/example/benchmark/jsLagSampler.ts new file mode 100644 index 0000000..bf25980 --- /dev/null +++ b/example/benchmark/jsLagSampler.ts @@ -0,0 +1,50 @@ +import { percentile } from './frameStats'; + +/** How late timer callbacks fire on the JS thread, in ms. */ +export interface LagSummary { + samples: number; + p50: number; + p95: number; + p99: number; + max: number; +} + +export interface LagSampler { + stop(): LagSummary; +} + +/** + * Measures JS-thread stalls: a timer is re-armed every `intervalMs`, and the + * amount by which it fires late is the time the JS thread was busy with + * something else, such as serializing a marker array during a commit. + */ +export function startJsLagSampler(intervalMs = 16): LagSampler { + const samples: number[] = []; + let expectedAt = performance.now() + intervalMs; + let timer: ReturnType | null = null; + + const tick = () => { + const now = performance.now(); + samples.push(Math.max(0, now - expectedAt)); + expectedAt = now + intervalMs; + timer = setTimeout(tick, intervalMs); + }; + timer = setTimeout(tick, intervalMs); + + return { + stop() { + if (timer != null) { + clearTimeout(timer); + timer = null; + } + const sorted = [...samples].sort((a, b) => a - b); + return { + samples: sorted.length, + p50: percentile(sorted, 50), + p95: percentile(sorted, 95), + p99: percentile(sorted, 99), + max: sorted.length > 0 ? sorted[sorted.length - 1] : 0, + }; + }, + }; +} diff --git a/example/benchmark/runner.ts b/example/benchmark/runner.ts new file mode 100644 index 0000000..eac78c0 --- /dev/null +++ b/example/benchmark/runner.ts @@ -0,0 +1,108 @@ +import { Platform } from 'react-native'; +import type { MapProvider } from 'react-native-better-maps'; +import { + displayRefreshRateHz, + logBenchmarkLine, + memoryFootprintBytes, + startFrameRecording, + stopFrameRecording, +} from '../modules/frame-stats'; +import { computeFrameStats, type FrameStatsSummary } from './frameStats'; +import { startJsLagSampler, type LagSummary } from './jsLagSampler'; +import type { BenchmarkScenario, ScenarioContext } from './scenarios'; +import { evaluateFrameStats, type Evaluation } from './thresholds'; + +export interface ScenarioResult { + id: string; + name: string; + platform: string; + provider: MapProvider; + recordedAt: string; + frames: FrameStatsSummary; + jsLag: LagSummary; + memory: { beforeMB: number; afterMB: number; deltaMB: number }; + evaluation: Evaluation; +} + +export interface RunOptions { + provider: MapProvider; + /** Mounts the scenario's map and resolves once `onMapReady` fired. */ + mount(scenario: BenchmarkScenario): Promise; + context: ScenarioContext; + onStatus?(message: string): void; +} + +const MB = 1024 * 1024; + +/** Mounts, settles, records, scripts and evaluates one scenario. */ +export async function runScenario( + scenario: BenchmarkScenario, + options: RunOptions, +): Promise { + options.onStatus?.(`${scenario.name}: mounting`); + await options.mount(scenario); + await options.context.sleep(scenario.settleMs ?? 1500); + + const beforeBytes = await memoryFootprintBytes(); + options.onStatus?.(`${scenario.name}: recording`); + const lag = startJsLagSampler(); + await startFrameRecording(); + try { + await scenario.run(options.context); + } catch (error) { + await stopFrameRecording().catch(() => undefined); + lag.stop(); + throw error; + } + + const recording = await stopFrameRecording(); + const jsLag = lag.stop(); + const afterBytes = await memoryFootprintBytes(); + const refreshRateHz = + recording.refreshRateHz || (await displayRefreshRateHz()); + const frames = computeFrameStats({ ...recording, refreshRateHz }); + const evaluation = evaluateFrameStats(frames, jsLag, { + jsLag: scenario.checkJsLag, + }); + const result: ScenarioResult = { + id: scenario.id, + name: scenario.name, + platform: Platform.OS, + provider: options.provider, + recordedAt: new Date().toISOString(), + frames, + jsLag, + memory: { + beforeMB: beforeBytes / MB, + afterMB: afterBytes / MB, + deltaMB: (afterBytes - beforeBytes) / MB, + }, + evaluation, + }; + await publishResult(result); + return result; +} + +/** One JSON line per result: Metro output in debug, the system log always. */ +export async function publishResult(result: ScenarioResult): Promise { + const line = `[benchmark] ${JSON.stringify(result)}`; + console.log(line); + await logBenchmarkLine(line).catch(() => undefined); +} + +/** One line per scenario, for the on-screen table and for log grepping. */ +export function formatResultLine(result: ScenarioResult): string { + const { frames, jsLag, memory, evaluation } = result; + const verdict = evaluation.passed ? 'PASS' : 'FAIL'; + return [ + `${verdict} ${result.id}`, + `fps ${frames.averageFps.toFixed(0)}`, + `p50 ${frames.p50.toFixed(1)}`, + `p95 ${frames.p95.toFixed(1)}`, + `p99 ${frames.p99.toFixed(1)}`, + `max ${frames.max.toFixed(0)}`, + `jank ${(frames.jankRatio * 100).toFixed(1)}%`, + `js p95 ${jsLag.p95.toFixed(1)}`, + `mem ${memory.deltaMB >= 0 ? '+' : ''}${memory.deltaMB.toFixed(0)}MB`, + ].join(' · '); +} diff --git a/example/benchmark/scenarios.ts b/example/benchmark/scenarios.ts new file mode 100644 index 0000000..a0d9446 --- /dev/null +++ b/example/benchmark/scenarios.ts @@ -0,0 +1,245 @@ +import type { + Camera, + MapViewRef, + MarkerDescriptor, + Region, +} from 'react-native-better-maps'; +import { + POLAND_REGION, + WARSAW_REGION, + longRoute, + markers, + polygonGrid, + stepMarkers, + type PolygonDescriptor, + type PolylineDescriptor, +} from './datasets'; + +/** The subset of `MapView` props a scenario controls. */ +export interface BenchmarkMapProps { + region: Region; + markers?: MarkerDescriptor[]; + polylines?: PolylineDescriptor[]; + polygons?: PolygonDescriptor[]; + clusteringEnabled?: boolean; +} + +/** What a scenario script can do while the recorder is running. */ +export interface ScenarioContext { + map(): MapViewRef | null; + /** Applies new props to the mounted map and waits for the commit. */ + setProps(patch: Partial): Promise; + sleep(ms: number): Promise; +} + +export interface BenchmarkScenario { + id: string; + name: string; + description: string; + props: BenchmarkMapProps; + /** Extra settle time after `onMapReady` before recording starts. */ + settleMs?: number; + /** Also fail the scenario when the JS thread cannot keep up with the frame budget. */ + checkJsLag?: boolean; + run(context: ScenarioContext): Promise; +} + +function cameraAt(region: Region, overrides: Partial = {}): Camera { + return { + center: { latitude: region.latitude, longitude: region.longitude }, + zoom: 12, + heading: 0, + pitch: 0, + ...overrides, + }; +} + +async function animate( + context: ScenarioContext, + camera: Camera, + durationMs: number, +): Promise { + await context.map()?.animateCamera(camera, durationMs / 1000); + await context.sleep(durationMs + 120); +} + +/** Pans the camera in a square around `region`, one animated leg at a time. */ +export async function pan( + context: ScenarioContext, + region: Region, + legs = 6, + stepDegrees = 0.03, + legMs = 600, +): Promise { + const offsets = [ + [1, 0], + [1, 1], + [0, 1], + [-1, 1], + [-1, 0], + [-1, -1], + [0, -1], + [1, -1], + ]; + for (let leg = 0; leg < legs; leg += 1) { + const [dy, dx] = offsets[leg % offsets.length]; + await animate( + context, + cameraAt(region, { + center: { + latitude: region.latitude + dy * stepDegrees * (1 + leg * 0.3), + longitude: region.longitude + dx * stepDegrees * (1 + leg * 0.3), + }, + }), + legMs, + ); + } + await animate(context, cameraAt(region), legMs); +} + +/** Zooms through several octaves so clustering re-forms at each level. */ +export async function zoomSweep( + context: ScenarioContext, + region: Region, + zooms = [9, 13, 7, 11, 6], + legMs = 900, +): Promise { + for (const zoom of zooms) { + await animate(context, cameraAt(region, { zoom }), legMs); + } +} + +export async function rotate( + context: ScenarioContext, + region: Region, + headings = [90, 180, 270, 0], + legMs = 700, +): Promise { + for (const heading of headings) { + await animate(context, cameraAt(region, { heading }), legMs); + } +} + +export const SCENARIOS: BenchmarkScenario[] = [ + { + id: 'A-empty-idle', + name: 'A · Empty map', + description: 'No overlays. 3 s idle, then a short pan.', + props: { region: WARSAW_REGION }, + async run(context) { + await context.sleep(3000); + await pan(context, WARSAW_REGION, 3); + }, + }, + { + id: 'B-markers-100', + name: 'B · 100 markers', + description: 'Pan with 100 markers.', + props: { region: WARSAW_REGION, markers: markers(100) }, + run: (context) => pan(context, WARSAW_REGION), + }, + { + id: 'C-markers-1k', + name: 'C · 1,000 markers', + description: 'Pan with 1,000 markers (viewport pipeline, no clustering).', + props: { region: WARSAW_REGION, markers: markers(1_000) }, + run: (context) => pan(context, WARSAW_REGION), + }, + { + id: 'D-markers-10k', + name: 'D · 10,000 markers', + description: 'Pan with 10,000 markers (viewport LOD, no clustering).', + props: { region: WARSAW_REGION, markers: markers(10_000) }, + settleMs: 2500, + run: (context) => pan(context, WARSAW_REGION), + }, + { + id: 'E-clustered-10k', + name: 'E · 10,000 clustered', + description: 'Zoom sweep across octaves, then a pan, with clustering on.', + props: { + region: POLAND_REGION, + markers: markers(10_000), + clusteringEnabled: true, + }, + settleMs: 2500, + async run(context) { + await zoomSweep(context, POLAND_REGION); + await pan(context, POLAND_REGION, 4, 0.4); + }, + }, + { + id: 'F-pan-10k', + name: 'F · Long pan', + description: 'Ten animated pan legs with 10,000 markers.', + props: { region: WARSAW_REGION, markers: markers(10_000) }, + settleMs: 2500, + run: (context) => pan(context, WARSAW_REGION, 10, 0.02, 500), + }, + { + id: 'G-zoom-10k', + name: 'G · Zoom sweep', + description: 'Zoom across five levels with 10,000 markers.', + props: { region: WARSAW_REGION, markers: markers(10_000) }, + settleMs: 2500, + run: (context) => zoomSweep(context, WARSAW_REGION), + }, + { + id: 'H-rotate-10k', + name: 'H · Rotation', + description: 'Four heading changes with 10,000 markers.', + props: { region: WARSAW_REGION, markers: markers(10_000) }, + settleMs: 2500, + run: (context) => rotate(context, WARSAW_REGION), + }, + { + id: 'I-animated-markers', + name: 'I · Animated markers', + description: + '100 of 1,000 markers move at 10 Hz for 5 s through prop updates.', + props: { region: WARSAW_REGION, markers: markers(1_000) }, + checkJsLag: true, + async run(context) { + let current = markers(1_000); + for (let tick = 1; tick <= 50; tick += 1) { + current = stepMarkers(current, 100, tick); + await context.setProps({ markers: current }); + await context.sleep(100); + } + }, + }, + { + id: 'K-shapes', + name: 'K · Polylines and polygons', + description: + 'A 5,000-point route and 200 polygons; five style changes, then a pan.', + props: { + region: WARSAW_REGION, + polylines: [longRoute()], + polygons: polygonGrid(), + }, + async run(context) { + const colors = ['#FF9500', '#34C759', '#AF52DE', '#FF2D55', '#FF3B30']; + for (const color of colors) { + await context.setProps({ polylines: [longRoute(5_000, color)] }); + await context.sleep(400); + } + await pan(context, WARSAW_REGION, 4); + }, + }, + { + id: 'L-idle-after-pan', + name: 'L · Idle after a pan', + description: 'Three pan legs with 10,000 markers, then 5 s of nothing.', + props: { region: WARSAW_REGION, markers: markers(10_000) }, + settleMs: 2500, + async run(context) { + await pan(context, WARSAW_REGION, 3); + await context.sleep(5000); + }, + }, +]; + +export const SKIPPED_SCENARIOS = [ + 'J · Live location: needs location permission and a scripted GPS feed; run manually with the simulator location menu.', +]; diff --git a/example/benchmark/thresholds.ts b/example/benchmark/thresholds.ts new file mode 100644 index 0000000..bda95d1 --- /dev/null +++ b/example/benchmark/thresholds.ts @@ -0,0 +1,69 @@ +import type { FrameStatsSummary } from './frameStats'; +import type { LagSummary } from './jsLagSampler'; + +/** + * Pass/fail rules from the performance audit, section 16, expressed against + * the display's own frame budget so the same rules apply at 60, 90 and 120 Hz. + */ +export interface EvaluationOptions { + /** Also require the JS thread to keep up with the frame budget. */ + jsLag?: boolean; +} + +export interface Evaluation { + passed: boolean; + failures: string[]; + budgetMs: number; +} + +const WORST_FRAME_FACTOR = 3; +const P99_FACTOR = 1.5; +const MAX_JANK_RATIO = 0.01; +/** + * A display link reports 16.67 ms frames with a little jitter either side of + * the nominal interval, so a steady run has p50 and p95 fractionally above the + * budget. Five percent covers that without hiding a real dropped frame. + */ +const PERCENTILE_TOLERANCE = 1.05; + +function ms(value: number): string { + return `${value.toFixed(2)} ms`; +} + +export function evaluateFrameStats( + frames: FrameStatsSummary, + jsLag: LagSummary | null, + options: EvaluationOptions = {}, +): Evaluation { + const budgetMs = 1000 / Math.max(1, frames.refreshRateHz); + const failures: string[] = []; + + if (frames.frames === 0) { + failures.push('no frames recorded'); + } + const percentileLimit = budgetMs * PERCENTILE_TOLERANCE; + if (frames.p50 > percentileLimit) { + failures.push(`p50 ${ms(frames.p50)} > budget ${ms(percentileLimit)}`); + } + if (frames.p95 > percentileLimit) { + failures.push(`p95 ${ms(frames.p95)} > budget ${ms(percentileLimit)}`); + } + if (frames.p99 > budgetMs * P99_FACTOR) { + failures.push(`p99 ${ms(frames.p99)} > ${ms(budgetMs * P99_FACTOR)}`); + } + if (frames.max > budgetMs * WORST_FRAME_FACTOR) { + failures.push( + `worst frame ${ms(frames.max)} > ${ms(budgetMs * WORST_FRAME_FACTOR)}`, + ); + } + if (frames.jankRatio > MAX_JANK_RATIO) { + failures.push(`jank ${(frames.jankRatio * 100).toFixed(2)}% > 1%`); + } + if (options.jsLag && jsLag != null && jsLag.p95 > percentileLimit) { + failures.push( + `JS lag p95 ${ms(jsLag.p95)} > budget ${ms(percentileLimit)}`, + ); + } + + return { passed: failures.length === 0, failures, budgetMs }; +} diff --git a/example/examples/advancedFeatures.ts b/example/examples/advancedFeatures.ts index 579f271..40197b7 100644 --- a/example/examples/advancedFeatures.ts +++ b/example/examples/advancedFeatures.ts @@ -51,7 +51,7 @@ function clamp(value: number, min: number, max: number): number { * by population) plus a sparse rural scatter. This makes clustering look * organic instead of a rigid lattice. */ -function generatePolandMarkers( +export function generatePolandMarkers( count: number, ): NonNullable { const rng = mulberry32(0x5eed); diff --git a/example/index.js b/example/index.js index 1420b2f..d235d69 100644 --- a/example/index.js +++ b/example/index.js @@ -1,7 +1,12 @@ import 'react-native-reanimated'; import { registerRootComponent } from 'expo'; import { SafeAreaProvider } from 'react-native-safe-area-context'; -import App from './App'; +// `EXPO_PUBLIC_*` variables are inlined at bundle time, so the demo bundle +// never includes the harness unless it was built with the flag set. +const App = + process.env.EXPO_PUBLIC_BENCHMARK === '1' + ? require('./benchmark/BenchmarkApp').default + : require('./App').default; registerRootComponent(function Root() { return ( diff --git a/example/maestro/benchmark-pan.yaml b/example/maestro/benchmark-pan.yaml new file mode 100644 index 0000000..e8d58dc --- /dev/null +++ b/example/maestro/benchmark-pan.yaml @@ -0,0 +1,34 @@ +# Real-gesture pan on the 10k-marker scenario. +# +# Build and launch the example with EXPO_PUBLIC_BENCHMARK=1 first, then: +# maestro test example/maestro/benchmark-pan.yaml +# The result line is printed by the app as `[benchmark] {...}` in Metro's output. +appId: com.nitromaps.example +--- +- launchApp +- tapOn: + id: 'benchmark-scenario-D-markers-10k' +- waitForAnimationToEnd: + timeout: 5000 +- tapOn: + id: 'benchmark-record-toggle' +- swipe: + start: 80%, 55% + end: 20%, 55% + duration: 800 +- swipe: + start: 20%, 55% + end: 80%, 55% + duration: 800 +- swipe: + start: 50%, 70% + end: 50%, 30% + duration: 800 +- swipe: + start: 50%, 30% + end: 50%, 70% + duration: 800 +- tapOn: + id: 'benchmark-record-toggle' +- assertVisible: + id: 'benchmark-result-manual-D-markers-10k' diff --git a/example/maestro/benchmark-run-all.yaml b/example/maestro/benchmark-run-all.yaml new file mode 100644 index 0000000..3671d9e --- /dev/null +++ b/example/maestro/benchmark-run-all.yaml @@ -0,0 +1,18 @@ +# Runs every scripted scenario and waits for the last result line. +# +# Build and launch the example with EXPO_PUBLIC_BENCHMARK=1 first, then: +# maestro test example/maestro/benchmark-run-all.yaml +# Results are printed by the app as `[benchmark] {...}` lines (see docs/benchmarks.md). +appId: com.nitromaps.example +--- +- launchApp +- extendedWaitUntil: + visible: + id: 'benchmark-run-all' + timeout: 60000 +- tapOn: + id: 'benchmark-run-all' +- extendedWaitUntil: + visible: + id: 'benchmark-result-L-idle-after-pan' + timeout: 300000 diff --git a/example/modules/frame-stats/android/build.gradle b/example/modules/frame-stats/android/build.gradle new file mode 100644 index 0000000..2bff6e3 --- /dev/null +++ b/example/modules/frame-stats/android/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'com.android.library' + id 'expo-module-gradle-plugin' +} + +group = 'expo.modules.framestats' +version = '0.1.0' + +android { + namespace "expo.modules.framestats" + defaultConfig { + versionCode 1 + versionName "0.1.0" + } +} diff --git a/example/modules/frame-stats/android/src/main/AndroidManifest.xml b/example/modules/frame-stats/android/src/main/AndroidManifest.xml new file mode 100644 index 0000000..94cbbcf --- /dev/null +++ b/example/modules/frame-stats/android/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameRecorder.kt b/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameRecorder.kt new file mode 100644 index 0000000..d3a356b --- /dev/null +++ b/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameRecorder.kt @@ -0,0 +1,61 @@ +package expo.modules.framestats + +import android.view.Choreographer +import android.view.Display + +/** + * Records main-thread frame intervals with a [Choreographer] callback. + * + * The callback runs once per vsync while the main thread is free, so the gap + * between two callbacks is the frame interval the user experienced: a blocked + * main thread shows up as one long interval. The display's refresh interval + * is captured per frame because adaptive displays change rate on their own. + */ +internal class FrameRecorder( + private val displayProvider: () -> Display?, +) : Choreographer.FrameCallback { + private val intervalsMs = ArrayList(4096) + private val expectedMs = ArrayList(4096) + private var lastFrameNanos = 0L + private var startedAtNanos = 0L + private var running = false + + fun start() { + running = true + startedAtNanos = System.nanoTime() + lastFrameNanos = 0L + Choreographer.getInstance().postFrameCallback(this) + } + + override fun doFrame(frameTimeNanos: Long) { + if (!running) { + return + } + if (lastFrameNanos != 0L) { + intervalsMs.add((frameTimeNanos - lastFrameNanos) / 1_000_000.0) + expectedMs.add(1000.0 / (displayProvider()?.refreshRate ?: 60f)) + } + lastFrameNanos = frameTimeNanos + Choreographer.getInstance().postFrameCallback(this) + } + + fun stop(): Map { + running = false + Choreographer.getInstance().removeFrameCallback(this) + return mapOf( + "intervalsMs" to intervalsMs.toDoubleArray(), + "expectedMs" to expectedMs.toDoubleArray(), + "durationMs" to (System.nanoTime() - startedAtNanos) / 1_000_000.0, + "refreshRateHz" to (displayProvider()?.refreshRate ?: 60f).toDouble(), + ) + } + + companion object { + fun emptyRecording(display: Display?): Map = mapOf( + "intervalsMs" to DoubleArray(0), + "expectedMs" to DoubleArray(0), + "durationMs" to 0.0, + "refreshRateHz" to (display?.refreshRate ?: 60f).toDouble(), + ) + } +} diff --git a/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameStatsModule.kt b/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameStatsModule.kt new file mode 100644 index 0000000..657b169 --- /dev/null +++ b/example/modules/frame-stats/android/src/main/java/expo/modules/framestats/FrameStatsModule.kt @@ -0,0 +1,56 @@ +package expo.modules.framestats + +import android.os.Build +import android.os.Debug +import android.util.Log +import android.view.Display +import expo.modules.kotlin.functions.Queues +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition + +/** + * Exposes [FrameRecorder] to JS. Recording starts and stops on the main thread + * because that is the thread whose frame intervals are measured. + */ +class FrameStatsModule : Module() { + private var recorder: FrameRecorder? = null + + override fun definition() = ModuleDefinition { + Name("FrameStats") + + AsyncFunction("start") { + recorder?.stop() + recorder = FrameRecorder(::currentDisplay).also { it.start() } + }.runOnQueue(Queues.MAIN) + + AsyncFunction("stop") { + val active = recorder + recorder = null + active?.stop() ?: FrameRecorder.emptyRecording(currentDisplay()) + }.runOnQueue(Queues.MAIN) + + AsyncFunction("memoryFootprint") { + // Proportional set size in bytes: the closest Android equivalent of + // iOS `phys_footprint`. + Debug.getPss().toDouble() * 1024.0 + } + + AsyncFunction("displayRefreshRate") { + (currentDisplay()?.refreshRate ?: 60f).toDouble() + }.runOnQueue(Queues.MAIN) + + AsyncFunction("logLine") { line: String -> + Log.i("NitroMapsBenchmark", line) + } + } + + private fun currentDisplay(): Display? { + val activity = appContext.currentActivity ?: return null + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + activity.display + } else { + @Suppress("DEPRECATION") + activity.windowManager.defaultDisplay + } + } +} diff --git a/example/modules/frame-stats/expo-module.config.json b/example/modules/frame-stats/expo-module.config.json new file mode 100644 index 0000000..af0ea1a --- /dev/null +++ b/example/modules/frame-stats/expo-module.config.json @@ -0,0 +1,9 @@ +{ + "platforms": ["apple", "android"], + "apple": { + "modules": ["FrameStatsModule"] + }, + "android": { + "modules": ["expo.modules.framestats.FrameStatsModule"] + } +} diff --git a/example/modules/frame-stats/index.ts b/example/modules/frame-stats/index.ts new file mode 100644 index 0000000..0f049eb --- /dev/null +++ b/example/modules/frame-stats/index.ts @@ -0,0 +1 @@ +export * from './src/FrameStats'; diff --git a/example/modules/frame-stats/ios/FrameRecorder.swift b/example/modules/frame-stats/ios/FrameRecorder.swift new file mode 100644 index 0000000..7c61060 --- /dev/null +++ b/example/modules/frame-stats/ios/FrameRecorder.swift @@ -0,0 +1,79 @@ +import QuartzCore +import UIKit + +/// Records main-thread frame intervals with a `CADisplayLink`. +/// +/// The link fires once per display refresh while the main thread is free, so +/// the gap between two callbacks is the frame interval the user experienced: +/// a blocked main thread shows up as one long interval. The interval the +/// display was running at is captured per frame, because ProMotion displays +/// change refresh rate on their own. +final class FrameRecorder { + private var displayLink: CADisplayLink? + private var lastTimestamp: CFTimeInterval = 0 + private var startedAt: CFTimeInterval = 0 + private var intervalsMs: [Double] = [] + private var expectedMs: [Double] = [] + + func start() { + intervalsMs.reserveCapacity(4096) + expectedMs.reserveCapacity(4096) + + let link = CADisplayLink(target: self, selector: #selector(step(_:))) + let maximum = Float(UIScreen.main.maximumFramesPerSecond) + // Ask for the display's full rate so a 120 Hz device is measured at 120 Hz. + // On iPhone this also needs `CADisableMinimumFrameDurationOnPhone` in + // Info.plist, which the example app sets. + link.preferredFrameRateRange = CAFrameRateRange( + minimum: 30, + maximum: maximum, + preferred: maximum + ) + link.add(to: .main, forMode: .common) + displayLink = link + startedAt = CACurrentMediaTime() + lastTimestamp = 0 + } + + @objc private func step(_ link: CADisplayLink) { + if lastTimestamp > 0 { + intervalsMs.append((link.timestamp - lastTimestamp) * 1000) + expectedMs.append((link.targetTimestamp - link.timestamp) * 1000) + } + lastTimestamp = link.timestamp + } + + func stop() -> [String: Any] { + displayLink?.invalidate() + displayLink = nil + return [ + "intervalsMs": intervalsMs, + "expectedMs": expectedMs, + "durationMs": (CACurrentMediaTime() - startedAt) * 1000, + "refreshRateHz": Double(UIScreen.main.maximumFramesPerSecond), + ] + } + + static func emptyRecording() -> [String: Any] { + [ + "intervalsMs": [Double](), + "expectedMs": [Double](), + "durationMs": 0.0, + "refreshRateHz": Double(UIScreen.main.maximumFramesPerSecond), + ] + } + + /// `phys_footprint`: the number Xcode's memory gauge and jetsam use. + static func memoryFootprintBytes() -> UInt64 { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size + ) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return result == KERN_SUCCESS ? info.phys_footprint : 0 + } +} diff --git a/example/modules/frame-stats/ios/FrameStats.podspec b/example/modules/frame-stats/ios/FrameStats.podspec new file mode 100644 index 0000000..e3d85f3 --- /dev/null +++ b/example/modules/frame-stats/ios/FrameStats.podspec @@ -0,0 +1,25 @@ +require 'json' + +package = JSON.parse(File.read(File.join(__dir__, '..', 'package.json'))) + +Pod::Spec.new do |s| + s.name = 'FrameStats' + s.version = package['version'] + s.summary = package['description'] + s.description = package['description'] + s.license = package['license'] + s.author = package['author'] + s.homepage = package['homepage'] + s.platforms = { :ios => '16.0' } + s.swift_version = '5.9' + s.source = { git: package['homepage'] } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + + s.source_files = "**/*.{h,m,swift}" + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'SWIFT_COMPILATION_MODE' => 'wholemodule' + } +end diff --git a/example/modules/frame-stats/ios/FrameStatsModule.swift b/example/modules/frame-stats/ios/FrameStatsModule.swift new file mode 100644 index 0000000..e11c1ee --- /dev/null +++ b/example/modules/frame-stats/ios/FrameStatsModule.swift @@ -0,0 +1,39 @@ +import ExpoModulesCore +import UIKit + +/// Exposes `FrameRecorder` to JS. Recording starts and stops on the main +/// thread because that is the thread whose frame intervals are measured. +public final class FrameStatsModule: Module { + private var recorder: FrameRecorder? + + public func definition() -> ModuleDefinition { + Name("FrameStats") + + AsyncFunction("start") { () -> Void in + self.recorder?.stop() + let recorder = FrameRecorder() + recorder.start() + self.recorder = recorder + }.runOnQueue(.main) + + AsyncFunction("stop") { () -> [String: Any] in + guard let recorder = self.recorder else { + return FrameRecorder.emptyRecording() + } + self.recorder = nil + return recorder.stop() + }.runOnQueue(.main) + + AsyncFunction("memoryFootprint") { () -> Double in + Double(FrameRecorder.memoryFootprintBytes()) + } + + AsyncFunction("displayRefreshRate") { () -> Double in + Double(UIScreen.main.maximumFramesPerSecond) + }.runOnQueue(.main) + + AsyncFunction("logLine") { (line: String) -> Void in + NSLog("%@", line) + } + } +} diff --git a/example/modules/frame-stats/package.json b/example/modules/frame-stats/package.json new file mode 100644 index 0000000..eadbdd5 --- /dev/null +++ b/example/modules/frame-stats/package.json @@ -0,0 +1,10 @@ +{ + "name": "frame-stats", + "version": "0.1.0", + "private": true, + "description": "Frame-interval recorder for the react-native-better-maps benchmark harness", + "main": "index.ts", + "license": "MIT", + "author": "gmi.software", + "homepage": "https://github.com/gmi-software/react-native-better-maps" +} diff --git a/example/modules/frame-stats/src/FrameStats.ts b/example/modules/frame-stats/src/FrameStats.ts new file mode 100644 index 0000000..4295ed3 --- /dev/null +++ b/example/modules/frame-stats/src/FrameStats.ts @@ -0,0 +1,56 @@ +import { requireNativeModule } from 'expo'; + +/** + * Raw output of one recording session. + * + * `intervalsMs[i]` is the time between display refresh callbacks i and i+1 on + * the main thread; `expectedMs[i]` is the refresh interval the display was + * running at for that frame. Both are needed: ProMotion and adaptive Android + * displays change rate on their own, so jank is "longer than the interval the + * display asked for", not "longer than 16.67 ms". + */ +export interface FrameRecording { + intervalsMs: number[]; + expectedMs: number[]; + durationMs: number; + refreshRateHz: number; +} + +interface NativeFrameStats { + start(): Promise; + stop(): Promise; + memoryFootprint(): Promise; + displayRefreshRate(): Promise; + logLine(line: string): Promise; +} + +const native = requireNativeModule('FrameStats'); + +/** Starts recording main-thread frame intervals. Stops any recording in progress. */ +export function startFrameRecording(): Promise { + return native.start(); +} + +/** Stops recording and returns every frame interval seen since `start`. */ +export function stopFrameRecording(): Promise { + return native.stop(); +} + +/** Resident memory of the process in bytes (`phys_footprint` on iOS, PSS on Android). */ +export function memoryFootprintBytes(): Promise { + return native.memoryFootprint(); +} + +/** Maximum refresh rate of the main display, in Hz. */ +export function displayRefreshRateHz(): Promise { + return native.displayRefreshRate(); +} + +/** + * Writes a line to the system log (`NSLog` on iOS, `Log.i` tag + * `NitroMapsBenchmark` on Android), which release builds keep while + * `console.log` output is dropped. + */ +export function logBenchmarkLine(line: string): Promise { + return native.logLine(line); +} diff --git a/example/scripts/benchmark-table.mjs b/example/scripts/benchmark-table.mjs new file mode 100644 index 0000000..7e46940 --- /dev/null +++ b/example/scripts/benchmark-table.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node +// Turns `[benchmark] {...}` lines from Metro, `simctl log stream` or `adb logcat` +// into a Markdown table. Usage: node example/scripts/benchmark-table.mjs ... +import { readFileSync } from 'node:fs'; + +const MARKER = '[benchmark] '; +const files = process.argv.slice(2); +if (files.length === 0) { + console.error('usage: benchmark-table.mjs ...'); + process.exit(1); +} + +const results = []; +for (const file of files) { + for (const line of readFileSync(file, 'utf8').split('\n')) { + const start = line.indexOf(MARKER); + if (start < 0) { + continue; + } + try { + results.push(JSON.parse(line.slice(start + MARKER.length))); + } catch { + // A truncated line from a log buffer; skip it. + } + } +} + +if (results.length === 0) { + console.error('no [benchmark] lines found'); + process.exit(1); +} + +const first = results[0]; +const budget = 1000 / first.frames.refreshRateHz; +console.log( + `Platform: ${first.platform} · provider: ${first.provider} · ${first.frames.refreshRateHz.toFixed(0)} Hz (budget ${budget.toFixed(2)} ms) · recorded ${first.recordedAt.slice(0, 10)}`, +); +console.log(''); +console.log( + '| Scenario | Result | FPS | p50 | p95 | p99 | Worst | Jank | JS lag p95 | RSS Δ |', +); +console.log('| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |'); +for (const result of results) { + const { frames, jsLag, memory, evaluation } = result; + console.log( + [ + result.id, + evaluation.passed ? 'pass' : `fail (${evaluation.failures.length})`, + frames.averageFps.toFixed(0), + `${frames.p50.toFixed(1)} ms`, + `${frames.p95.toFixed(1)} ms`, + `${frames.p99.toFixed(1)} ms`, + `${frames.max.toFixed(0)} ms`, + `${(frames.jankRatio * 100).toFixed(1)} %`, + `${jsLag.p95.toFixed(1)} ms`, + `${memory.deltaMB >= 0 ? '+' : ''}${memory.deltaMB.toFixed(0)} MB`, + ] + .map((cell) => `| ${cell} `) + .join('') + '|', + ); +} +const failed = results.filter((result) => !result.evaluation.passed); +if (failed.length > 0) { + console.log(''); + for (const result of failed) { + console.log(`- ${result.id}: ${result.evaluation.failures.join('; ')}`); + } +} diff --git a/example/tsconfig.json b/example/tsconfig.json index c7b95c2..136eace 100644 --- a/example/tsconfig.json +++ b/example/tsconfig.json @@ -3,5 +3,12 @@ "compilerOptions": { "strict": true }, - "include": ["**/*.ts", "**/*.tsx"] + "include": [ + "**/*.ts", + "**/*.tsx" + ], + "exclude": [ + "node_modules", + "**/__tests__/**" + ] } From edf659c3c89e37f20ea9ccd699ef02abce897e0c Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 13:37:29 +0200 Subject: [PATCH 2/6] perf: add signpost and systrace markers around the marker pipeline os_signpost intervals on iOS (subsystem com.nitromaps, category MarkerPipeline) and android.os.Trace sections on Android (prefix NitroMaps.) around the marker fingerprint, the spatial index build, the viewport compute and the diff apply, so Instruments and Perfetto show where the pipeline spends its time. No-ops without a tracer attached. --- .../nitro/nitromaps/MapOverlayController.kt | 14 ++++++++------ .../com/margelo/nitro/nitromaps/MapTrace.kt | 16 ++++++++++++++++ package/ios/GoogleMapOverlayController.swift | 3 +++ package/ios/MapOverlayController.swift | 3 +++ package/ios/MapTrace.swift | 18 ++++++++++++++++++ package/ios/MarkerClusterEngine.swift | 6 ++++++ 6 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 package/android/src/main/java/com/margelo/nitro/nitromaps/MapTrace.kt create mode 100644 package/ios/MapTrace.swift diff --git a/package/android/src/main/java/com/margelo/nitro/nitromaps/MapOverlayController.kt b/package/android/src/main/java/com/margelo/nitro/nitromaps/MapOverlayController.kt index 08cfb67..e4270a1 100644 --- a/package/android/src/main/java/com/margelo/nitro/nitromaps/MapOverlayController.kt +++ b/package/android/src/main/java/com/margelo/nitro/nitromaps/MapOverlayController.kt @@ -130,7 +130,7 @@ class MapOverlayController( fun setMarkers(descriptors: Array?) { val next = descriptors ?: emptyArray() - val fingerprint = next.markersFingerprint() + val fingerprint = traceSection("NitroMaps.markersFingerprint") { next.markersFingerprint() } if (fingerprint == markersFingerprint) { return } @@ -216,7 +216,7 @@ class MapOverlayController( return@execute } - val index = MarkerSpatialIndex(descriptors) + val index = traceSection("NitroMaps.buildSpatialIndex") { MarkerSpatialIndex(descriptors) } mainHandler.post { if (builtForDataset != datasetGeneration) { return@post @@ -227,7 +227,9 @@ class MapOverlayController( } } - private fun computeViewportDiff(request: ViewportRefreshRequest): MarkerRenderDiff { + private fun computeViewportDiff( + request: ViewportRefreshRequest, + ): MarkerRenderDiff = traceSection("NitroMaps.computeViewportDiff") { val candidates = request.index.candidates(request.bounds) val elements: List = if (request.clustering) { MarkerClusterEngine.clusters( @@ -242,7 +244,7 @@ class MapOverlayController( .map { ClusterElement.Single(it) } } - return computeMarkerRenderDiff(elements, request.displayedVersions) + computeMarkerRenderDiff(elements, request.displayedVersions) } private fun advanceDatasetGeneration() { @@ -254,8 +256,8 @@ class MapOverlayController( diff: MarkerRenderDiff, animateEntering: Boolean = true, maxAnimatedMarkers: Int = MAX_ANIMATED_MARKERS_PER_DIFF, - ) { - val map = googleMap ?: return + ) = traceSection("NitroMaps.applyMarkerDiff") { + val map = googleMap ?: return@traceSection for (key in diff.removedKeys) { cancelEnteringAnimation(key) diff --git a/package/android/src/main/java/com/margelo/nitro/nitromaps/MapTrace.kt b/package/android/src/main/java/com/margelo/nitro/nitromaps/MapTrace.kt new file mode 100644 index 0000000..373fc44 --- /dev/null +++ b/package/android/src/main/java/com/margelo/nitro/nitromaps/MapTrace.kt @@ -0,0 +1,16 @@ +package com.margelo.nitro.nitromaps + +import android.os.Trace + +/** + * Wraps [block] in a systrace section so the marker pipeline shows up in + * Perfetto and Android Studio's system trace. A no-op when tracing is off. + */ +internal inline fun traceSection(name: String, block: () -> T): T { + Trace.beginSection(name) + try { + return block() + } finally { + Trace.endSection() + } +} diff --git a/package/ios/GoogleMapOverlayController.swift b/package/ios/GoogleMapOverlayController.swift index dd0bea8..f3000c8 100644 --- a/package/ios/GoogleMapOverlayController.swift +++ b/package/ios/GoogleMapOverlayController.swift @@ -222,6 +222,9 @@ final class GoogleMapOverlayController { return } + let signpost = MapTrace.begin("applyMarkerDiff") + defer { MapTrace.end("applyMarkerDiff", signpost) } + for key in diff.removedKeys { markers.removeValue(forKey: key)?.map = nil markerVersions.removeValue(forKey: key) diff --git a/package/ios/MapOverlayController.swift b/package/ios/MapOverlayController.swift index c3929e0..c9dd83d 100644 --- a/package/ios/MapOverlayController.swift +++ b/package/ios/MapOverlayController.swift @@ -119,6 +119,9 @@ final class MapOverlayController { return } + let signpost = MapTrace.begin("applyMarkerDiff") + defer { MapTrace.end("applyMarkerDiff", signpost) } + if !diff.removedKeys.isEmpty { let removed = diff.removedKeys.compactMap { key in displayedAnnotationVersions.removeValue(forKey: key) diff --git a/package/ios/MapTrace.swift b/package/ios/MapTrace.swift new file mode 100644 index 0000000..2d87393 --- /dev/null +++ b/package/ios/MapTrace.swift @@ -0,0 +1,18 @@ +import os.signpost + +/// Signpost intervals for the marker pipeline. Visible in Instruments under +/// Points of Interest (subsystem `com.nitromaps`, category `MarkerPipeline`); +/// a no-op when no tracer is attached. +enum MapTrace { + private static let log = OSLog(subsystem: "com.nitromaps", category: "MarkerPipeline") + + static func begin(_ name: StaticString) -> OSSignpostID { + let id = OSSignpostID(log: log) + os_signpost(.begin, log: log, name: name, signpostID: id) + return id + } + + static func end(_ name: StaticString, _ id: OSSignpostID) { + os_signpost(.end, log: log, name: name, signpostID: id) + } +} diff --git a/package/ios/MarkerClusterEngine.swift b/package/ios/MarkerClusterEngine.swift index c9ead35..b585f1e 100644 --- a/package/ios/MarkerClusterEngine.swift +++ b/package/ios/MarkerClusterEngine.swift @@ -489,7 +489,9 @@ final class MarkerRenderPipeline { func setMarkers(_ descriptors: [MarkerDescriptor]?) -> Bool { let next = descriptors ?? [] + let signpost = MapTrace.begin("markersFingerprint") let fingerprint = next.markersFingerprint() + MapTrace.end("markersFingerprint", signpost) guard fingerprint != markersFingerprint else { return false } @@ -629,7 +631,9 @@ final class MarkerRenderPipeline { return } + let signpost = MapTrace.begin("buildSpatialIndex") let index = MarkerSpatialIndex(markers: descriptors) + MapTrace.end("buildSpatialIndex", signpost) DispatchQueue.main.async { [weak self] in guard let self, builtForDataset == self.datasetGeneration else { return @@ -653,6 +657,8 @@ final class MarkerRenderPipeline { _ request: ViewportRefreshRequest, clusterCellPoints: Double ) -> MarkerRenderDiff { + let signpost = MapTrace.begin("computeViewportDiff") + defer { MapTrace.end("computeViewportDiff", signpost) } let parameters = request.parameters let candidates = request.index.candidates(in: parameters.region) let elements: [MarkerClusterEngine.Element] From 1733420c5cd0644d073e0007ac549249914017f7 Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 13:37:30 +0200 Subject: [PATCH 3/6] fix(android): keep React Native codegen out of the package node_modules Release builds failed with "Type com.facebook.fbreact.specs.NativeAccessibilityInfoSpec is defined multiple times". The library applies com.facebook.react, whose codegen root defaults to the package directory; with an isolated installer (bun, pnpm) that directory contains node_modules/react-native, so the plugin generated React Native's own core specs into this library and they collided with react-android when the release dex was merged. Debug builds hide it because project and library dex files are merged separately. Point jsRootDir at src, which holds no React Native codegen specs; nitrogen generates this library's bindings. --- package/android/build.gradle | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/package/android/build.gradle b/package/android/build.gradle index 646a4b0..7a2038f 100644 --- a/package/android/build.gradle +++ b/package/android/build.gradle @@ -29,6 +29,16 @@ apply plugin: 'org.jetbrains.kotlin.android' apply from: '../nitrogen/generated/android/NitroMaps+autolinking.gradle' apply plugin: 'com.facebook.react' +react { + // Nitrogen generates this library's bindings; React Native's codegen has + // nothing to generate here. Its default root is the package directory, and + // with an isolated installer (bun, pnpm) that directory contains + // `node_modules/react-native`, so the plugin generated React Native's own + // core specs into this library and release dex merging then failed with + // "Type com.facebook.fbreact.specs.* is defined multiple times". + jsRootDir = file("$projectDir/../src") +} + android { namespace 'com.margelo.nitro.nitromaps' From cf0a985facb83ed4458e92a662a544ab76efad02 Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 13:38:12 +0200 Subject: [PATCH 4/6] fix(example): wait for the benchmark summary in the Maestro flow The last result row can sit below the fold of the results list, so waiting for it times out; the summary line shows "/11 passed" once every scenario has a result. --- example/maestro/benchmark-run-all.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/example/maestro/benchmark-run-all.yaml b/example/maestro/benchmark-run-all.yaml index 3671d9e..547335f 100644 --- a/example/maestro/benchmark-run-all.yaml +++ b/example/maestro/benchmark-run-all.yaml @@ -1,4 +1,4 @@ -# Runs every scripted scenario and waits for the last result line. +# Runs every scripted scenario and waits for the summary line. # # Build and launch the example with EXPO_PUBLIC_BENCHMARK=1 first, then: # maestro test example/maestro/benchmark-run-all.yaml @@ -12,7 +12,9 @@ appId: com.nitromaps.example timeout: 60000 - tapOn: id: 'benchmark-run-all' +# The summary reads "/11 passed" once every scenario has a result; the +# last result row can sit below the fold of the results list. - extendedWaitUntil: visible: - id: 'benchmark-result-L-idle-after-pan' + text: '.*/11 passed' timeout: 300000 From 1fa88f500110e27ae01549c6eeffb5efb203e238 Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 14:21:25 +0200 Subject: [PATCH 5/6] docs(example): describe the benchmark thresholds without the audit reference --- example/benchmark/thresholds.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/benchmark/thresholds.ts b/example/benchmark/thresholds.ts index bda95d1..cfdddb8 100644 --- a/example/benchmark/thresholds.ts +++ b/example/benchmark/thresholds.ts @@ -2,7 +2,7 @@ import type { FrameStatsSummary } from './frameStats'; import type { LagSummary } from './jsLagSampler'; /** - * Pass/fail rules from the performance audit, section 16, expressed against + * Pass/fail rules for the benchmark scenarios, expressed against * the display's own frame budget so the same rules apply at 60, 90 and 120 Hz. */ export interface EvaluationOptions { From d0ea27306156c2a7fb289e730f5de76ae509d056 Mon Sep 17 00:00:00 2001 From: Jakub Kasprzyk Date: Tue, 8 Sep 2026 14:28:31 +0200 Subject: [PATCH 6/6] docs: drop the pull request number from the benchmark baselines note --- docs/benchmarks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9ae8f1d..96071e1 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -115,7 +115,7 @@ informational. ### Harness smoke run (not a device baseline) iPhone 17 Pro simulator, iOS 26.5, release build, MapKit provider, 60 Hz, on an -Apple Silicon Mac. Recorded 2026-09-08 with the harness from #64, evaluated with the thresholds above. The point +Apple Silicon Mac. Recorded 2026-09-08 with this harness, evaluated with the thresholds above. The point of this table is that the harness produces the numbers; a simulator says nothing about a phone's GPU or CPU. The failures it does show are the ones the audit predicted: p99 climbs to two frames on the clustered zoom sweep and on rotation,