From 1197a48ad67abb51690be1719fff105d650bde37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Wed, 16 Sep 2026 10:54:39 +0200 Subject: [PATCH 1/4] fix(ios): Simulator AX bridge reports enabled from the NotEnabled trait The bridge requested no state attribute, so a disabled control read as a plain button and clients could not observe it; the XCTest runner answered the same screen with `enabled: false`. The guest now reads `XC_kAXXCAttributeTraits` and the decoder derives `enabled` from `UIAccessibilityTraitNotEnabled`. The word is masked as a BigInt because `UIAccessibilityTraitToggleButton` is 2^53, so every switch carries a value past the safe integer range; a missing word leaves `enabled` unknown and a non-integer or negative one is a malformed tree. Source version v1.6.0 rebuilds the cached bridge. Verified on an iPhone 17 Pro simulator (iOS 26.5) against a React Native app with `Pressable disabled` and `accessibilityState={{ disabled: true }}`: the bridge-served raw snapshot now carries `enabled: false` on both controls, and a client `toBeDisabled` assertion that failed on 0.21.1 passes. --- CHANGELOG.md | 6 ++++ apple/snapshot-bridge/SnapshotBridgeRuntime.m | 4 ++- .../fixtures/wire-vocabulary.json | 3 +- .../src/snapshot-source/protocol.test.ts | 2 +- .../src/snapshot-source/protocol.ts | 3 +- .../src/snapshot-source/tree.test.ts | 30 +++++++++++++++++++ .../src/snapshot-source/tree.ts | 17 +++++++++++ 7 files changed, 61 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 333e01bff5..07a1d2cd83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ `session.name` (`default`) instead of the store address (`cwd::default`), so the record read as missing, the `log stream` child leaked, and the next `logs start` on that device failed with "has not reached a confirmed terminal state" (#2647). +- Fixed (ios): Simulator AX bridge snapshots report `enabled`. The bridge requested no state + attribute, so a disabled control — a React Native `Pressable` with `disabled`, for example — read + as a plain button while the XCTest runner answered the same screen with `enabled: false`. The + bridge now reads the element's accessibility traits and derives `enabled` from + `UIAccessibilityTraitNotEnabled`; disabled controls are no longer presented as tappable. Source + version `agent-device-simulator-ax-v1.6.0` rebuilds the cached bridge on first use. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index f384eeb31c..37bb9b2a78 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -18,7 +18,7 @@ NSString *const kProtocolVersionKey = @"protocolVersion"; NSString *const kSourceVersionKey = @"sourceVersion"; NSString *const kRequestIdKey = @"requestId"; -NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.5.5"; +NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.6.0"; const NSUInteger kProtocolVersion = 1; const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024; const NSUInteger kMaximumDepth = 128; @@ -32,6 +32,7 @@ static NSString *const kAttributeIdentifier = @"XC_kAXXCAttributeIdentifier"; static NSString *const kAttributeFrame = @"XC_kAXXCAttributeFrame"; static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType"; +static NSString *const kAttributeTraits = @"XC_kAXXCAttributeTraits"; static NSString *const kAttributeChildren = @"XC_kAXXCAttributeChildren"; static NSString *const kSnapshotAttributes = @"UIAccessibilitySnapshotKeyAttributes"; static NSString *const kSnapshotChildren = @"UIAccessibilitySnapshotKeyChildren"; @@ -291,6 +292,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid kAttributeIdentifier, kAttributeFrame, kAttributeAutomationType, + kAttributeTraits, kAttributeChildren, ]; NSArray *numbers = _attributeNumbersForNames(names); diff --git a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json index ef70facf56..3b78906a73 100644 --- a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json +++ b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json @@ -1,6 +1,6 @@ { "protocolVersion": 1, - "sourceVersion": "agent-device-simulator-ax-v1.5.5", + "sourceVersion": "agent-device-simulator-ax-v1.6.0", "requestKeys": [ "verb", "requestId", @@ -37,6 +37,7 @@ "XC_kAXXCAttributeIdentifier", "XC_kAXXCAttributeFrame", "XC_kAXXCAttributeAutomationType", + "XC_kAXXCAttributeTraits", "XC_kAXXCAttributeChildren" ] } diff --git a/packages/platform-apple/src/snapshot-source/protocol.test.ts b/packages/platform-apple/src/snapshot-source/protocol.test.ts index 443935b9c3..a2839aed01 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.test.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.test.ts @@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async () assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS); assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS); assert.match(nativeSource, /kProtocolVersion = 1/); - assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.5\.5"/); + assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.6\.0"/); for (const key of [ ...wireVocabulary.requestKeys, ...wireVocabulary.responseKeys, diff --git a/packages/platform-apple/src/snapshot-source/protocol.ts b/packages/platform-apple/src/snapshot-source/protocol.ts index 234fee3742..cb2f887f30 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.ts @@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts'; import type { SnapshotSourceLimits } from './types.ts'; export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1; -export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.5.5'; +export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.6.0'; const FRAME_HEADER_BYTES = 4; export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([ @@ -44,6 +44,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([ 'XC_kAXXCAttributeIdentifier', 'XC_kAXXCAttributeFrame', 'XC_kAXXCAttributeAutomationType', + 'XC_kAXXCAttributeTraits', 'XC_kAXXCAttributeChildren', ] as const); diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 1aa266a85c..0b71769103 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -17,6 +17,7 @@ const frame = 'XC_kAXXCAttributeFrame'; const children = 'XC_kAXXCAttributeChildren'; const label = 'XC_kAXXCAttributeLabel'; const automationType = 'XC_kAXXCAttributeAutomationType'; +const traits = 'XC_kAXXCAttributeTraits'; test('the bridge tree becomes one depth-first raw snapshot with viewport evidence', () => { const result = decodeSnapshotBridgeTree( @@ -144,6 +145,35 @@ test('the bridge tree counts web-hosted remote leaves that reach the viewport', ); }); +test('the bridge tree reads enabled from the NotEnabled trait', () => { + const button = (word?: unknown) => ({ + [automationType]: 9, + [label]: 'Place order', + [frame]: { X: 20, Y: 700, Width: 120, Height: 48 }, + ...(word === undefined ? {} : { [traits]: word }), + [children]: [], + }); + const decode = (word?: unknown) => + decodeSnapshotBridgeTree( + { [application]: 'Application', [children]: [button(word)] }, + { truncated: false }, + limits, + ).nodes[1]; + + const buttonTrait = 1; + const notEnabledTrait = 256; + const toggleButtonTrait = 2 ** 53; + assert.equal(decode(buttonTrait)?.enabled, true); + assert.equal(decode(buttonTrait + notEnabledTrait)?.enabled, false); + assert.equal(decode(0)?.enabled, true); + assert.equal(decode(toggleButtonTrait)?.enabled, true, 'a switch reads past the safe range'); + assert.equal(decode(toggleButtonTrait + notEnabledTrait)?.enabled, false, 'a disabled switch'); + assert.equal(decode()?.enabled, undefined, 'no traits word leaves enabled unknown'); + assert.throws(() => decode('256'), /traits-invalid/); + assert.throws(() => decode(1.5), /traits-invalid/); + assert.throws(() => decode(-1), /traits-invalid/); +}); + test('the bridge tree rejects unknown fields, invalid frames, and bounded overflows', () => { assert.throws( () => decodeSnapshotBridgeTree({ [children]: [], unknown: true }, { truncated: false }, limits), diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 596ba93edd..17e5c4e85a 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -14,6 +14,7 @@ const ATTRIBUTE = Object.freeze({ identifier: 'XC_kAXXCAttributeIdentifier', frame: 'XC_kAXXCAttributeFrame', automationType: 'XC_kAXXCAttributeAutomationType', + traits: 'XC_kAXXCAttributeTraits', children: 'XC_kAXXCAttributeChildren', }); @@ -110,6 +111,12 @@ const CLASS_PROMOTED_TYPES: Readonly> = { const NODE_KEYS = new Set(Object.values(ATTRIBUTE)); +/** + * `UIAccessibilityTraitNotEnabled`, the trait UIKit sets on a disabled control. The runner path + * answers `enabled: false` for the same node, so the bridge derives the fact from this bit. + */ +const NOT_ENABLED_TRAIT = 1n << 8n; + /** * A WebKit page — Safari's, or a `WKWebView`'s — lives in a WebContent process and reaches UIKit's * tree as an `AXRemoteElement` under the web view, with its children in that other process. The @@ -211,6 +218,7 @@ function nodeFacts( const baseClass = optionalString(value[ATTRIBUTE.elementBaseType]); const automationType = optionalInteger(value[ATTRIBUTE.automationType]); const frame = frameFromGuest(value[ATTRIBUTE.frame]); + const enabled = enabledFromTraits(value[ATTRIBUTE.traits]); return { index, ...(parentIndex === undefined ? {} : { parentIndex }), @@ -229,6 +237,7 @@ function nodeFacts( ? { identifier: optionalString(value[ATTRIBUTE.identifier]) } : {}), ...(frame ? { rect: frame } : {}), + ...(enabled === undefined ? {} : { enabled }), depth, }; } @@ -295,6 +304,14 @@ function optionalScalar(value: unknown): string | undefined { return undefined; } +function enabledFromTraits(value: unknown): boolean | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) { + throw snapshotSourceError('malformed-tree', 'traits-invalid'); + } + return (BigInt(value) & NOT_ENABLED_TRAIT) === 0n; +} + function optionalInteger(value: unknown): number | undefined { if (value === undefined || value === null) return undefined; if (!Number.isSafeInteger(value)) From 722afea512eebdfd69b1d81fa56f695b1e1c8205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Wed, 16 Sep 2026 11:21:11 +0200 Subject: [PATCH 2/4] fix(ios): a source-declared disabled node folds to non-hittable without hittability evidence Review found the new `enabled` fact never reached presentation on the bridge path: hittability evidence is unavailable there, so the fold returned before consulting `enabled`, and a disabled control kept an open `hittable`. The fold now marks a node the source declared disabled as `hittable: false` regardless of evidence availability, which is what the runner path already presents. The tree test asserts the typed `failureCode` instead of matching error text. --- .../src/ios-snapshot-engine/engine.test.ts | 16 ++++++++++++++++ .../src/ios-snapshot-engine/geometry.ts | 2 +- .../src/snapshot-source/tree.test.ts | 9 ++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index cb0d1cd2c3..07f696f4e1 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -324,6 +324,22 @@ test('unavailable hittability never becomes regular actionability', () => { ); }); +test('a source-declared disabled node is not actionable without hittability evidence', () => { + const request = createIosSnapshotRequest(); + const nodes = nestedNodes().map((entry) => + entry.label === 'Partially visible' ? { ...entry, enabled: false } : entry, + ); + const unavailable = { + ...acquisition(request, nodes), + residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }], + } satisfies IosSnapshotAcquisition; + const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); + const disabled = acquired.payload.nodes.find((node) => node.label === 'Partially visible'); + assert.ok(disabled); + assert.equal(disabled.enabled, false); + assert.equal(disabled.hittable, false); +}); + test('interactive compaction stays available through the engine boundary', () => { const rowRect = { x: 16, y: 80, width: 288, height: 52 }; const compacted = presentIosInteractiveSnapshot([ diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 3c1406e946..fd8bf27f6b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -139,7 +139,7 @@ function foldedHittability( options: IosSnapshotFoldOptions, ): Partial> { if (options.hittabilityAvailable === false) { - return sourceHittable === false ? { hittable: false } : {}; + return sourceHittable === false || !enabled ? { hittable: false } : {}; } return { hittable: diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 0b71769103..63298665f0 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { SnapshotSourceError } from './errors.ts'; import { decodeSnapshotBridgeTree } from './tree.ts'; import type { SnapshotSourceLimits } from './types.ts'; @@ -169,9 +170,11 @@ test('the bridge tree reads enabled from the NotEnabled trait', () => { assert.equal(decode(toggleButtonTrait)?.enabled, true, 'a switch reads past the safe range'); assert.equal(decode(toggleButtonTrait + notEnabledTrait)?.enabled, false, 'a disabled switch'); assert.equal(decode()?.enabled, undefined, 'no traits word leaves enabled unknown'); - assert.throws(() => decode('256'), /traits-invalid/); - assert.throws(() => decode(1.5), /traits-invalid/); - assert.throws(() => decode(-1), /traits-invalid/); + const traitsInvalid = (error: unknown) => + error instanceof SnapshotSourceError && error.failureCode === 'traits-invalid'; + assert.throws(() => decode('256'), traitsInvalid); + assert.throws(() => decode(1.5), traitsInvalid); + assert.throws(() => decode(-1), traitsInvalid); }); test('the bridge tree rejects unknown fields, invalid frames, and bounded overflows', () => { From d92c5a413efdbf5f4673087e7ed53c7a5d3d027d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Wed, 16 Sep 2026 11:35:00 +0200 Subject: [PATCH 3/4] fix(ios): a promoted navigation title affordance retracts the field's hittability The navigation title rule presents a disabled title field as an enabled Button for the whole row, but the fold had already marked the field `hittable: false` from its disabled state, so the Button carried a claim it contradicted; the cloud iOS provider scenario caught it. A replacement patch may now retract a fact by setting it `undefined`, and the rule retracts `hittable`, leaving the row's actionability to the evidence the acquisition actually has. --- .../ios-snapshot-engine/presentation.test.ts | 54 +++++++++++++++++++ .../src/ios-snapshot-engine/transitions.ts | 2 + .../src/ios-snapshot-engine/tree.ts | 13 ++++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts b/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts index 0eed84e345..a2f321968c 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts @@ -529,6 +529,60 @@ test('buildSnapshotState keeps React Native warning banner instead of full-scree ]); }); +test('buildSnapshotState promotes a disabled navigation title field without its hittability', () => { + const nodes: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Demo', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'NavigationBar', + label: 'Team Standup', + rect: { x: 0, y: 56, width: 390, height: 44 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Image', + identifier: 'RoomDetailsIconImageView', + rect: { x: 81, y: 80, width: 14, height: 14 }, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'TextField', + label: 'Team Standup', + value: 'Team Standup', + identifier: 'DisplayNameTextField', + enabled: false, + hittable: false, + rect: { x: 100, y: 67, width: 113, height: 22 }, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Team Standup', + rect: { x: 219, y: 58, width: 85, height: 40 }, + }, + ]; + + const presented = presentIosInteractiveSnapshot(nodes); + const affordance = presented.find((node) => node.identifier === 'DisplayNameTextField'); + + expect(affordance).toMatchObject({ type: 'Button', label: 'Team Standup', enabled: true }); + expect(affordance && 'hittable' in affordance).toBe(false); +}); + test('buildSnapshotState collapses iOS backdrop dismiss wrappers', () => { const nodes = [ { index: 0, depth: 0, type: 'Application', label: 'New Expensify Dev' }, diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts index 5439c6eda9..9ac2f2c1b8 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts @@ -28,10 +28,12 @@ function collectNavigationTitleAffordances( ); if (candidates.length !== 1) continue; const { field, title, image, label } = candidates[0]!; + // The affordance is the whole row, so the disabled field's own actionability does not carry. mergeReplacement(context.replacements, field, { type: 'Button', label, enabled: true, + hittable: undefined, rect: unionRects([image.rect!, field.rect!, title.rect!]), }); context.semanticRepresentativeIndexes.add(field.index); diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index 9c8f0592f1..d856526c46 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -187,12 +187,13 @@ export function isRepeatedStaticNode(node: RawSnapshotNode, parentLabel: string) return type === 'other' || type === 'statictext' || type === 'link'; } +/** A patch key set to `undefined` retracts that fact from the presented node. */ export function mergeReplacement( replacements: Map, node: RawSnapshotNode, patch: Partial, ): void { - replacements.set(node.index, { ...currentReplacement(replacements, node), ...patch }); + replacements.set(node.index, patched(currentReplacement(replacements, node), patch)); } export function updateReplacement( @@ -201,7 +202,15 @@ export function updateReplacement( update: (current: RawSnapshotNode) => Partial, ): void { const current = currentReplacement(replacements, node); - replacements.set(node.index, { ...current, ...update(current) }); + replacements.set(node.index, patched(current, update(current))); +} + +function patched(current: RawSnapshotNode, patch: Partial): RawSnapshotNode { + const next: Record = { ...current, ...patch }; + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) delete next[key]; + } + return next as RawSnapshotNode; } function currentReplacement( From ee22c24eb80a533a3d541fffd8f1d0f4765caa0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Thu, 17 Sep 2026 10:14:08 +0200 Subject: [PATCH 4/4] fix(ios): keep retracted facts through later patches and carry the traits word exactly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (thymikee): `currentReplacement` rebuilt a node from its source under the stored replacement, so a later patch on the same node — a scroll clip, a row label — brought back a fact an earlier rule had retracted: the web Heading's `value`, and the promoted navigation title's `hittable`. The stored replacement is now authoritative, as every other reader of the map already treats it; a regression applies two patches in sequence. The guest sends the accessibility traits word as a decimal string and the decoder parses it with BigInt, so a bit set past 2^53 cannot round into or out of the NotEnabled bit on the way through JSON; the test covers a word past 2^60. The navigation title test moves to transitions.test.ts, keeping presentation.test.ts under the size ratchet. The CHANGELOG names the two presentation effects: bridge disabled controls leave the interactive and Maestro atomic-dispatch counts, and the runner-path title Button joins them. --- CHANGELOG.md | 12 +++- apple/snapshot-bridge/SnapshotBridgeRuntime.m | 5 ++ .../ios-snapshot-engine/presentation.test.ts | 54 ------------------ .../ios-snapshot-engine/transitions.test.ts | 57 +++++++++++++++++++ .../src/ios-snapshot-engine/tree.test.ts | 20 +++++++ .../src/ios-snapshot-engine/tree.ts | 2 +- .../src/snapshot-source/tree.test.ts | 35 ++++++++---- .../src/snapshot-source/tree.ts | 3 +- 8 files changed, 118 insertions(+), 70 deletions(-) create mode 100644 packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 07a1d2cd83..911c51febc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,15 @@ - Fixed (ios): Simulator AX bridge snapshots report `enabled`. The bridge requested no state attribute, so a disabled control — a React Native `Pressable` with `disabled`, for example — read as a plain button while the XCTest runner answered the same screen with `enabled: false`. The - bridge now reads the element's accessibility traits and derives `enabled` from - `UIAccessibilityTraitNotEnabled`; disabled controls are no longer presented as tappable. Source - version `agent-device-simulator-ax-v1.6.0` rebuilds the cached bridge on first use. + bridge now reads the element's accessibility traits (sent as a decimal string, since the word has + bits past 2^53) and derives `enabled` from `UIAccessibilityTraitNotEnabled`. Source version + `agent-device-simulator-ax-v1.6.0` rebuilds the cached bridge on first use. +- Changed (ios): a node the source declares disabled is presented `hittable: false` even when the + capture has no hittability evidence, so on the Simulator bridge a disabled control stops counting + as an interactive node and as a Maestro atomic-dispatch candidate. The navigation title affordance + (a disabled title field presented as an enabled Button for the whole row) no longer carries the + field's `hittable: false`; on the XCTest runner path that Button now counts as interactive and + becomes a Maestro atomic-dispatch candidate where it was excluded before. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index 37bb9b2a78..6f79767b9a 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -186,6 +186,11 @@ - (BOOL)isPrimaryForegroundProcess:(pid_t)pid - (nullable id)jsonValue:(id)value name:(NSString *)name { if (!value || value == [NSNull null]) return nil; + // The traits word is a uint64 bit set; JSON numbers lose its high bits past 2^53, a decimal + // string keeps every bit for the host to parse exactly. + if ([name isEqualToString:kAttributeTraits] && [value isKindOfClass:NSNumber.class]) { + return ((NSNumber *)value).stringValue; + } if ([value isKindOfClass:NSString.class] || [value isKindOfClass:NSNumber.class]) return value; const void *raw = (__bridge const void *)value; diff --git a/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts b/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts index a2f321968c..0eed84e345 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts @@ -529,60 +529,6 @@ test('buildSnapshotState keeps React Native warning banner instead of full-scree ]); }); -test('buildSnapshotState promotes a disabled navigation title field without its hittability', () => { - const nodes: RawSnapshotNode[] = [ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Demo', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'NavigationBar', - label: 'Team Standup', - rect: { x: 0, y: 56, width: 390, height: 44 }, - }, - { - index: 2, - depth: 2, - parentIndex: 1, - type: 'Image', - identifier: 'RoomDetailsIconImageView', - rect: { x: 81, y: 80, width: 14, height: 14 }, - }, - { - index: 3, - depth: 2, - parentIndex: 1, - type: 'TextField', - label: 'Team Standup', - value: 'Team Standup', - identifier: 'DisplayNameTextField', - enabled: false, - hittable: false, - rect: { x: 100, y: 67, width: 113, height: 22 }, - }, - { - index: 4, - depth: 2, - parentIndex: 1, - type: 'StaticText', - label: 'Team Standup', - rect: { x: 219, y: 58, width: 85, height: 40 }, - }, - ]; - - const presented = presentIosInteractiveSnapshot(nodes); - const affordance = presented.find((node) => node.identifier === 'DisplayNameTextField'); - - expect(affordance).toMatchObject({ type: 'Button', label: 'Team Standup', enabled: true }); - expect(affordance && 'hittable' in affordance).toBe(false); -}); - test('buildSnapshotState collapses iOS backdrop dismiss wrappers', () => { const nodes = [ { index: 0, depth: 0, type: 'Application', label: 'New Expensify Dev' }, diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts new file mode 100644 index 0000000000..22522371a2 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; + +test('a disabled navigation title field is promoted to a Button without its hittability', () => { + const nodes: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Demo', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'NavigationBar', + label: 'Team Standup', + rect: { x: 0, y: 56, width: 390, height: 44 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Image', + identifier: 'RoomDetailsIconImageView', + rect: { x: 81, y: 80, width: 14, height: 14 }, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'TextField', + label: 'Team Standup', + value: 'Team Standup', + identifier: 'DisplayNameTextField', + enabled: false, + hittable: false, + rect: { x: 100, y: 67, width: 113, height: 22 }, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Team Standup', + rect: { x: 219, y: 58, width: 85, height: 40 }, + }, + ]; + + const presented = presentIosInteractiveSnapshot(nodes); + const affordance = presented.find((node) => node.identifier === 'DisplayNameTextField'); + + expect(affordance).toMatchObject({ type: 'Button', label: 'Team Standup', enabled: true }); + expect(affordance && 'hittable' in affordance).toBe(false); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts index ce388f57c1..6a21a561dd 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts @@ -18,3 +18,23 @@ test('replacement updates derive patches from the composed node', () => { hiddenContentBelow: true, }); }); + +test('a retracted fact stays retracted when a later rule patches the same node', () => { + const heading: RawSnapshotNode = { + index: 3, + depth: 3, + parentIndex: 2, + type: 'Other', + label: 'Welcome', + value: '1', + rect: { x: 0, y: 700, width: 390, height: 300 }, + }; + const replacements = new Map(); + mergeReplacement(replacements, heading, { type: 'Heading', value: undefined }); + mergeReplacement(replacements, heading, { rect: { x: 0, y: 700, width: 390, height: 144 } }); + updateReplacement(replacements, heading, () => ({ label: 'Welcome!' })); + + const presented = replacements.get(heading.index); + expect(presented).toMatchObject({ type: 'Heading', label: 'Welcome!' }); + expect(presented && 'value' in presented).toBe(false); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index d856526c46..8cd1d5c77e 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -217,7 +217,7 @@ function currentReplacement( replacements: ReadonlyMap, node: RawSnapshotNode, ): RawSnapshotNode { - return { ...node, ...replacements.get(node.index) }; + return replacements.get(node.index) ?? node; } export function findLargestViewportRect(nodes: Iterable): RawSnapshotNode['rect'] { diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 63298665f0..7586bd72d7 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -161,20 +161,33 @@ test('the bridge tree reads enabled from the NotEnabled trait', () => { limits, ).nodes[1]; - const buttonTrait = 1; - const notEnabledTrait = 256; - const toggleButtonTrait = 2 ** 53; - assert.equal(decode(buttonTrait)?.enabled, true); - assert.equal(decode(buttonTrait + notEnabledTrait)?.enabled, false); - assert.equal(decode(0)?.enabled, true); - assert.equal(decode(toggleButtonTrait)?.enabled, true, 'a switch reads past the safe range'); - assert.equal(decode(toggleButtonTrait + notEnabledTrait)?.enabled, false, 'a disabled switch'); + const buttonTrait = 1n; + const notEnabledTrait = 1n << 8n; + const toggleButtonTrait = 1n << 53n; + const privateHighTrait = 1n << 60n; + const word = (traits: bigint) => traits.toString(); + assert.equal(decode(word(buttonTrait))?.enabled, true); + assert.equal(decode(word(buttonTrait | notEnabledTrait))?.enabled, false); + assert.equal(decode(word(0n))?.enabled, true); + assert.equal(decode(word(toggleButtonTrait))?.enabled, true, 'a switch reads past 2^53'); + assert.equal( + decode(word(toggleButtonTrait | notEnabledTrait))?.enabled, + false, + 'a disabled switch', + ); + assert.equal( + decode(word(privateHighTrait | notEnabledTrait))?.enabled, + false, + 'a word past double precision keeps bit 8', + ); + assert.equal(decode(word(privateHighTrait | 255n))?.enabled, true, 'no carry into bit 8'); assert.equal(decode()?.enabled, undefined, 'no traits word leaves enabled unknown'); const traitsInvalid = (error: unknown) => error instanceof SnapshotSourceError && error.failureCode === 'traits-invalid'; - assert.throws(() => decode('256'), traitsInvalid); - assert.throws(() => decode(1.5), traitsInvalid); - assert.throws(() => decode(-1), traitsInvalid); + assert.throws(() => decode(256), traitsInvalid); + assert.throws(() => decode('1.5'), traitsInvalid); + assert.throws(() => decode('-1'), traitsInvalid); + assert.throws(() => decode(''), traitsInvalid); }); test('the bridge tree rejects unknown fields, invalid frames, and bounded overflows', () => { diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 17e5c4e85a..58f08a283a 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -304,9 +304,10 @@ function optionalScalar(value: unknown): string | undefined { return undefined; } +/** The guest sends the uint64 traits word as a decimal string so no bit is lost to a double. */ function enabledFromTraits(value: unknown): boolean | undefined { if (value === undefined || value === null) return undefined; - if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) { + if (typeof value !== 'string' || !/^\d{1,20}$/.test(value)) { throw snapshotSourceError('malformed-tree', 'traits-invalid'); } return (BigInt(value) & NOT_ENABLED_TRAIT) === 0n;