diff --git a/CHANGELOG.md b/CHANGELOG.md index 333e01bff5..911c51febc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ `session.name` (`default`) instead of the store address (`cwd::default`), so the record read as missing, the `log stream` child leaked, and the next `logs start` on that device failed with "has not reached a confirmed terminal state" (#2647). +- Fixed (ios): Simulator AX bridge snapshots report `enabled`. The bridge requested no state + attribute, so a disabled control — a React Native `Pressable` with `disabled`, for example — read + as a plain button while the XCTest runner answered the same screen with `enabled: false`. The + bridge now reads the element's accessibility traits (sent as a decimal string, since the word has + bits past 2^53) and derives `enabled` from `UIAccessibilityTraitNotEnabled`. Source version + `agent-device-simulator-ax-v1.6.0` rebuilds the cached bridge on first use. +- Changed (ios): a node the source declares disabled is presented `hittable: false` even when the + capture has no hittability evidence, so on the Simulator bridge a disabled control stops counting + as an interactive node and as a Maestro atomic-dispatch candidate. The navigation title affordance + (a disabled title field presented as an enabled Button for the whole row) no longer carries the + field's `hittable: false`; on the XCTest runner path that Button now counts as interactive and + becomes a Maestro atomic-dispatch candidate where it was excluded before. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index f384eeb31c..6f79767b9a 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -18,7 +18,7 @@ NSString *const kProtocolVersionKey = @"protocolVersion"; NSString *const kSourceVersionKey = @"sourceVersion"; NSString *const kRequestIdKey = @"requestId"; -NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.5.5"; +NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.6.0"; const NSUInteger kProtocolVersion = 1; const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024; const NSUInteger kMaximumDepth = 128; @@ -32,6 +32,7 @@ static NSString *const kAttributeIdentifier = @"XC_kAXXCAttributeIdentifier"; static NSString *const kAttributeFrame = @"XC_kAXXCAttributeFrame"; static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType"; +static NSString *const kAttributeTraits = @"XC_kAXXCAttributeTraits"; static NSString *const kAttributeChildren = @"XC_kAXXCAttributeChildren"; static NSString *const kSnapshotAttributes = @"UIAccessibilitySnapshotKeyAttributes"; static NSString *const kSnapshotChildren = @"UIAccessibilitySnapshotKeyChildren"; @@ -185,6 +186,11 @@ - (BOOL)isPrimaryForegroundProcess:(pid_t)pid - (nullable id)jsonValue:(id)value name:(NSString *)name { if (!value || value == [NSNull null]) return nil; + // The traits word is a uint64 bit set; JSON numbers lose its high bits past 2^53, a decimal + // string keeps every bit for the host to parse exactly. + if ([name isEqualToString:kAttributeTraits] && [value isKindOfClass:NSNumber.class]) { + return ((NSNumber *)value).stringValue; + } if ([value isKindOfClass:NSString.class] || [value isKindOfClass:NSNumber.class]) return value; const void *raw = (__bridge const void *)value; @@ -291,6 +297,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid kAttributeIdentifier, kAttributeFrame, kAttributeAutomationType, + kAttributeTraits, kAttributeChildren, ]; NSArray *numbers = _attributeNumbersForNames(names); diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index cb0d1cd2c3..07f696f4e1 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -324,6 +324,22 @@ test('unavailable hittability never becomes regular actionability', () => { ); }); +test('a source-declared disabled node is not actionable without hittability evidence', () => { + const request = createIosSnapshotRequest(); + const nodes = nestedNodes().map((entry) => + entry.label === 'Partially visible' ? { ...entry, enabled: false } : entry, + ); + const unavailable = { + ...acquisition(request, nodes), + residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }], + } satisfies IosSnapshotAcquisition; + const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); + const disabled = acquired.payload.nodes.find((node) => node.label === 'Partially visible'); + assert.ok(disabled); + assert.equal(disabled.enabled, false); + assert.equal(disabled.hittable, false); +}); + test('interactive compaction stays available through the engine boundary', () => { const rowRect = { x: 16, y: 80, width: 288, height: 52 }; const compacted = presentIosInteractiveSnapshot([ diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 3c1406e946..fd8bf27f6b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -139,7 +139,7 @@ function foldedHittability( options: IosSnapshotFoldOptions, ): Partial> { if (options.hittabilityAvailable === false) { - return sourceHittable === false ? { hittable: false } : {}; + return sourceHittable === false || !enabled ? { hittable: false } : {}; } return { hittable: diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts new file mode 100644 index 0000000000..22522371a2 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; + +test('a disabled navigation title field is promoted to a Button without its hittability', () => { + const nodes: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Demo', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'NavigationBar', + label: 'Team Standup', + rect: { x: 0, y: 56, width: 390, height: 44 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Image', + identifier: 'RoomDetailsIconImageView', + rect: { x: 81, y: 80, width: 14, height: 14 }, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'TextField', + label: 'Team Standup', + value: 'Team Standup', + identifier: 'DisplayNameTextField', + enabled: false, + hittable: false, + rect: { x: 100, y: 67, width: 113, height: 22 }, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Team Standup', + rect: { x: 219, y: 58, width: 85, height: 40 }, + }, + ]; + + const presented = presentIosInteractiveSnapshot(nodes); + const affordance = presented.find((node) => node.identifier === 'DisplayNameTextField'); + + expect(affordance).toMatchObject({ type: 'Button', label: 'Team Standup', enabled: true }); + expect(affordance && 'hittable' in affordance).toBe(false); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts index 5439c6eda9..9ac2f2c1b8 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts @@ -28,10 +28,12 @@ function collectNavigationTitleAffordances( ); if (candidates.length !== 1) continue; const { field, title, image, label } = candidates[0]!; + // The affordance is the whole row, so the disabled field's own actionability does not carry. mergeReplacement(context.replacements, field, { type: 'Button', label, enabled: true, + hittable: undefined, rect: unionRects([image.rect!, field.rect!, title.rect!]), }); context.semanticRepresentativeIndexes.add(field.index); diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts index ce388f57c1..6a21a561dd 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts @@ -18,3 +18,23 @@ test('replacement updates derive patches from the composed node', () => { hiddenContentBelow: true, }); }); + +test('a retracted fact stays retracted when a later rule patches the same node', () => { + const heading: RawSnapshotNode = { + index: 3, + depth: 3, + parentIndex: 2, + type: 'Other', + label: 'Welcome', + value: '1', + rect: { x: 0, y: 700, width: 390, height: 300 }, + }; + const replacements = new Map(); + mergeReplacement(replacements, heading, { type: 'Heading', value: undefined }); + mergeReplacement(replacements, heading, { rect: { x: 0, y: 700, width: 390, height: 144 } }); + updateReplacement(replacements, heading, () => ({ label: 'Welcome!' })); + + const presented = replacements.get(heading.index); + expect(presented).toMatchObject({ type: 'Heading', label: 'Welcome!' }); + expect(presented && 'value' in presented).toBe(false); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index 9c8f0592f1..8cd1d5c77e 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -187,12 +187,13 @@ export function isRepeatedStaticNode(node: RawSnapshotNode, parentLabel: string) return type === 'other' || type === 'statictext' || type === 'link'; } +/** A patch key set to `undefined` retracts that fact from the presented node. */ export function mergeReplacement( replacements: Map, node: RawSnapshotNode, patch: Partial, ): void { - replacements.set(node.index, { ...currentReplacement(replacements, node), ...patch }); + replacements.set(node.index, patched(currentReplacement(replacements, node), patch)); } export function updateReplacement( @@ -201,14 +202,22 @@ export function updateReplacement( update: (current: RawSnapshotNode) => Partial, ): void { const current = currentReplacement(replacements, node); - replacements.set(node.index, { ...current, ...update(current) }); + replacements.set(node.index, patched(current, update(current))); +} + +function patched(current: RawSnapshotNode, patch: Partial): RawSnapshotNode { + const next: Record = { ...current, ...patch }; + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) delete next[key]; + } + return next as RawSnapshotNode; } function currentReplacement( replacements: ReadonlyMap, node: RawSnapshotNode, ): RawSnapshotNode { - return { ...node, ...replacements.get(node.index) }; + return replacements.get(node.index) ?? node; } export function findLargestViewportRect(nodes: Iterable): RawSnapshotNode['rect'] { diff --git a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json index ef70facf56..3b78906a73 100644 --- a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json +++ b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json @@ -1,6 +1,6 @@ { "protocolVersion": 1, - "sourceVersion": "agent-device-simulator-ax-v1.5.5", + "sourceVersion": "agent-device-simulator-ax-v1.6.0", "requestKeys": [ "verb", "requestId", @@ -37,6 +37,7 @@ "XC_kAXXCAttributeIdentifier", "XC_kAXXCAttributeFrame", "XC_kAXXCAttributeAutomationType", + "XC_kAXXCAttributeTraits", "XC_kAXXCAttributeChildren" ] } diff --git a/packages/platform-apple/src/snapshot-source/protocol.test.ts b/packages/platform-apple/src/snapshot-source/protocol.test.ts index 443935b9c3..a2839aed01 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.test.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.test.ts @@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async () assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS); assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS); assert.match(nativeSource, /kProtocolVersion = 1/); - assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.5\.5"/); + assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.6\.0"/); for (const key of [ ...wireVocabulary.requestKeys, ...wireVocabulary.responseKeys, diff --git a/packages/platform-apple/src/snapshot-source/protocol.ts b/packages/platform-apple/src/snapshot-source/protocol.ts index 234fee3742..cb2f887f30 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.ts @@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts'; import type { SnapshotSourceLimits } from './types.ts'; export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1; -export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.5.5'; +export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.6.0'; const FRAME_HEADER_BYTES = 4; export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([ @@ -44,6 +44,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([ 'XC_kAXXCAttributeIdentifier', 'XC_kAXXCAttributeFrame', 'XC_kAXXCAttributeAutomationType', + 'XC_kAXXCAttributeTraits', 'XC_kAXXCAttributeChildren', ] as const); diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 1aa266a85c..7586bd72d7 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { SnapshotSourceError } from './errors.ts'; import { decodeSnapshotBridgeTree } from './tree.ts'; import type { SnapshotSourceLimits } from './types.ts'; @@ -17,6 +18,7 @@ const frame = 'XC_kAXXCAttributeFrame'; const children = 'XC_kAXXCAttributeChildren'; const label = 'XC_kAXXCAttributeLabel'; const automationType = 'XC_kAXXCAttributeAutomationType'; +const traits = 'XC_kAXXCAttributeTraits'; test('the bridge tree becomes one depth-first raw snapshot with viewport evidence', () => { const result = decodeSnapshotBridgeTree( @@ -144,6 +146,50 @@ test('the bridge tree counts web-hosted remote leaves that reach the viewport', ); }); +test('the bridge tree reads enabled from the NotEnabled trait', () => { + const button = (word?: unknown) => ({ + [automationType]: 9, + [label]: 'Place order', + [frame]: { X: 20, Y: 700, Width: 120, Height: 48 }, + ...(word === undefined ? {} : { [traits]: word }), + [children]: [], + }); + const decode = (word?: unknown) => + decodeSnapshotBridgeTree( + { [application]: 'Application', [children]: [button(word)] }, + { truncated: false }, + limits, + ).nodes[1]; + + const buttonTrait = 1n; + const notEnabledTrait = 1n << 8n; + const toggleButtonTrait = 1n << 53n; + const privateHighTrait = 1n << 60n; + const word = (traits: bigint) => traits.toString(); + assert.equal(decode(word(buttonTrait))?.enabled, true); + assert.equal(decode(word(buttonTrait | notEnabledTrait))?.enabled, false); + assert.equal(decode(word(0n))?.enabled, true); + assert.equal(decode(word(toggleButtonTrait))?.enabled, true, 'a switch reads past 2^53'); + assert.equal( + decode(word(toggleButtonTrait | notEnabledTrait))?.enabled, + false, + 'a disabled switch', + ); + assert.equal( + decode(word(privateHighTrait | notEnabledTrait))?.enabled, + false, + 'a word past double precision keeps bit 8', + ); + assert.equal(decode(word(privateHighTrait | 255n))?.enabled, true, 'no carry into bit 8'); + assert.equal(decode()?.enabled, undefined, 'no traits word leaves enabled unknown'); + const traitsInvalid = (error: unknown) => + error instanceof SnapshotSourceError && error.failureCode === 'traits-invalid'; + assert.throws(() => decode(256), traitsInvalid); + assert.throws(() => decode('1.5'), traitsInvalid); + assert.throws(() => decode('-1'), traitsInvalid); + assert.throws(() => decode(''), traitsInvalid); +}); + test('the bridge tree rejects unknown fields, invalid frames, and bounded overflows', () => { assert.throws( () => decodeSnapshotBridgeTree({ [children]: [], unknown: true }, { truncated: false }, limits), diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 596ba93edd..58f08a283a 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -14,6 +14,7 @@ const ATTRIBUTE = Object.freeze({ identifier: 'XC_kAXXCAttributeIdentifier', frame: 'XC_kAXXCAttributeFrame', automationType: 'XC_kAXXCAttributeAutomationType', + traits: 'XC_kAXXCAttributeTraits', children: 'XC_kAXXCAttributeChildren', }); @@ -110,6 +111,12 @@ const CLASS_PROMOTED_TYPES: Readonly> = { const NODE_KEYS = new Set(Object.values(ATTRIBUTE)); +/** + * `UIAccessibilityTraitNotEnabled`, the trait UIKit sets on a disabled control. The runner path + * answers `enabled: false` for the same node, so the bridge derives the fact from this bit. + */ +const NOT_ENABLED_TRAIT = 1n << 8n; + /** * A WebKit page — Safari's, or a `WKWebView`'s — lives in a WebContent process and reaches UIKit's * tree as an `AXRemoteElement` under the web view, with its children in that other process. The @@ -211,6 +218,7 @@ function nodeFacts( const baseClass = optionalString(value[ATTRIBUTE.elementBaseType]); const automationType = optionalInteger(value[ATTRIBUTE.automationType]); const frame = frameFromGuest(value[ATTRIBUTE.frame]); + const enabled = enabledFromTraits(value[ATTRIBUTE.traits]); return { index, ...(parentIndex === undefined ? {} : { parentIndex }), @@ -229,6 +237,7 @@ function nodeFacts( ? { identifier: optionalString(value[ATTRIBUTE.identifier]) } : {}), ...(frame ? { rect: frame } : {}), + ...(enabled === undefined ? {} : { enabled }), depth, }; } @@ -295,6 +304,15 @@ function optionalScalar(value: unknown): string | undefined { return undefined; } +/** The guest sends the uint64 traits word as a decimal string so no bit is lost to a double. */ +function enabledFromTraits(value: unknown): boolean | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'string' || !/^\d{1,20}$/.test(value)) { + throw snapshotSourceError('malformed-tree', 'traits-invalid'); + } + return (BigInt(value) & NOT_ENABLED_TRAIT) === 0n; +} + function optionalInteger(value: unknown): number | undefined { if (value === undefined || value === null) return undefined; if (!Number.isSafeInteger(value))