From 445b915d4501b22d43477cf9eeaf3f91087a1aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 9 Aug 2026 22:49:57 +0200 Subject: [PATCH 1/9] Avoid interactive children in parent taps --- CONTEXT.md | 4 + .../__tests__/help-conformance-bench.test.ts | 5 + scripts/help-conformance-cases.mjs | 33 +++ scripts/help-conformance-sample-outputs.mjs | 6 +- skills/agent-device/SKILL.md | 2 +- src/__tests__/cli-help.test.ts | 2 +- .../parser/__tests__/cli-help-topics.test.ts | 8 +- src/cli/parser/cli-help.ts | 4 +- .../runtime/resolution-touch-point.test.ts | 81 +++++++ .../interaction/runtime/resolution.ts | 36 +++- src/core/interaction-touch-point.test.ts | 204 ++++++++++++++++++ src/core/interaction-touch-point.ts | 200 +++++++++++++++++ 12 files changed, 570 insertions(+), 15 deletions(-) create mode 100644 src/commands/interaction/runtime/resolution-touch-point.test.ts create mode 100644 src/core/interaction-touch-point.test.ts create mode 100644 src/core/interaction-touch-point.ts diff --git a/CONTEXT.md b/CONTEXT.md index d1ac350e7..b9cfbec45 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -113,6 +113,10 @@ task touches: center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. +- Parent-owned touch point: runtime ref/selector activation keeps the resolved parent identity but, + when its center belongs to an independently interactive descendant, moves the coordinate to the + nearest region with bounded clearance from those child controls. The exact center remains the + zero-cost default when no child competes; a fully tiled parent fails closed so a child must be named. - Guarantee cell: one (dispatch path, guarantee) entry in `packages/contracts/src/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested. diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts index 2872197ce..0f13ef9ae 100644 --- a/scripts/__tests__/help-conformance-bench.test.ts +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -535,6 +535,11 @@ test('compact skill starts a known-app task with foreground open and an initial assert.deepEqual(openingCommands, ['agent-device open --foreground']); assert.match(ordinaryStart, /returns the initial interactive snapshot in the same call/); + assert.match( + ordinaryStart, + /copy refs? byte-for-byte.*leading `@`.*part of the ref/i, + 'the always-loaded skill must preserve the @ prefix before topic help is available', + ); }); test('plan validator applies narrow grammar to permitted external commands', async () => { diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 90b5cf2f7..c7230d7d4 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -19,6 +19,11 @@ import { const RAW_COORDINATE_TARGET = /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; +// Ref pasted without its leading @: byte-for-byte copying is the contract, +// and a dropped @ silently stops targeting the observed element. +const BARE_REF_TARGET = + /(?:^|\n)(?:agent-device\s+)?(?:press|tap|click|fill|longpress)\s+['"]?e\d+\b/i; + function quiz(sample, question) { return `Read this previous agent-device output, then plan the next command: @@ -436,6 +441,7 @@ Use the output already shown to determine whether the feed-search UI is present, { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, { id: 'noWaitStable', pattern: /wait\s+stable/i }, { id: 'noFill', pattern: /\bfill\b/i }, + { id: 'noBareRefTarget', pattern: BARE_REF_TARGET }, { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, ], }, @@ -457,6 +463,31 @@ Use the output already shown to determine whether the feed-search UI is present, { id: 'noWaitStable', pattern: /wait\s+stable/i }, { id: 'noFill', pattern: /\bfill\b/i }, { id: 'noCallstackLeakage', pattern: /(?:callstack|@e64)/i }, + { id: 'noBareRefTarget', pattern: BARE_REF_TARGET }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + // ADR 0014: settled tails pin unchanged interactive refs because the + // partial frame admits only refs copied with the response generation. + id: 'sample-output-settle-tail-pinned-ref-copied-exactly', + docs: ['--help:first30'], + task: quiz( + SETTLE_TAIL_SAMPLE, + 'The task is to open the Profile tab. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'pressesPinnedTailRefOrExactLabel', + pattern: + /(?:^|\n)agent-device\s+(?:press|click)\s+(?:@e40~s5\b|'?label="?Profile"?'?)[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { id: 'noUnpinnedRef', pattern: /@e40(?!~s5\b)/i }, + { id: 'noBareRefTarget', pattern: BARE_REF_TARGET }, + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, ], }, @@ -683,6 +714,8 @@ Use the output already shown to determine whether the feed-search UI is present, id: 'noPressingActionNameAsSelector', pattern: /(?:^|\n)agent-device\s+(?:press|click|find)\b[^\n]*(?:label|text)="?reply"?/i, }, + { id: 'noSnapshotDetour', pattern: /\bsnapshot\b/i }, + { id: 'noBareRefTarget', pattern: BARE_REF_TARGET }, { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, ], }, diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index f03e1ba22..09fabe059 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -160,9 +160,9 @@ Warning: The session is open, but the initial interactive snapshot failed (COMMA // Reply/Repost/menu controls have no separate child nodes in the tree, so // snapshot -i alone would show a plain link with no way to act on it. // snapshot -i --actions names the hidden affordances instead of hiding them -// silently; the names are evidence only, never directly invokable (help -// workflow: "reach via its detail screen, labeled children elsewhere, or -// coordinates"). +// silently; the names are destination hints, never directly invokable. The +// parent ref reaches the detail screen, whose settled diff reveals the child +// control. Coordinates remain the last resort. export const MERGED_CARD_ACTIONS_SAMPLE = { command: 'agent-device snapshot -i --actions', output: `Snapshot: 4 nodes diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 9f47d7cf3..48afeba68 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -11,7 +11,7 @@ For an ordinary app-driving task with a known app or bundle id, start directly. agent-device open --foreground ``` -`open --foreground` keeps normal configured target selection and returns the initial interactive snapshot in the same call. Continue from its current refs. Prefer a concrete `@eN` ref from the current snapshot over a broad mutation selector. When a response prints a pinned ref such as `@e12~s42` — including an ambiguity candidate or settled diff — copy the whole pinned ref exactly; a bare ref from a partial result is intentionally rejected. +`open --foreground` keeps normal configured target selection and returns the initial interactive snapshot in the same call. Continue from its current refs. Prefer a concrete `@eN` ref from the current snapshot over a broad mutation selector. Copy refs byte-for-byte as printed: the leading `@` is part of the ref (`press @e72`, never `press e72`). When a response prints a pinned ref such as `@e12~s42` — including an ambiguity candidate or settled diff — copy the whole pinned ref exactly; a bare ref from a partial result is intentionally rejected. Default loop: `open -> act with a current ref or specific selector -> verify -> close`. Use `--settle` on planned `press`, `click`, `fill`, `longpress`, `scroll`, or `back`; continue from the settled diff when it already proves the next state. If a mutation returns `AMBIGUOUS_MATCH`, retry one listed pinned candidate rather than adding `--first` or guessing by coordinates. diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index 30c36cc55..b1936aa44 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -145,7 +145,7 @@ test('help workflow advertises open --foreground and snapshot -i --actions', asy ); assert.match( result.stdout, - /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, + /iOS merged: child ref => press it; else press parent @ref --settle\. Names are not selectors/, ); }); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index bb92888a9..b55c74b2d 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -96,7 +96,11 @@ test('usage includes agent workflows, config, environment, and examples footers' assert.match(usageText, /type never accepts --settle/); assert.match(usageText, /explicit success confirmation is visible, stop/); assert.match(usageText, /Follow structured command hints before choosing a recovery action/); - assert.match(usageText, /Targets are concrete refs or selectors/); + assert.match(usageText, /Refs\/selectors/); + assert.match( + usageText, + /Copy refs exactly, including @ and ~sN pins \(@e72, @e12~s4\); without @ it is not a ref/, + ); assert.ok( usageText.includes(`Selector keys are only: ${SELECTOR_KEY_NAMES.join(', ')}.`), 'The first-screen selector vocabulary must match the parser source of truth.', @@ -308,7 +312,7 @@ test('usageForCommand resolves workflow help topic', async () => { assert.match(help, /get text alone, or stopping one screen early, is not enough/); assert.match( help, - /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, + /iOS merged: child ref => press it; else press parent @ref --settle\. Names are not selectors/, ); assert.match(help, /Perf\/memory\/log\/network\/trace\/crash: help debugging/); assert.match(help, /Recording, save-script, batch, replay repair: help scripting/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 88f54ecb6..a20f7478b 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -81,7 +81,7 @@ const AGENT_START_LINES = [ 'Default app loop: agent-device open --foreground -> mutate a current target from its initial snapshot with --settle -> continue from that settled diff -> agent-device close.', 'Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', 'Follow structured command hints before choosing a recovery action.', - 'Targets are concrete refs or selectors: @e12, label="Query", role=button label="Submit".', + 'Refs/selectors: @e12, label="Query". Copy refs exactly, including @ and ~sN pins (@e72, @e12~s4); without @ it is not a ref.', 'Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.', 'A literal @ handle is a label such as label="@account.example", never a bare @ref.', 'Boundary shapes: agent-device fill \'label="Query"\' "text" --settle; agent-device press \'role=button label="Submit"\' --settle.', @@ -264,7 +264,7 @@ Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. - iOS sim: snapshot -i --actions shows merged actions; use detail/coords, not names. + iOS merged: child ref => press it; else press parent @ref --settle. Names are not selectors. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset. diff --git a/src/commands/interaction/runtime/resolution-touch-point.test.ts b/src/commands/interaction/runtime/resolution-touch-point.test.ts new file mode 100644 index 000000000..a687ff04a --- /dev/null +++ b/src/commands/interaction/runtime/resolution-touch-point.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; +import { ref, selector } from './selector-read-utils.ts'; + +function postSnapshot() { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Link', + label: 'feedItem-by-whiskers.test', + rect: { x: 0, y: 180, width: 402, height: 520 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Link', + label: "whiskers's avatar", + rect: { x: 16, y: 196, width: 48, height: 48 }, + hittable: true, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Mochi napping in a sunbeam #caturday', + rect: { x: 72, y: 246, width: 300, height: 36 }, + hittable: true, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'A kitten asleep in sunlight.', + rect: { x: 16, y: 292, width: 370, height: 330 }, + hittable: true, + }, + ]); +} + +test('runtime ref and selector paths dispatch the same parent-owned post point', async () => { + const refCalls: Point[] = []; + const selectorCalls: Point[] = []; + const refDevice = createInteractionDevice(postSnapshot(), { + tap: async (_context, point) => { + refCalls.push(point); + return { ok: true }; + }, + }); + const selectorDevice = createInteractionDevice(postSnapshot(), { + tap: async (_context, point) => { + selectorCalls.push(point); + return { ok: true }; + }, + }); + + await refDevice.interactions.click(ref('@e2'), { session: 'default' }); + await selectorDevice.interactions.click(selector('label="feedItem-by-whiskers.test"'), { + session: 'default', + }); + + assert.equal(refCalls.length, 1); + assert.deepEqual(selectorCalls, refCalls); + assert.ok(refCalls[0]!.y < 292); + assert.notDeepEqual(refCalls[0], { x: 201, y: 440 }); +}); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 975eb8b82..2d68504cb 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -44,6 +44,7 @@ import type { } from '@agent-device/contracts/interaction'; import { now, toBackendContext } from '../../runtime-common.ts'; import { resolveActionableTouchResolution } from '../../../core/interaction-targeting.ts'; +import { resolveInteractionTouchPoint } from '../../../core/interaction-touch-point.ts'; import { localIdentitiesEqual, readNodeLocalIdentity, @@ -301,7 +302,11 @@ async function resolveRefInteractionTarget( target.ref, params.action, ); - const point = resolveNodeCenter(visibleNode, `Ref ${target.ref} not found or has invalid bounds`); + const point = resolveNodeTouchPoint( + visibleNode, + nodes, + `Ref ${target.ref} not found or has invalid bounds`, + ); return { kind: 'ref', point, @@ -374,8 +379,9 @@ async function resolveSelectorInteractionTarget( resolved.selector, params.action, ); - const point = resolveNodeCenter( + const point = resolveNodeTouchPoint( visibleNode, + capture.snapshot.nodes, `Selector ${resolved.selector} resolved to invalid bounds`, ); return { @@ -764,10 +770,28 @@ type ResolvedRefNode = { resolution: ResolutionDisclosure; }; -function resolveNodeCenter(node: SnapshotNode, message: string): Point { - const point = resolveRectCenter(node.rect); - if (!point) throw new AppError('COMMAND_FAILED', message); - return point; +function resolveNodeTouchPoint( + node: SnapshotNode, + nodes: SnapshotState['nodes'], + message: string, +): Point { + const effectiveViewport = resolveEffectiveViewportRect(node, nodes); + const rootViewport = node.rect ? resolveViewportRect(nodes, node.rect) : null; + const resolution = resolveInteractionTouchPoint(nodes, node, { + bounds: [effectiveViewport, rootViewport].filter((rect) => rect !== null), + }); + if (resolution.kind === 'resolved') return resolution.point; + if (resolution.kind === 'invalid') throw new AppError('COMMAND_FAILED', message); + throw new AppError( + 'COMMAND_FAILED', + `Ref @${node.ref} has no parent-owned touch point outside its interactive descendants`, + { + reason: 'covered_by_interactive_descendants', + ref: `@${node.ref}`, + competitorRefs: resolution.competitorRefs.slice(0, 5).map((ref) => `@${ref}`), + hint: 'Tap the specific interactive child you intend, or use a more specific selector. Every safely tappable region of the parent belongs to one of its child controls.', + }, + ); } function isUsableResolvedNode(node: SnapshotNode | null | undefined): node is SnapshotNode { diff --git a/src/core/interaction-touch-point.test.ts b/src/core/interaction-touch-point.test.ts new file mode 100644 index 000000000..113831c63 --- /dev/null +++ b/src/core/interaction-touch-point.test.ts @@ -0,0 +1,204 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '../__tests__/test-utils/index.ts'; +import { resolveInteractionTouchPoint } from './interaction-touch-point.ts'; + +function blueskyPostNodes() { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Link', + label: 'feedItem-by-whiskers.test', + rect: { x: 0, y: 180, width: 402, height: 520 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Link', + label: "whiskers's avatar", + rect: { x: 16, y: 196, width: 48, height: 48 }, + hittable: true, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Mochi napping in a sunbeam #caturday', + rect: { x: 72, y: 246, width: 300, height: 36 }, + hittable: true, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'A kitten asleep in sunlight.', + rect: { x: 16, y: 292, width: 370, height: 330 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Reply (0 replies)', + rect: { x: 16, y: 638, width: 48, height: 44 }, + hittable: true, + }, + { + index: 6, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Repost (0 reposts)', + rect: { x: 82, y: 638, width: 48, height: 44 }, + hittable: true, + }, + ]).nodes; +} + +test('chooses a parent-owned post point outside interactive descendants', () => { + const nodes = blueskyPostNodes(); + + const resolution = resolveInteractionTouchPoint(nodes, nodes[1]!); + + assert.equal(resolution.kind, 'resolved'); + if (resolution.kind !== 'resolved') return; + assert.equal(resolution.strategy, 'parent-owned'); + assert.ok(resolution.point.y < 292, 'point should land in the post text band above the image'); + assert.notDeepEqual(resolution.point, { x: 201, y: 440 }); +}); + +test('keeps the exact center when there are no competing interactive descendants', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + rect: { x: 20, y: 30, width: 80, height: 40 }, + hittable: true, + }, + ]).nodes; + + assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { + kind: 'resolved', + point: { x: 60, y: 50 }, + strategy: 'center', + }); +}); + +test('fails closed when interactive descendants tile the parent', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Card', + rect: { x: 0, y: 0, width: 100, height: 100 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Top', + rect: { x: 0, y: 0, width: 100, height: 50 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Bottom', + rect: { x: 0, y: 50, width: 100, height: 50 }, + hittable: true, + }, + ]).nodes; + + assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { + kind: 'blocked', + competitorRefs: ['e2', 'e3'], + }); +}); + +test('same-rect wrapper descendants remain part of the parent touch surface', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Card', + rect: { x: 10, y: 20, width: 200, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Pressable wrapper', + rect: { x: 10, y: 20, width: 200, height: 80 }, + hittable: true, + }, + ]).nodes; + + assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { + kind: 'resolved', + point: { x: 110, y: 60 }, + strategy: 'center', + }); +}); + +test('point selection is independent of node array order', () => { + const nodes = blueskyPostNodes(); + const shuffled = [nodes[0]!, nodes[1]!, ...nodes.slice(2).reverse()]; + + assert.deepEqual( + resolveInteractionTouchPoint(shuffled, shuffled[1]!), + resolveInteractionTouchPoint(nodes, nodes[1]!), + ); +}); + +test('keeps a parent-owned point inside the supplied viewport bounds', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Partially clipped card', + rect: { x: -50, y: 20, width: 100, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Right child', + rect: { x: 30, y: 20, width: 20, height: 80 }, + hittable: true, + }, + ]).nodes; + + const resolution = resolveInteractionTouchPoint(nodes, nodes[0]!, { + bounds: [{ x: 0, y: 0, width: 100, height: 120 }], + }); + + assert.equal(resolution.kind, 'resolved'); + if (resolution.kind === 'resolved') assert.ok(resolution.point.x >= 0); +}); diff --git a/src/core/interaction-touch-point.ts b/src/core/interaction-touch-point.ts new file mode 100644 index 000000000..7963509ce --- /dev/null +++ b/src/core/interaction-touch-point.ts @@ -0,0 +1,200 @@ +import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + areRectsApproximatelyEqual, + normalizeRect, + resolveRectCenter, +} from '../utils/rect-center.ts'; +import { isSemanticTouchTarget } from './interaction-targeting.ts'; + +export type InteractionTouchPointResolution = + | { kind: 'resolved'; point: Point; strategy: 'center' | 'parent-owned' } + | { kind: 'blocked'; competitorRefs: string[] } + | { kind: 'invalid' }; + +const MIN_PARENT_OWNED_CLEARANCE = 12; + +type InteractiveDescendantRect = { ref: string; index: number; rect: Rect }; +type RankedPoint = { point: Point; distanceSquared: number; clearance: number }; + +export function resolveInteractionTouchPoint( + nodes: readonly SnapshotNode[], + node: SnapshotNode, + options: { bounds?: readonly Rect[] } = {}, +): InteractionTouchPointResolution { + const targetRect = normalizeRect(node.rect); + const center = resolveRectCenter(targetRect ?? undefined); + if (!targetRect || !center) return { kind: 'invalid' }; + + const competitors = collectInteractiveDescendantRects(nodes, node, targetRect); + if (competitors.length === 0) { + return { kind: 'resolved', point: center, strategy: 'center' }; + } + + const searchRect = (options.bounds ?? []).reduce( + (current, bound) => (current ? intersectRects(current, bound) : null), + targetRect, + ); + const point = searchRect + ? findNearestParentOwnedPoint(targetRect, searchRect, center, competitors) + : null; + if (!point) { + return { + kind: 'blocked', + competitorRefs: competitors.map((competitor) => competitor.ref), + }; + } + return { kind: 'resolved', point, strategy: 'parent-owned' }; +} + +function collectInteractiveDescendantRects( + nodes: readonly SnapshotNode[], + node: SnapshotNode, + targetRect: Rect, +): InteractiveDescendantRect[] { + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + return ( + nodes + .filter((candidate) => candidate.index !== node.index) + .filter((candidate) => isDescendantOf(candidate, node, byIndex)) + // Deliberately use semantic roles, not `hittable`: iOS commonly marks + // static text hittable, and treating that as a competing control would + // erase the parent-owned text region we need to preserve. + .filter(isSemanticTouchTarget) + .flatMap((candidate) => { + const rect = normalizeRect(candidate.rect); + if (!rect || areRectsApproximatelyEqual(rect, targetRect)) return []; + const clipped = intersectRects(rect, targetRect); + return clipped && clipped.width > 0 && clipped.height > 0 + ? [{ ref: candidate.ref, index: candidate.index, rect: clipped }] + : []; + }) + .sort((left, right) => left.index - right.index) + ); +} + +function isDescendantOf( + candidate: SnapshotNode, + ancestor: SnapshotNode, + byIndex: ReadonlyMap, +): boolean { + const visited = new Set(); + let current = candidate; + while (typeof current.parentIndex === 'number' && !visited.has(current.index)) { + visited.add(current.index); + if (current.parentIndex === ancestor.index) return true; + const parent = byIndex.get(current.parentIndex); + if (!parent) return false; + current = parent; + } + return false; +} + +function intersectRects(left: Rect, right: Rect): Rect | null { + const x = Math.max(left.x, right.x); + const y = Math.max(left.y, right.y); + const maxX = Math.min(left.x + left.width, right.x + right.width); + const maxY = Math.min(left.y + left.height, right.y + right.height); + if (maxX <= x || maxY <= y) return null; + return { x, y, width: maxX - x, height: maxY - y }; +} + +/** + * Finds the smallest deterministic move away from the historical center that + * leaves enough clearance from the parent edge and every independently + * interactive descendant. Candidate axes come from every geometry boundary, + * so any usable free region contributes at least one point without a pixel + * scan or device round trip. + */ +function findNearestParentOwnedPoint( + targetRect: Rect, + searchRect: Rect, + center: Point, + competitors: readonly InteractiveDescendantRect[], +): Point | null { + const xCandidates = candidateAxisCoordinates( + searchRect.x, + searchRect.width, + center.x, + competitors.flatMap(({ rect }) => [rect.x, rect.x + rect.width]), + ); + const yCandidates = candidateAxisCoordinates( + searchRect.y, + searchRect.height, + center.y, + competitors.flatMap(({ rect }) => [rect.y, rect.y + rect.height]), + ); + + const rankedPoints = xCandidates.flatMap((x) => + yCandidates.flatMap((y) => { + const ranked = rankParentOwnedPoint( + { x: Math.round(x), y: Math.round(y) }, + targetRect, + center, + competitors, + ); + return ranked ? [ranked] : []; + }), + ); + return rankedPoints.sort(compareRankedPoints)[0]?.point ?? null; +} + +function rankParentOwnedPoint( + point: Point, + targetRect: Rect, + center: Point, + competitors: readonly InteractiveDescendantRect[], +): RankedPoint | null { + const clearance = pointClearance(point, targetRect, competitors); + if (clearance < MIN_PARENT_OWNED_CLEARANCE) return null; + const distanceSquared = (point.x - center.x) ** 2 + (point.y - center.y) ** 2; + return { point, distanceSquared, clearance }; +} + +function compareRankedPoints(left: RankedPoint, right: RankedPoint): number { + return ( + left.distanceSquared - right.distanceSquared || + right.clearance - left.clearance || + left.point.y - right.point.y || + left.point.x - right.point.x + ); +} + +function candidateAxisCoordinates( + origin: number, + size: number, + center: number, + obstacleEdges: readonly number[], +): number[] { + const min = origin + MIN_PARENT_OWNED_CLEARANCE; + const max = origin + size - MIN_PARENT_OWNED_CLEARANCE; + if (min > max) return []; + const boundaries = [...new Set([origin, origin + size, ...obstacleEdges])].sort((a, b) => a - b); + const candidates = new Set([center, min, max]); + for (const edge of obstacleEdges) { + candidates.add(edge - MIN_PARENT_OWNED_CLEARANCE); + candidates.add(edge + MIN_PARENT_OWNED_CLEARANCE); + } + for (let index = 1; index < boundaries.length; index += 1) { + candidates.add((boundaries[index - 1]! + boundaries[index]!) / 2); + } + return [...candidates].filter((value) => value >= min && value <= max).sort((a, b) => a - b); +} + +function pointClearance( + point: Point, + targetRect: Rect, + competitors: readonly InteractiveDescendantRect[], +): number { + let clearance = Math.min( + point.x - targetRect.x, + targetRect.x + targetRect.width - point.x, + point.y - targetRect.y, + targetRect.y + targetRect.height - point.y, + ); + for (const { rect } of competitors) { + const dx = point.x < rect.x ? rect.x - point.x : Math.max(0, point.x - (rect.x + rect.width)); + const dy = point.y < rect.y ? rect.y - point.y : Math.max(0, point.y - (rect.y + rect.height)); + clearance = Math.min(clearance, Math.max(dx, dy)); + } + return clearance; +} From 57588cdf0ecd3ad0b2dbbacea32dbde45903da19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 09:15:09 +0200 Subject: [PATCH 2/9] docs: streamline no-skill CLI help --- .../help-conformance-topic-coverage.test.ts | 2 + src/__tests__/cli-help.test.ts | 12 +- .../__tests__/cli-help-overview.test.ts | 34 +++ .../parser/__tests__/cli-help-topics.test.ts | 193 ++++-------------- src/cli/parser/cli-help-overview.ts | 58 ++++++ src/cli/parser/cli-help.ts | 121 ++--------- test/integration/smoke-cli.test.ts | 6 +- 7 files changed, 155 insertions(+), 271 deletions(-) create mode 100644 src/cli/parser/__tests__/cli-help-overview.test.ts create mode 100644 src/cli/parser/cli-help-overview.ts diff --git a/scripts/__tests__/help-conformance-topic-coverage.test.ts b/scripts/__tests__/help-conformance-topic-coverage.test.ts index 01c1e07bb..7df2ae35b 100644 --- a/scripts/__tests__/help-conformance-topic-coverage.test.ts +++ b/scripts/__tests__/help-conformance-topic-coverage.test.ts @@ -10,6 +10,8 @@ import { helpTopicIds } from '../../src/cli/parser/cli-help.ts'; // topics are a visible decision instead of silent drift. const WAIVED_TOPICS: Record = { cdp: 'JS-heap forensics niche; add cases when heap-guidance regressions show up in practice.', + commands: + 'Derived command/configuration reference, not a planning loop; catalog completeness is structurally tested.', macos: 'macOS surface guidance is thin and stable; no observed planning regressions yet.', maestro: 'Compatibility reference, not a planning loop; conformance is oracle-tested instead.', 'physical-device': 'Needs device-specific setup guidance; no portable planning task defined yet.', diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index b1936aa44..a52bce798 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -200,22 +200,22 @@ test('help gestures prints the multi-touch topic and skips daemon dispatch', asy assert.match(result.stdout, /agent-device gesture transform 200 420 80 -40 2 35 700/); }); -test('help unknown command prints error plus global usage and skips daemon dispatch', async () => { +test('help unknown command prints error plus root decision card and skips daemon dispatch', async () => { const result = await runCliCapture(['help', 'not-a-command']); assert.equal(result.code, 1); assert.equal(result.calls.length, 0); assert.match(result.stderr, /Error \(INVALID_ARGS\): Unknown command: not-a-command/); - assert.match(result.stdout, /Commands:/); - assert.match(result.stdout, /Global Flags:/); - assert.match(result.stdout, /--config /); + assert.match(result.stdout, /All \d+ commands: agent-device help commands/); + assert.match(result.stdout, /When starting a task with a known app/); + assert.doesNotMatch(result.stdout, /Global Flags:/); }); -test('unknown command --help prints error plus global usage and skips daemon dispatch', async () => { +test('unknown command --help prints error plus root decision card and skips daemon dispatch', async () => { const result = await runCliCapture(['not-a-command', '--help']); assert.equal(result.code, 1); assert.equal(result.calls.length, 0); assert.match(result.stderr, /Error \(INVALID_ARGS\): Unknown command: not-a-command/); - assert.match(result.stdout, /Commands:/); + assert.match(result.stdout, /All \d+ commands: agent-device help commands/); }); test('runtime command is rejected before daemon dispatch', async () => { diff --git a/src/cli/parser/__tests__/cli-help-overview.test.ts b/src/cli/parser/__tests__/cli-help-overview.test.ts new file mode 100644 index 000000000..8f41ec6f6 --- /dev/null +++ b/src/cli/parser/__tests__/cli-help-overview.test.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { usage, usageForCommand } from '../args.ts'; + +test('root help is a bounded first-success decision card', async () => { + const help = await usage(); + + assert.ok( + Buffer.byteLength(help, 'utf8') <= 3000, + `root help must stay within the 3000-byte decision-card budget, was ${Buffer.byteLength(help, 'utf8')} bytes`, + ); + assert.match(help, /All \d+ commands: agent-device help commands/); + assert.match(help, /starting a task with a known app/); + assert.match(help, /agent-device open --foreground/); + assert.match(help, /scroll \[amount\] --settle; back --settle/); + assert.match(help, /Do not probe first with devices, apps, appstate, snapshot, or screenshot/); + assert.match(help, /Copy refs exactly: @e12, @e12~s4/); + assert.match(help, /Coordinates are last resort/); + assert.doesNotMatch(help, /^Commands:/m); + assert.doesNotMatch(help, /^Global Flags:/m); +}); + +test('help commands preserves the complete reference displaced from root help', async () => { + const help = await usageForCommand('commands'); + if (help === null) throw new Error('Expected commands help text'); + + assert.match(help, /^agent-device \S+ — commands/); + assert.match(help, /^Commands:/m); + assert.match(help, /install-from-source\s{2,}Install app builds from URLs or CI artifacts/); + assert.match(help, /^Global Flags:/m); + assert.match(help, /^Configuration:/m); + assert.match(help, /^Environment:/m); + assert.match(help, /^Examples:/m); +}); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index b55c74b2d..09c9d252d 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -1,10 +1,10 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { SELECTOR_KEY_NAMES } from '@agent-device/selectors'; import { usage, usageForCommand } from '../args.ts'; -test('usage includes concise top-level commands', async () => { - const usageText = await usage(); +test('commands topic includes concise command catalog entries', async () => { + const usageText = await usageForCommand('commands'); + if (usageText === null) throw new Error('Expected commands help text'); assert.match(usageText, /install-from-source\s{2,}Install app builds from URLs or CI artifacts/); assert.match(usageText, /prepare\s{2,}Pre-warm platform helpers before automation/); assert.match(usageText, /metro\s{2,}Prepare the dev server or reload apps/); @@ -37,11 +37,12 @@ test('gesture help documents selectors and pinned refs for both drag endpoints', assert.match(help, /drag /); }); -test('usage includes only global flags in the top-level global flags section', async () => { - const usageText = await usage(); +test('commands topic includes only global flags in its global flags section', async () => { + const usageText = await usageForCommand('commands'); + if (usageText === null) throw new Error('Expected commands help text'); const flagsSection = usageText.slice( usageText.indexOf('Global Flags:'), - usageText.indexOf('Agent Quickstart:'), + usageText.indexOf('Configuration:'), ); assert.match(flagsSection, /^Global Flags:/); assert.match(flagsSection, /--config /); @@ -72,154 +73,30 @@ test('usage includes only global flags in the top-level global flags section', a assert.doesNotMatch(flagsSection, /--metadata/); }); -test('usage includes agent workflows, config, environment, and examples footers', async () => { - const usageText = await usage(); - assert.match( - usageText, - /CLI to automate supported app, device, desktop, and web targets for AI agents/, - ); - assert.ok( - usageText.indexOf('Agent Workflows:') < usageText.indexOf('Commands:'), - 'Agent workflows should appear before the command list for agents that only read the top of help.', - ); - assert.ok( - usageText.indexOf('Agent Starting Point:') < usageText.indexOf('Agent Workflows:'), - 'The agent starting point should appear before topic selection.', - ); - assert.match(usageText, /Agent Starting Point:/); - assert.match(usageText, /Write full command lines starting with agent-device/); - assert.match(usageText, /Default app loop: agent-device open --foreground/); - assert.match( - usageText, - /Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close/, - ); - assert.match(usageText, /type never accepts --settle/); - assert.match(usageText, /explicit success confirmation is visible, stop/); - assert.match(usageText, /Follow structured command hints before choosing a recovery action/); - assert.match(usageText, /Refs\/selectors/); - assert.match( - usageText, - /Copy refs exactly, including @ and ~sN pins \(@e72, @e12~s4\); without @ it is not a ref/, - ); - assert.ok( - usageText.includes(`Selector keys are only: ${SELECTOR_KEY_NAMES.join(', ')}.`), - 'The first-screen selector vocabulary must match the parser source of truth.', - ); - assert.match(usageText, /placeholder, index, and key are not selector keys/); - assert.match(usageText, /A literal @ handle is a label/); - assert.match(usageText, /agent-device fill 'label="Query"' "text" --settle/); - assert.match(usageText, /key=Enter is not a supported target/); - assert.match(usageText, /Pick the help mode below/); - const firstThirtyLines = usageText.split('\n').slice(0, 30).join('\n'); - assert.match( - firstThirtyLines, - /agent-device help macos/, - 'Every workflow pointer must remain in the 30-line first-screen benchmark slice.', - ); - assert.match(usageText, /Agent Quickstart:/); - assert.match(usageText, /Planning output contract/); - assert.match( - usageText, - /Plain snapshot reads state; snapshot -i refreshes current interactive refs only/, - ); - assert.match(usageText, /agent-facing, token-efficient view for planning and targeting actions/); - assert.match(usageText, /Truncated text\/input preview: expand first with snapshot -s @e12/); - assert.match(usageText, /React Native apps: read help react-native/); - assert.match(usageText, /use fill --settle to replace a field value/); - assert.match(usageText, /Use type only to append after focusing a field with press/); - assert.match(usageText, /do not use fill ""/); - assert.match(usageText, /Implicit default sessions are scoped to the current worktree/); - assert.match(usageText, /if a prompt names a Session, include --session /); - assert.match(usageText, /Run mutating commands serially within one session/); - assert.match(usageText, /After mutation: refs are stale/); - assert.match(usageText, /use its selector directly; otherwise refresh with snapshot -i/); - assert.match(usageText, /fill \[text\]\s+Replace text in/); - assert.match(usageText, /type \s+Append text to the focused input/); - assert.match(usageText, /macOS context menus use click --button secondary/); - assert.match( - usageText, - /Remote lifecycle: use connect, then open, commands, close, and disconnect/, - ); - // Deep topic-specific detail (Metro/Expo recovery, Android IME capture, coordinate - // fallback verification, sparse/AX recovery, direct-proxy flags, back/system-back - // wording, the full web command sequence) moved out of the bare-help Agent - // Quickstart section and now lives only in the owning topic (help react-native, - // help workflow, help remote, help web) so `agent-device help` alone stays small. - // Those topics assert the same content in their own usageForCommand tests below. - assert.match(usageText, /TV\/D-pad targets: read help tv\. Web browser sessions: read help web/); - assert.match( - usageText, - /Routine QA loop with concrete command shapes: agent-device help manual-qa/, - ); - assert.match(usageText, /Session state contains request diagnostics and runner\.log/); - assert.match(usageText, /logs clear --restart\/mark\/path/); - assert.match(usageText, /network dump --include headers/); - assert.match(usageText, /Full operating guide: agent-device help workflow/); - assert.match(usageText, /Exploratory QA: agent-device help dogfood/); - assert.match(usageText, /Agent Workflows:/); - assert.match( - usageText, - /agent-device help manual-qa\s+Follow a manual test script with exact interactions and verification/, - ); - assert.match( - usageText, - /agent-device help dogfood\s+Explore an app and report issues with evidence/, - ); - assert.match( - usageText, - /agent-device help validate\s+Validate code changes, perf, visuals, logs, and cleanup/, - ); - assert.match( - usageText, - /agent-device help workflow\s+Full app automation reference for commands, refs, selectors, and waits/, - ); - assert.match( - usageText, - /agent-device help debugging\s+Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter/, - ); - assert.match( - usageText, - /agent-device help tv\s+Use when navigating Android TV, tvOS, or Vega VVD focus-first surfaces/, - ); - assert.match( - usageText, - /agent-device help react-devtools\s+Use when inspecting components, props\/state\/hooks, renders, or profiles/, - ); - assert.match( - usageText, - /agent-device help physical-device\s+Use when using a connected phone\/tablet or iOS signing setup/, - ); - assert.match( - usageText, - /agent-device help react-native\s+Use when the target app is React Native, Expo, or a dev client/, - ); - assert.match( - usageText, - /agent-device help web\s+Use when automating a browser through agent-device sessions/, - ); - assert.match(usageText, /Configuration:/); - assert.match( - usageText, - /Default config files: ~\/\.agent-device\/config\.json, \.\/agent-device\.json/, - ); - assert.match( - usageText, - /Use --config or AGENT_DEVICE_CONFIG for explicit connection\/provider defaults; project config cannot select endpoints or credentials\./, - ); - assert.match(usageText, /Environment:/); - assert.match(usageText, /AGENT_DEVICE_SESSION\s+Explicit session name/); - assert.match(usageText, /AGENT_DEVICE_PLATFORM\s+Default platform binding/); - assert.match(usageText, /AGENT_DEVICE_SESSION_LOCK\s+Bound-session conflict mode/); - assert.match(usageText, /AGENT_DEVICE_DAEMON_BASE_URL\s+Connect to remote daemon/); - assert.match(usageText, /Examples:/); - assert.match(usageText, /agent-device open Settings --platform ios/); - assert.match(usageText, /agent-device open https:\/\/example\.com --platform web/); - assert.match(usageText, /agent-device snapshot -i/); - assert.match(usageText, /agent-device fill @e3 "test@example\.com"/); - assert.match(usageText, /agent-device replay \.\/session\.ad/); - assert.match(usageText, /agent-device test \.\/suite --platform android/); -}); +test('root help routes detailed reference material to progressive topics', async () => { + const rootHelp = await usage(); + const commandsHelp = await usageForCommand('commands'); + const workflowHelp = await usageForCommand('workflow'); + if (commandsHelp === null || workflowHelp === null) { + throw new Error('Expected commands and workflow help text'); + } + + assert.match(rootHelp, /All \d+ commands: agent-device help commands/); + assert.match(rootHelp, /workflow\s+full refs, selectors, waits, recovery/); + assert.doesNotMatch(rootHelp, /^Configuration:/m); + assert.doesNotMatch(rootHelp, /^Environment:/m); + assert.match(commandsHelp, /^Configuration:/m); + assert.match(commandsHelp, /Default config files: ~\/\.agent-device\/config\.json/); + assert.match(commandsHelp, /^Environment:/m); + assert.match(commandsHelp, /AGENT_DEVICE_SESSION\s+Explicit session name/); + assert.match(commandsHelp, /^Examples:/m); + assert.match(commandsHelp, /agent-device open Settings --platform ios/); + + assert.match(workflowHelp, /Command shapes, refs, selectors, waits, recovery/); + assert.match(workflowHelp, /run serially within one session/i); + assert.match(workflowHelp, /Wait failure contract:/); +}); test('usageForCommand resolves Maestro compatibility help topic', async () => { const help = await usageForCommand('maestro'); if (help === null) throw new Error('Expected Maestro help text'); @@ -744,8 +621,9 @@ test('usageForCommand resolves react-native help topic', async () => { assert.match(help, /Report React render offenders separately/); }); -test('usage includes swipe and press series options', async () => { - const help = await usage(); +test('commands topic includes swipe and press series options', async () => { + const help = await usageForCommand('commands'); + if (help === null) throw new Error('Expected commands help text'); assert.match(help, /diff /); assert.match(help, /swipe /); assert.match(help, /settings \[area\] \[options\]/); @@ -753,8 +631,9 @@ test('usage includes swipe and press series options', async () => { assert.doesNotMatch(help, /--interval-ms/); }); -test('usage renders concise commands inline with descriptions', async () => { - const help = await usage(); +test('commands topic renders concise commands inline with descriptions', async () => { + const help = await usageForCommand('commands'); + if (help === null) throw new Error('Expected commands help text'); assert.match(help, /Commands:[\s\S]*\n boot\s{2,}Boot target device\/simulator/); assert.match(help, /Commands:[\s\S]*\n shutdown\s{2,}Shutdown target simulator\/emulator/); assert.match(help, / prepare\s{2,}Pre-warm platform helpers/); diff --git a/src/cli/parser/cli-help-overview.ts b/src/cli/parser/cli-help-overview.ts new file mode 100644 index 000000000..4d7025913 --- /dev/null +++ b/src/cli/parser/cli-help-overview.ts @@ -0,0 +1,58 @@ +import { SELECTOR_KEY_NAMES } from '@agent-device/selectors'; +import { listCliCommandNames } from '../../command-catalog.ts'; + +/** + * The root help is the no-skill agent's decision card, not the command reference. + * Keep situational detail in `help workflow` and the derived catalog in `help commands`. + */ +export function renderCliHelpOverview(): string { + return `agent-device [args] [--json] + +Automates iOS, Android, macOS, TV, and web apps for AI agents. +All ${listCliCommandNames().length} commands: agent-device help commands + +Start: + When starting a task with a known app, first run: + agent-device open --foreground + It starts the session and returns the initial interactive snapshot with @refs. + Do not probe first with devices, apps, appstate, snapshot, or screenshot. + Unknown app id: devices, then apps, then open . Never invent ids. + Resuming an existing session: continue from its current state; do not reopen it. + +Loop: + press|click|fill|longpress ... --settle + scroll [amount] --settle; back --settle + acts, waits for quiet, and prints the UI diff. Continue from that diff. + Run snapshot -i only when the diff lacks the next target or did not settle. + Verify a named expectation with the diff, wait text "...", wait , + is, get, or find. A bare screenshot is not verification. + End with: agent-device close + +Targets: + Copy refs exactly: @e12, @e12~s4. Keep @ and any ~sN pin; refs go stale + after mutations. A literal @handle is label="@handle", not a bare ref. + Prefer refs, then id/label/role selectors. Selector keys: ${SELECTOR_KEY_NAMES.join(' ')}. + Coordinates are last resort: only after snapshot -i shows no semantic target. + Then screenshot, press , and re-snapshot on the changed screen. + +Rules: + --settle is only for press/click/fill/longpress/scroll/back; never open, + snapshot, or close. type never accepts --settle. + fill --settle replaces; type appends after focus. + Late network/debounce result: wait text "Expected", not snapshot polling. + Output full agent-device commands; no pipes, grep, jq, or pseudo-commands. + Stop when the requested end state is visible. Mutations run serially. + +More commands (exact shapes: agent-device help ): + open install devices apps boot close app and device lifecycle + screenshot record logs network perf trace evidence and diagnostics + replay test batch session scripted flows + alert keyboard clipboard settings gesture system and input + +Guides (agent-device help ): + workflow full refs, selectors, waits, recovery, and platform limits + manual-qa / dogfood / validate / debugging / scripting / gestures + react-native / react-devtools / cdp / tv / web / macos / remote + physical-device / ios-system-ui / maestro +`; +} diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index a20f7478b..9bad8fded 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -16,100 +16,7 @@ import { } from '../../cli-schema/command-schema.ts'; import { buildCommandUsage } from '../../cli-schema/usage.ts'; import { readVersion } from '../../utils/version.ts'; - -const AGENT_WORKFLOWS = [ - { - label: 'agent-device help manual-qa', - description: 'Follow a manual test script with exact interactions and verification', - }, - { - label: 'agent-device help dogfood', - description: 'Explore an app and report issues with evidence', - }, - { - label: 'agent-device help validate', - description: 'Validate code changes, perf, visuals, logs, and cleanup', - }, - { - label: 'agent-device help workflow', - description: 'Full app automation reference for commands, refs, selectors, and waits', - }, - { - label: 'agent-device help debugging', - description: - 'Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter', - }, - { - label: 'agent-device help tv', - description: 'Use when navigating Android TV, tvOS, or Vega VVD focus-first surfaces', - }, - { - label: 'agent-device help react-native', - description: 'Use when the target app is React Native, Expo, or a dev client', - }, - { - label: 'agent-device help react-devtools', - description: 'Use when inspecting components, props/state/hooks, renders, or profiles', - }, - { - label: 'agent-device help cdp', - description: 'Use when investigating JS heap growth, heap snapshots, or retainers', - }, - { - label: 'agent-device help physical-device', - description: 'Use when using a connected phone/tablet or iOS signing setup', - }, - { - label: 'agent-device help remote', - description: 'Use when working through cloud config, tenants, leases, or local tunnels', - }, - { - label: 'agent-device help web', - description: 'Use when automating a browser through agent-device sessions', - }, - { - label: 'agent-device help macos', - description: 'Use when targeting desktop, frontmost app, or menu bar surfaces', - }, -] as const; - -const AGENT_START_LINES = [ - 'Write full command lines starting with agent-device; do not output pseudo commands, helper prose, pipes, grep, jq, or hidden stderr.', - // The explicit loop and closed target grammar are conformance-gated below; - // the foreground-open form was separately validated by the 2026-08-08 - // controlled app-driving benchmark recorded in PR #1693. - 'Default app loop: agent-device open --foreground -> mutate a current target from its initial snapshot with --settle -> continue from that settled diff -> agent-device close.', - 'Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', - 'Follow structured command hints before choosing a recovery action.', - 'Refs/selectors: @e12, label="Query". Copy refs exactly, including @ and ~sN pins (@e72, @e12~s4); without @ it is not a ref.', - 'Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.', - 'A literal @ handle is a label such as label="@account.example", never a bare @ref.', - 'Boundary shapes: agent-device fill \'label="Query"\' "text" --settle; agent-device press \'role=button label="Submit"\' --settle.', - 'fill takes a target then text. press targets an accessible element; key=Enter is not a supported target.', - 'Pick the help mode below when the task is manual QA, dogfooding, engineering validation, or debugging.', -] as const; - -const AGENT_QUICKSTART_LINES = [ - 'Planning output contract: when asked to plan commands, output command lines only: no prose, numbering, Markdown fences, pipes, or shell helpers.', - 'If you did not use --settle, verify a mutation with diff snapshot (or diff snapshot -i), not a full snapshot: it prints only the added/removed/changed lines since the last snapshot in this session.', - 'Network-backed or debounced results may arrive after the --settle quiet window; follow the settled action with wait text "Expected result" or wait instead of polling full snapshots.', - 'Pin a raw CLI ref to the response that minted it with ~s (n = refsGeneration): press @e12~s4. On iOS, stale refs are rejected for mutations; refresh refs or use a stable selector.', - 'Plain snapshot reads state; snapshot -i refreshes current interactive refs only.', - 'Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions.', - 'Read-only visible/state question: use snapshot/get/is/find; use snapshot -i only when refs are needed.', - 'Truncated text/input preview: expand first with snapshot -s @e12, not get text.', - 'React Native apps: read help react-native.', - 'Text fields: use fill --settle to replace a field value. Use type only to append after focusing a field with press.', - 'Clearing text: do not use fill ""; use a visible clear/reset control or report that clearing is unsupported.', - 'Implicit default sessions are scoped to the current worktree; if a prompt names a Session, include --session on every command in that flow.', - 'Run mutating commands serially within one session; parallelize only read-only commands or separate sessions/devices.', - 'After mutation: refs are stale. If the next target is known, use its selector directly; otherwise refresh with snapshot -i, scoped with -s when a stable container is known. Use press/click for taps.', - 'macOS context menus use click --button secondary, then snapshot -i. Longpress is for mobile hold gestures, not macOS secondary-click menus.', - 'Remote lifecycle: use connect, then open, commands, close, and disconnect. Read help remote for proxy, cloud, and device-cloud provider flows.', - 'TV/D-pad targets: read help tv. Web browser sessions: read help web.', - 'Debug evidence: Session state contains request diagnostics and runner.log; use logs clear --restart/mark/path, trace, and network dump --include headers for app evidence.', - 'Routine QA loop with concrete command shapes: agent-device help manual-qa. Full operating guide: agent-device help workflow. Exploratory QA: agent-device help dogfood.', -] as const; +import { renderCliHelpOverview } from './cli-help-overview.ts'; const CONFIGURATION_LINES = [ 'Default config files: ~/.agent-device/config.json, ./agent-device.json (project-safe defaults only).', @@ -154,6 +61,10 @@ const WAIT_FAILURE_CONTRACT = `Wait failure contract: `; const HELP_TOPICS = { + commands: { + summary: 'Full command catalog, global flags, configuration, and environment', + body: 'agent-device help commands', + }, 'manual-qa': { summary: 'Follow manual test scripts with exact interactions and verification', body: `agent-device help manual-qa @@ -1120,9 +1031,13 @@ function buildCommandListUsage(commandName: string, schema: CommandSchema): stri } function renderUsageText(): string { - const header = `agent-device [args] [--json] + return renderCliHelpOverview(); +} -CLI to automate supported app, device, desktop, and web targets for AI agents. +function renderFullCommandReferenceText(): string { + const header = `agent-device help commands + +Full command catalog. Use agent-device help for exact flags and behavior. `; const commands = listCliCommandNames().map((name) => { @@ -1137,24 +1052,15 @@ CLI to automate supported app, device, desktop, and web targets for AI agents. const helpFlags = listHelpFlags(GLOBAL_FLAG_KEYS); const flagsSection = renderFlagSection('Global Flags:', helpFlags); - const startSection = renderTextSection('Agent Starting Point:', AGENT_START_LINES); - const quickstartSection = renderTextSection('Agent Quickstart:', AGENT_QUICKSTART_LINES); - const workflowsSection = renderAlignedSection('Agent Workflows:', AGENT_WORKFLOWS); const configSection = renderTextSection('Configuration:', CONFIGURATION_LINES); const environmentSection = renderAlignedSection('Environment:', ENVIRONMENT_LINES); const examplesSection = renderTextSection('Examples:', EXAMPLE_LINES); return `${header} -${startSection} - -${workflowsSection} - ${commandLines} ${flagsSection} -${quickstartSection} - ${configSection} ${environmentSection} @@ -1280,7 +1186,8 @@ export function helpTopicIds(): string[] { function buildHelpTopicUsageText(topicName: string): string | null { const topic = HELP_TOPICS[topicName as keyof typeof HELP_TOPICS]; if (!topic) return null; - return `${withVersionHeader(topicName, topic.body)} + const body = topicName === 'commands' ? renderFullCommandReferenceText() : topic.body; + return `${withVersionHeader(topicName, body)} Related: agent-device help command-specific flags @@ -1290,7 +1197,7 @@ Related: } // Every topic body's first line is authored as `agent-device help `. Swapping in the -// installed version here (instead of hand-editing 17 topic strings) gives the skill router a +// installed version here (instead of hand-editing every topic string) gives the skill router a // single, reliable header to read the CLI version from without a separate --version call: a // missing/old header on `help workflow` means an old CLI that predates this format. function withVersionHeader(topicId: string, body: string): string { diff --git a/test/integration/smoke-cli.test.ts b/test/integration/smoke-cli.test.ts index 7a068b442..2ab1a7d18 100644 --- a/test/integration/smoke-cli.test.ts +++ b/test/integration/smoke-cli.test.ts @@ -15,7 +15,11 @@ test('cli --help returns usage', () => { const result = runCli(['--help']); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /agent-device/i); - assert.match(result.stdout, /reinstall /i); + assert.match(result.stdout, /agent-device help commands/i); + + const commands = runCli(['help', 'commands']); + assert.equal(commands.status, 0, commands.stderr); + assert.match(commands.stdout, /reinstall /i); }); test('cli --version prints semver and exits 0', () => { From 3abe2b1fe22af135e4e89cfc474e062472f26940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 13:51:42 +0200 Subject: [PATCH 3/9] perf: recover faster from sparse iOS trees --- .../RunnerTests+CommandExecution.swift | 16 ++++- .../RunnerTests+SnapshotCapturePlan.swift | 59 ++++++++++++++++--- ...RunnerTests+SynthesizedGesturePolicy.swift | 9 +++ .../__tests__/help-conformance-bench.test.ts | 19 +++--- skills/agent-device/SKILL.md | 39 ++---------- src/cli/parser/cli-help-overview.ts | 3 +- .../interaction/runtime/settle.test.ts | 28 +++++++++ src/commands/interaction/runtime/settle.ts | 2 +- src/snapshot/snapshot-quality.ts | 2 +- src/utils/__tests__/snapshot-quality.test.ts | 16 +++++ 10 files changed, 139 insertions(+), 54 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index e99855663..991487d9a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1798,7 +1798,21 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found")) } if let x = command.x, let y = command.y { - let textInput = textInputAt(app: activeApp, x: x, y: y) + let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized( + bundleId: currentBundleId + ) + let textInput: XCUIElement? + if shouldProbeCoordinateTapTextInput( + xCTestChannelPenalized: xCTestChannelPenalized + ) { + textInput = textInputAt(app: activeApp, x: x, y: y) + } else { + textInput = nil + NSLog( + "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@", + currentBundleId ?? "" + ) + } var fallback: GestureFallback? if command.synthesized == true { let policyKind = SynthesizedGesturePolicyKind.coordinateTap diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 490612e2a..ad22ddc66 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -500,7 +500,7 @@ extension RunnerTests { return ("snapshot returned no nodes", "no-nodes") } if isSparseApplicationWindowTree(nodes) { - return ("snapshot returned only structural application/window nodes", "sparse-tree") + return ("snapshot returned no semantic controls or content", "sparse-tree") } if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold { return ("snapshot was cut off by its budget with almost nothing collected", "budget") @@ -531,19 +531,36 @@ extension RunnerTests { static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { guard !nodes.isEmpty else { return false } + let rootRects = nodes.compactMap { node in + node.type == "Application" || node.type == "Window" ? node.rect : nil + } return nodes.allSatisfy { node in // Application/Window labels are just the app/window name, and full-screen roots // compute as hittable; neither says anything about tree health. let isRootContainer = node.type == "Application" || node.type == "Window" - let hasContent = (!isRootContainer && node.label?.isEmpty == false) - || node.identifier?.isEmpty == false - || node.value?.isEmpty == false - return !hasContent - && (isRootContainer || !node.hittable) - && Self.structuralOnlyNodeTypes.contains(node.type) + guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false } + guard !isRootContainer else { return true } + + let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in + rootRect.x == node.rect.x && rootRect.y == node.rect.y + && rootRect.width == node.rect.width && rootRect.height == node.rect.height + } + let hasAddressableIdentifier = node.identifier?.isEmpty == false && !isFullScreenContainer + return !Self.isSemanticSnapshotText(node.label) + && !Self.isSemanticSnapshotText(node.value) + && !hasAddressableIdentifier } } + /// Private AX can stringify an unserializable accessibility label as the JavaScript object + /// placeholder. It is transport residue, not UI content, and must not make a shell-only tree + /// look healthy. + static func isSemanticSnapshotText(_ text: String?) -> Bool { + guard let text else { return false } + let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) + return !normalized.isEmpty && normalized.lowercased() != "[object object]" + } + /// A leaf whose label joins many short segments is a container marked as an accessibility /// element: the platform folds every descendant into one merged node. Nothing below it can /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it. @@ -686,6 +703,18 @@ extension RunnerTests { let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) let window = planTestNode(index: 1, type: "Window", parentIndex: 0) let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + let shell = planTestNode( + index: 1, + type: "Other", + identifier: "appShell", + parentIndex: 0 + ) + let serializationPlaceholder = planTestNode( + index: 2, + type: "Other", + label: "[object Object]", + parentIndex: 1 + ) // Labeled, hittable root over a bare window is still sparse. XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) @@ -693,6 +722,22 @@ extension RunnerTests { XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) // The same tiny tree from a completed sweep is a legitimately minimal screen. XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Container metadata plus a stringified serialization placeholder is not readable UI. + XCTAssertNotNil( + Self.sparsePayloadReason( + DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) + ) + ) + let actionableShell = planTestNode( + index: 1, + type: "Other", + identifier: "checkout", + hittable: true, + parentIndex: 0 + ) + XCTAssertNil( + Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) + ) // Empty payloads are degraded. XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 379cc000e..90a7f4c6e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -101,6 +101,10 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz } } +func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { + !xCTestChannelPenalized +} + func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool { steps.contains { step in step.synthesized == true && step.kind == "tap" @@ -217,5 +221,10 @@ extension RunnerTests { ) } + func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { + XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) + XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) + } + } #endif diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts index 0f13ef9ae..454244442 100644 --- a/scripts/__tests__/help-conformance-bench.test.ts +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -523,23 +523,24 @@ test('foreground attach grammar accepts both auto-discovery and an explicit know test('compact skill starts a known-app task with foreground open and an initial snapshot', async () => { const skill = await readFile(AGENT_DEVICE_SKILL, 'utf8'); - const startIndex = skill.indexOf('For an ordinary app-driving task'); - const endIndex = skill.indexOf('Default loop:'); - assert.notEqual(startIndex, -1); - assert.ok(endIndex > startIndex); - const ordinaryStart = skill.slice(startIndex, endIndex); - const openingCommands = ordinaryStart + const openingCommands = skill .split('\n') .map((line) => line.trim()) .filter((line) => line.startsWith('agent-device open ')); + assert.ok( + skill.split('\n').length <= 30, + 'the complete skill must fit the reliable first-30-line read', + ); assert.deepEqual(openingCommands, ['agent-device open --foreground']); - assert.match(ordinaryStart, /returns the initial interactive snapshot in the same call/); + assert.match(skill, /returns the initial interactive snapshot with `@refs`/); assert.match( - ordinaryStart, - /copy refs? byte-for-byte.*leading `@`.*part of the ref/i, + skill, + /copy refs? byte-for-byte.*keep the `@`/i, 'the always-loaded skill must preserve the @ prefix before topic help is available', ); + assert.match(skill, /sparse\/AX-unavailable.*refs and selectors are invalid/i); + assert.match(skill, /`agent-device screenshot`.*use coordinates.*`snapshot -i`/i); }); test('plan validator applies narrow grammar to permitted external commands', async () => { diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 48afeba68..e8aafb0a4 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -5,45 +5,16 @@ description: Automates Apple-platform apps (iOS, tvOS, macOS), Android devices, # agent-device -For an ordinary app-driving task with a known app or bundle id, start directly. Do not run `--help`, `--version`, `appstate`, or `snapshot` first: +For a normal app-driving task, start immediately. Do not probe first with `--help`, `--version`, `devices`, `appstate`, `snapshot`, or `screenshot`: ```bash agent-device open --foreground ``` -`open --foreground` keeps normal configured target selection and returns the initial interactive snapshot in the same call. Continue from its current refs. Prefer a concrete `@eN` ref from the current snapshot over a broad mutation selector. Copy refs byte-for-byte as printed: the leading `@` is part of the ref (`press @e72`, never `press e72`). When a response prints a pinned ref such as `@e12~s42` — including an ambiguity candidate or settled diff — copy the whole pinned ref exactly; a bare ref from a partial result is intentionally rejected. +That starts the session and returns the initial interactive snapshot with `@refs`. -Default loop: `open -> act with a current ref or specific selector -> verify -> close`. Use `--settle` on planned `press`, `click`, `fill`, `longpress`, `scroll`, or `back`; continue from the settled diff when it already proves the next state. If a mutation returns `AMBIGUOUS_MATCH`, retry one listed pinned candidate rather than adding `--first` or guessing by coordinates. +Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. -Read the smallest version-matched CLI guide only when the task is specialized, you are planning rather than operating, or a command/hint does not answer the question. This single read also replaces a separate `agent-device --version` check: +Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. -```bash -agent-device help manual-qa # scripted/manual QA, acceptance checks, checklist execution -agent-device help validate # code/runtime validation, stale build or daemon risk -agent-device help dogfood # exploratory app dogfooding and evidence collection -agent-device help workflow # fallback reference for general app driving or mixed tasks -``` - -That topic's first line is `agent-device — ` (for example `agent-device 0.21.0 — workflow`). Read the version from it instead of running `agent-device --version` separately. If the first line instead reads `agent-device help ` with no version — or the command fails, or the topic is unrecognized — the installed CLI predates this header and its current help topics/Vega OS routing. Stop and tell the user to upgrade the trusted install or approve an exact-version npm command. Do not run `npm install -g agent-device@latest` or `npx -y agent-device@latest` autonomously, and do not include version/upgrade commands in final plans. - -If `agent-device` fails outright but the user may have installed it globally, check the user's configured login/interactive shell and environment before using `npx`. Resolve the command the same way the user would from a normal terminal session, then run the absolute binary path if found. This may require inspecting shell startup behavior or package-manager/global bin locations; do not assume the Codex process `PATH` is the user's `PATH`. - -Read additional topics only when relevant: - -```bash -agent-device help debugging -agent-device help scripting # save-script, secret-safe fills, batch JSON, replay repair -agent-device help gestures # multi-touch gesture shapes and platform quirks -agent-device help react-native -agent-device help react-devtools -agent-device help cdp -agent-device help remote -agent-device help macos -agent-device help dogfood -agent-device help tv -agent-device help ios-system-ui # iOS SpringBoard, widgets, and system-UI surfaces -``` - -When target-specific help says capture or selectors are unsupported, use its control-only loop and the device display as visual truth. Let help own advanced command shapes and platform limits; use `help workflow` as the fallback reference when the compact loop is insufficient. - -For precise location workflows, read the installed `settings` help before planning so coordinate support and platform limits come from the active CLI version. +Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/src/cli/parser/cli-help-overview.ts b/src/cli/parser/cli-help-overview.ts index 4d7025913..67a4b777b 100644 --- a/src/cli/parser/cli-help-overview.ts +++ b/src/cli/parser/cli-help-overview.ts @@ -32,7 +32,8 @@ Targets: Copy refs exactly: @e12, @e12~s4. Keep @ and any ~sN pin; refs go stale after mutations. A literal @handle is label="@handle", not a bare ref. Prefer refs, then id/label/role selectors. Selector keys: ${SELECTOR_KEY_NAMES.join(' ')}. - Coordinates are last resort: only after snapshot -i shows no semantic target. + Coordinates are last resort: after snapshot -i shows no semantic target, or a + sparse/AX-unavailable warning says its refs and selectors are invalid. Then screenshot, press , and re-snapshot on the changed screen. Rules: diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts index ff8844b77..0fd4c925b 100644 --- a/src/commands/interaction/runtime/settle.test.ts +++ b/src/commands/interaction/runtime/settle.test.ts @@ -249,6 +249,34 @@ test('penalty-deferred private-ax captures do not reset the settle budget', asyn assert.equal(settle.hint, NEVER_SETTLED_HINT); }); +test('a settled sparse capture directs agents away from invalid refs and selectors', async () => { + const before = buttonSnapshot(); + const sparseAfter = welcomeSnapshot(); + sparseAfter.snapshotQuality = { + state: 'sparse', + backend: 'private-ax', + reasonCode: 'sparse-tree', + reason: 'snapshot returned no semantic controls or content', + }; + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : sparseAfter }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + assert.match(result.settle?.hint ?? '', /refs\/selectors are invalid/); + assert.match(result.settle?.hint ?? '', /screenshot.*coordinate taps/i); + assert.equal(result.settle?.diff, undefined); +}); + test('a broken settle capture never fails the action', async () => { const before = buttonSnapshot(); let captures = 0; diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 96ca9d454..cce35d75a 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -373,7 +373,7 @@ function resolveSettleHint( if (!outcome.settled) return { hint: NEVER_SETTLED_HINT }; if (!stored) { return { - hint: 'Settled on a sparse, unreadable tree — the diff is omitted. Use screenshot as visual truth before interacting further.', + hint: 'Settled on a sparse, unreadable tree — the diff is omitted and its refs/selectors are invalid. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', }; } // Same weak-readiness signal wait stable reports: a settled-but-tiny tree diff --git a/src/snapshot/snapshot-quality.ts b/src/snapshot/snapshot-quality.ts index f03c6fea4..912ded8db 100644 --- a/src/snapshot/snapshot-quality.ts +++ b/src/snapshot/snapshot-quality.ts @@ -162,7 +162,7 @@ function stateWarning(verdict: SnapshotQualityVerdict): string[] { return [ 'No snapshot backend could read this screen' + (verdict.reason ? ` (${verdict.reason})` : '') + - '. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', + '. Its refs and selectors are invalid. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', ]; } return []; diff --git a/src/utils/__tests__/snapshot-quality.test.ts b/src/utils/__tests__/snapshot-quality.test.ts index b99dc495d..d339f0089 100644 --- a/src/utils/__tests__/snapshot-quality.test.ts +++ b/src/utils/__tests__/snapshot-quality.test.ts @@ -98,3 +98,19 @@ test('renderSnapshotQualityWarnings keeps recovered snapshot copy concise', () = 'Some deeper accessibility nodes were omitted; this tree is capped at depth 56. Re-run with --depth 56 --scope only if you need deeper content.', ]); }); + +test('renderSnapshotQualityWarnings rejects semantic targets from a sparse tree', () => { + const warnings = renderSnapshotQualityWarnings( + { + state: 'sparse', + backend: 'private-ax', + reason: 'snapshot returned no semantic controls or content', + reasonCode: 'sparse-tree', + }, + [], + ); + + assert.deepEqual(warnings, [ + 'No snapshot backend could read this screen (snapshot returned no semantic controls or content). Its refs and selectors are invalid. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', + ]); +}); From ef1fe8da95bc6efebfbac95469d83babb8566520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 15:33:25 +0200 Subject: [PATCH 4/9] fix: preserve selector context for blocked parent taps --- .../contracts/src/interaction-guarantees.ts | 33 +++++++ .../contracts/interaction-guarantees.test.ts | 2 + src/__tests__/test-utils/index.ts | 1 + .../test-utils/property-arbitraries.ts | 94 ++++++++++++++++++- .../interaction/runtime/resolution.ts | 34 ++++--- src/core/interaction-touch-point.test.ts | 54 ++++++++++- .../interaction-contract/fixtures.ts | 43 +++++++++ .../runtime-ref.contract.test.ts | 24 +++++ .../runtime-ref.coverage.ts | 2 + .../runtime-selector.contract.test.ts | 27 ++++++ .../runtime-selector.coverage.ts | 2 + .../target-drag.contract.test.ts | 29 +++++- .../target-drag.coverage.ts | 2 + 13 files changed, 330 insertions(+), 17 deletions(-) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 3088ca36b..f76612cd1 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -34,6 +34,10 @@ export const INTERACTION_GUARANTEES = [ 'disambiguation', // Targets covered by another visible element are refused. 'occlusion', + // Element-targeted coordinate paths keep the resolved parent identity while + // choosing a point outside independently interactive descendants. A parent + // whose safe region is fully tiled fails closed instead of activating a child. + 'parentOwnedTouchPoint', // The tap point (rect center) must lie inside the root viewport; closed // drawers / off-viewport carousels are refused, not silently no-op tapped. 'offscreen', @@ -149,6 +153,10 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { kind: 'runtime', via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked', }, + parentOwnedTouchPoint: { + kind: 'runtime', + via: 'src/core/interaction-touch-point.ts#resolveInteractionTouchPoint', + }, // #1542: the base decision is isNodeVisibleOnScreen (bulk accessibility // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end // enforcement point — on iOS (local, non-provider sessions only) a would-be @@ -242,6 +250,10 @@ export const INTERACTION_DISPATCH_PATHS: Record { // updates it here with a linked issue. It is the diffable debt list // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ + 'direct-ios-selector/parentOwnedTouchPoint', 'direct-ios-selector/responseIdentity', 'maestro-non-hittable-fallback/errorTaxonomy', + 'maestro-non-hittable-fallback/parentOwnedTouchPoint', ]); }); diff --git a/src/__tests__/test-utils/index.ts b/src/__tests__/test-utils/index.ts index a3205b8c1..03ddec2af 100644 --- a/src/__tests__/test-utils/index.ts +++ b/src/__tests__/test-utils/index.ts @@ -41,6 +41,7 @@ export { distinctRectPairArb, formatRef, gestureInViewportArb, + interactionTouchPointScenarioArb, PROPERTY_RUNS, PROPERTY_RUNS_SMALL, rawSnapshotNodesArb, diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 53d957392..12354d082 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -4,7 +4,13 @@ import { SWIPE_PRESETS, type GesturePayload, } from '@agent-device/contracts/interaction'; -import type { Point, RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import { + attachRefs, + type Point, + type RawSnapshotNode, + type Rect, + type SnapshotNode, +} from '@agent-device/kernel/snapshot'; import fc from 'fast-check'; import { INTERNAL_COMMANDS, PUBLIC_COMMANDS } from '../../command-catalog.ts'; @@ -94,6 +100,92 @@ export const distinctRectPairArb: fc.Arbitrary<{ ancestor: Rect; target: Rect }> })), ); +export type InteractionTouchPointScenario = { + nodes: SnapshotNode[]; + permutedNodes: SnapshotNode[]; + target: SnapshotNode; + bound: Rect; + competitorRects: Rect[]; +}; + +const halfPixel = (value: number): number => value / 2; + +const touchPointTargetRectArb = fc + .record({ + x: fc.integer({ min: -200, max: 200 }), + y: fc.integer({ min: -200, max: 200 }), + width: fc.integer({ min: 48, max: 800 }), + height: fc.integer({ min: 48, max: 800 }), + }) + .map(({ x, y, width, height }) => ({ + x: halfPixel(x), + y: halfPixel(y), + width: halfPixel(width), + height: halfPixel(height), + })); + +function containedRectArb(container: Rect): fc.Arbitrary { + const widthSteps = Math.round(container.width * 2); + const heightSteps = Math.round(container.height * 2); + return fc + .record({ + width: fc.integer({ min: 2, max: widthSteps - 2 }), + height: fc.integer({ min: 2, max: heightSteps - 2 }), + }) + .chain(({ width, height }) => + fc + .record({ + x: fc.integer({ min: 0, max: widthSteps - width }), + y: fc.integer({ min: 0, max: heightSteps - height }), + }) + .map(({ x, y }) => ({ + x: container.x + halfPixel(x), + y: container.y + halfPixel(y), + width: halfPixel(width), + height: halfPixel(height), + })), + ); +} + +export const interactionTouchPointScenarioArb: fc.Arbitrary = + touchPointTargetRectArb.chain((targetRect) => + fc + .tuple( + fc.array(containedRectArb(targetRect), { minLength: 1, maxLength: 6 }), + containedRectArb(targetRect), + ) + .chain(([competitorRects, bound]) => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Generated parent', + rect: targetRect, + hittable: true, + }, + ...competitorRects.map((rect, offset) => ({ + index: offset + 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: `Generated child ${offset + 1}`, + rect, + hittable: true, + })), + ]); + return fc + .shuffledSubarray(nodes, { minLength: nodes.length, maxLength: nodes.length }) + .map((permutedNodes) => ({ + nodes, + permutedNodes, + target: nodes[0]!, + bound, + competitorRects, + })); + }), + ); + /** A point sampled from the viewport's own box, so most gestures are plannable. */ function pointInViewportArb(viewport: Rect): fc.Arbitrary { return fc.record({ diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 2d68504cb..1d4cb5b91 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -302,11 +302,11 @@ async function resolveRefInteractionTarget( target.ref, params.action, ); - const point = resolveNodeTouchPoint( - visibleNode, - nodes, - `Ref ${target.ref} not found or has invalid bounds`, - ); + const point = resolveNodeTouchPoint(visibleNode, nodes, { + invalidMessage: `Ref ${target.ref} not found or has invalid bounds`, + blockedTargetLabel: `Ref ${target.ref}`, + blockedTargetDetails: { ref: `@${normalizeRef(target.ref) ?? visibleNode.ref}` }, + }); return { kind: 'ref', point, @@ -379,11 +379,11 @@ async function resolveSelectorInteractionTarget( resolved.selector, params.action, ); - const point = resolveNodeTouchPoint( - visibleNode, - capture.snapshot.nodes, - `Selector ${resolved.selector} resolved to invalid bounds`, - ); + const point = resolveNodeTouchPoint(visibleNode, capture.snapshot.nodes, { + invalidMessage: `Selector ${resolved.selector} resolved to invalid bounds`, + blockedTargetLabel: `Selector ${selectorExpression}`, + blockedTargetDetails: { selector: selectorExpression }, + }); return { kind: 'selector', point, @@ -773,7 +773,11 @@ type ResolvedRefNode = { function resolveNodeTouchPoint( node: SnapshotNode, nodes: SnapshotState['nodes'], - message: string, + failure: { + invalidMessage: string; + blockedTargetLabel: string; + blockedTargetDetails: { ref: string } | { selector: string }; + }, ): Point { const effectiveViewport = resolveEffectiveViewportRect(node, nodes); const rootViewport = node.rect ? resolveViewportRect(nodes, node.rect) : null; @@ -781,13 +785,15 @@ function resolveNodeTouchPoint( bounds: [effectiveViewport, rootViewport].filter((rect) => rect !== null), }); if (resolution.kind === 'resolved') return resolution.point; - if (resolution.kind === 'invalid') throw new AppError('COMMAND_FAILED', message); + if (resolution.kind === 'invalid') { + throw new AppError('COMMAND_FAILED', failure.invalidMessage); + } throw new AppError( 'COMMAND_FAILED', - `Ref @${node.ref} has no parent-owned touch point outside its interactive descendants`, + `${failure.blockedTargetLabel} has no parent-owned touch point outside its interactive descendants`, { reason: 'covered_by_interactive_descendants', - ref: `@${node.ref}`, + ...failure.blockedTargetDetails, competitorRefs: resolution.competitorRefs.slice(0, 5).map((ref) => `@${ref}`), hint: 'Tap the specific interactive child you intend, or use a more specific selector. Every safely tappable region of the parent belongs to one of its child controls.', }, diff --git a/src/core/interaction-touch-point.test.ts b/src/core/interaction-touch-point.test.ts index 113831c63..1f14f340a 100644 --- a/src/core/interaction-touch-point.test.ts +++ b/src/core/interaction-touch-point.test.ts @@ -1,8 +1,23 @@ import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { test } from 'vitest'; -import { makeSnapshotState } from '../__tests__/test-utils/index.ts'; +import { + interactionTouchPointScenarioArb, + makeSnapshotState, + PROPERTY_RUNS, +} from '../__tests__/test-utils/index.ts'; import { resolveInteractionTouchPoint } from './interaction-touch-point.ts'; +function containsPoint(rect: Rect, point: Point): boolean { + return ( + point.x >= rect.x && + point.x <= rect.x + rect.width && + point.y >= rect.y && + point.y <= rect.y + rect.height + ); +} + function blueskyPostNodes() { return makeSnapshotState([ { @@ -202,3 +217,40 @@ test('keeps a parent-owned point inside the supplied viewport bounds', () => { assert.equal(resolution.kind, 'resolved'); if (resolution.kind === 'resolved') assert.ok(resolution.point.x >= 0); }); + +test('property: resolved parent-owned points stay inside the target and supplied bounds', () => { + fc.assert( + fc.property(interactionTouchPointScenarioArb, ({ nodes, target, bound }) => { + const resolution = resolveInteractionTouchPoint(nodes, target, { bounds: [bound] }); + if (resolution.kind !== 'resolved') return; + assert.ok(containsPoint(target.rect!, resolution.point)); + assert.ok(containsPoint(bound, resolution.point)); + }), + { numRuns: PROPERTY_RUNS }, + ); +}); + +test('property: resolved parent-owned points stay outside every competing descendant', () => { + fc.assert( + fc.property(interactionTouchPointScenarioArb, ({ nodes, target, bound, competitorRects }) => { + const resolution = resolveInteractionTouchPoint(nodes, target, { bounds: [bound] }); + if (resolution.kind !== 'resolved') return; + for (const competitor of competitorRects) { + assert.equal(containsPoint(competitor, resolution.point), false); + } + }), + { numRuns: PROPERTY_RUNS }, + ); +}); + +test('property: touch-point resolution is invariant to node-array permutations', () => { + fc.assert( + fc.property(interactionTouchPointScenarioArb, ({ nodes, permutedNodes, target, bound }) => { + assert.deepEqual( + resolveInteractionTouchPoint(permutedNodes, target, { bounds: [bound] }), + resolveInteractionTouchPoint(nodes, target, { bounds: [bound] }), + ); + }), + { numRuns: PROPERTY_RUNS }, + ); +}); diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index c84fb9665..f1be81cbc 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -167,6 +167,49 @@ export function coveredButtonSnapshot(): SnapshotState { ]); } +// A semantic parent whose entire surface belongs to independently interactive +// children. Parent-targeted coordinate paths must fail closed and preserve the +// caller's selector/ref context instead of silently activating either child. +export function fullyTiledParentSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Link', + label: 'Card', + rect: { x: 20, y: 100, width: 360, height: 200 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Top action', + rect: { x: 20, y: 100, width: 360, height: 100 }, + hittable: true, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Bottom action', + rect: { x: 20, y: 200, width: 360, height: 100 }, + hittable: true, + }, + ]); +} + // A visible list cell that iOS reports as non-hittable (#1037 shape): the // interaction must proceed but be annotated. export function nonHittableCellSnapshot(): SnapshotState { diff --git a/test/integration/interaction-contract/runtime-ref.contract.test.ts b/test/integration/interaction-contract/runtime-ref.contract.test.ts index a10b19d92..a4f32163f 100644 --- a/test/integration/interaction-contract/runtime-ref.contract.test.ts +++ b/test/integration/interaction-contract/runtime-ref.contract.test.ts @@ -11,6 +11,7 @@ import { closedDrawerSnapshot, continueButtonSnapshot, coveredButtonSnapshot, + fullyTiledParentSnapshot, nonHittableCellSnapshot, RUNNER_CONTINUE_NODES, settledWelcomeSnapshot, @@ -40,6 +41,29 @@ test(scenario('occlusion'), async () => { assert.deepEqual(taps, []); }); +test(scenario('parentOwnedTouchPoint'), async () => { + const taps: Point[] = []; + const device = createContractDevice(fullyTiledParentSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /Ref @e2 has no parent-owned touch point/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'covered_by_interactive_descendants'); + assert.equal(details?.ref, '@e2'); + assert.equal(details?.selector, undefined); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + test(scenario('offscreen'), async () => { const taps: Point[] = []; const device = createContractDevice(closedDrawerSnapshot(), { diff --git a/test/integration/interaction-contract/runtime-ref.coverage.ts b/test/integration/interaction-contract/runtime-ref.coverage.ts index af3f31843..95190cf89 100644 --- a/test/integration/interaction-contract/runtime-ref.coverage.ts +++ b/test/integration/interaction-contract/runtime-ref.coverage.ts @@ -2,6 +2,8 @@ import { definePathCoverage } from './coverage-manifest.ts'; export const RUNTIME_REF_COVERAGE = definePathCoverage('runtime-ref', { occlusion: 'runtime-ref occlusion: covered ref is refused', + parentOwnedTouchPoint: + 'runtime-ref parentOwnedTouchPoint: a fully tiled parent ref refuses before dispatch', offscreen: 'runtime-ref offscreen: closed-drawer ref refused with offscreen_ref', nonHittable: 'runtime-ref nonHittable: non-hittable ref is annotated but still tapped', responseConstruction: diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index 7b3047b51..e4ac5bbf1 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -14,6 +14,7 @@ import { drawerWithVisibleTwinSnapshot, equivalentWrapperChainSnapshot, edgeGrazingDrawerSnapshot, + fullyTiledParentSnapshot, nonHittableButtonSnapshot, RUNNER_CONTINUE_NODES, settledWelcomeSnapshot, @@ -127,6 +128,32 @@ test(scenario('occlusion'), async () => { assert.deepEqual(taps, []); }); +test(scenario('parentOwnedTouchPoint'), async () => { + const taps: Point[] = []; + const device = createContractDevice(fullyTiledParentSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.click(selector('label=Card'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: unknown }).code, 'COMMAND_FAILED'); + assert.match(error.message, /Selector label=Card has no parent-owned touch point/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'covered_by_interactive_descendants'); + assert.equal(details?.selector, 'label=Card'); + assert.equal(details?.ref, undefined); + assert.deepEqual(details?.competitorRefs, ['@e3', '@e4']); + assert.match(String(details?.hint), /more specific selector/); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + test(scenario('nonHittable'), async () => { const taps: Point[] = []; const device = createContractDevice(nonHittableButtonSnapshot(), { diff --git a/test/integration/interaction-contract/runtime-selector.coverage.ts b/test/integration/interaction-contract/runtime-selector.coverage.ts index 4c2aef32f..288326921 100644 --- a/test/integration/interaction-contract/runtime-selector.coverage.ts +++ b/test/integration/interaction-contract/runtime-selector.coverage.ts @@ -6,6 +6,8 @@ export const RUNTIME_SELECTOR_COVERAGE = definePathCoverage('runtime-selector', 'runtime-selector disambiguation: distinct subtrees reject without a geometry winner', ], occlusion: 'runtime-selector occlusion: covered button is refused', + parentOwnedTouchPoint: + 'runtime-selector parentOwnedTouchPoint: a fully tiled parent selector preserves selector context and refuses before dispatch', offscreen: [ 'runtime-selector offscreen: closed drawer refused with offscreen_selector', 'runtime-selector offscreen: edge-grazing container is still refused', diff --git a/test/integration/interaction-contract/target-drag.contract.test.ts b/test/integration/interaction-contract/target-drag.contract.test.ts index 94f35c9f0..53de8c001 100644 --- a/test/integration/interaction-contract/target-drag.contract.test.ts +++ b/test/integration/interaction-contract/target-drag.contract.test.ts @@ -6,7 +6,11 @@ import { scenarioName } from './coverage-manifest.ts'; import { assertRpcOk } from '../provider-scenarios/assertions.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from '../provider-scenarios/test-timeouts.ts'; import { TARGET_DRAG_COVERAGE } from './target-drag.coverage.ts'; -import { coveredButtonSnapshot, dragEndpointsSnapshot } from './fixtures.ts'; +import { + coveredButtonSnapshot, + dragEndpointsSnapshot, + fullyTiledParentSnapshot, +} from './fixtures.ts'; import { createContractDevice } from './runtime-harness.ts'; import { runnerGestureEntry, @@ -74,6 +78,29 @@ test(scenario('occlusion'), async () => { assert.equal(dispatches, 0); }); +test(scenario('parentOwnedTouchPoint'), async () => { + let dispatches = 0; + const tiled = fullyTiledParentSnapshot(); + const source = { ...dragEndpointsSnapshot().nodes[1]!, index: 4, parentIndex: 0 }; + const snapshot = makeSnapshotState([...tiled.nodes, source]); + const device = createContractDevice(snapshot, { + resolveGestureViewport: async () => ({ x: 0, y: 0, width: 400, height: 800 }), + performGesture: async () => { + dispatches += 1; + }, + }); + + await assert.rejects( + () => + device.interactions.gesture({ + session: 'default', + gesture: { intent: 'drag', source: 'id="source"', destination: 'label=Card' }, + }), + /Selector label=Card has no parent-owned touch point/, + ); + assert.equal(dispatches, 0); +}); + test(scenario('offscreen'), async () => { let dispatches = 0; const snapshot = dragEndpointsSnapshot(); diff --git a/test/integration/interaction-contract/target-drag.coverage.ts b/test/integration/interaction-contract/target-drag.coverage.ts index 320298c49..88a573a4f 100644 --- a/test/integration/interaction-contract/target-drag.coverage.ts +++ b/test/integration/interaction-contract/target-drag.coverage.ts @@ -6,6 +6,8 @@ const DUAL_ENDPOINT_SUCCESS = export const TARGET_DRAG_COVERAGE = definePathCoverage('target-drag', { disambiguation: DUAL_ENDPOINT_SUCCESS, occlusion: 'target-drag occlusion: a covered destination is refused before gesture dispatch', + parentOwnedTouchPoint: + 'target-drag parentOwnedTouchPoint: a fully tiled selector endpoint refuses before gesture dispatch', offscreen: 'target-drag offscreen: an off-screen source is refused before gesture dispatch', responseConstruction: 'target-drag responseConstruction: daemon response carries the canonical dual-target gesture shape', From 14fcde02a13bf3892b796d2bf932f7ef04f8e088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 16:11:33 +0200 Subject: [PATCH 5/9] fix: preserve coordinate text-entry focus --- .github/workflows/ios.yml | 3 + .../RunnerTests+CommandExecution.swift | 25 +++--- ...RunnerTests+CoordinateTextEntryTests.swift | 57 +++++++++++++ .../RunnerTests+LifecycleCacheTests.swift | 4 +- .../RunnerTests+TextEntry.swift | 81 ++++++++++++++----- .../RunnerTests+TextTyping.swift | 36 +++++---- 6 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 63e1510df..7e5084cb3 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -94,9 +94,11 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapPreservesBareTypeWitnessWhenSoftwareKeyboardIsHidden \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextCommitProgressWalksExpectedPrefixOnly \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ @@ -112,6 +114,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 991487d9a..87212b995 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1801,10 +1801,11 @@ extension RunnerTests { let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized( bundleId: currentBundleId ) - let textInput: XCUIElement? - if shouldProbeCoordinateTapTextInput( + let xCTestTextInputProbeSkipped = !shouldProbeCoordinateTapTextInput( xCTestChannelPenalized: xCTestChannelPenalized - ) { + ) + let textInput: XCUIElement? + if !xCTestTextInputProbeSkipped { textInput = textInputAt(app: activeApp, x: x, y: y) } else { textInput = nil @@ -1825,7 +1826,10 @@ extension RunnerTests { } if case .performed = outcome { logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) - rememberTextEntryTap(textInput) + rememberCoordinateTextEntryTap( + textInput, + xCTestProbeSkipped: xCTestTextInputProbeSkipped + ) return gestureResponse(message: "tapped", timing: timing) } logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) @@ -1837,7 +1841,10 @@ extension RunnerTests { clearRememberedTextEntryTap() return response } - rememberTextEntryTap(textInput) + rememberCoordinateTextEntryTap( + textInput, + xCTestProbeSkipped: xCTestTextInputProbeSkipped + ) return gestureResponse( message: "tapped", timing: timing, @@ -2506,11 +2513,11 @@ extension RunnerTests { return message == "scrolled" ? "scroll" : "drag" } - private func currentXCTestFailureCount() -> Int { + func currentXCTestFailureCount() -> Int { return testRun?.failureCount ?? 0 } - private func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool { + func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool { return currentXCTestFailureCount() > failureCountBefore } @@ -2532,7 +2539,7 @@ extension RunnerTests { ) } - private func runnerCommandFixture(_ json: String) throws -> Command { + func runnerCommandFixture(_ json: String) throws -> Command { try JSONDecoder().decode(Command.self, from: Data(json.utf8)) } @@ -2616,7 +2623,7 @@ extension RunnerTests { return XCUIApplication(bundleIdentifier: bundleId) } - private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { + func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift new file mode 100644 index 000000000..a6465ebdb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift @@ -0,0 +1,57 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + func testPenalizedCoordinateTapPreservesBareTypeWitnessWhenSoftwareKeyboardIsHidden() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let frame = textField.frame + XCTAssertFalse(frame.isEmpty) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner" + currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-penalized-coordinate-input","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + XCTAssertFalse( + isKeyboardVisible(app: app), + "the test must exercise a focused responder with the software keyboard hidden" + ) + + let failureCountBefore = currentXCTestFailureCount() + let typeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-after-penalized-coordinate","text":"coordinate-witness"}"# + ) + let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + + XCTAssertTrue(typeResponse.ok, String(describing: typeResponse.error)) + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertEqual(typeResponse.data?.textEntryRoute, "synthesized-first-responder") + XCTAssertEqual(String(describing: textField.value ?? ""), "coordinate-witness") + + let secondFailureCountBefore = currentXCTestFailureCount() + let secondTypeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-after-consumed-coordinate-witness","text":"-again"}"# + ) + let secondTypeResponse = executeTypeCommand(activeApp: app, command: secondTypeCommand) + + XCTAssertFalse(secondTypeResponse.ok) + XCTAssertEqual(secondTypeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(didRecordXCTestFailure(since: secondFailureCountBefore)) + XCTAssertEqual(String(describing: textField.value ?? ""), "coordinate-witness") + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift index 6d3216bce..f1738b762 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift @@ -50,7 +50,7 @@ extension RunnerTests { RunnerTargetActivationSpy.state = .runningForeground textEntryTapWitness = TextEntryTapWitness( - element: app, + target: .element(app), bundleId: "com.example.previous", processIdentifier: 41 ) @@ -118,7 +118,7 @@ extension RunnerTests { func testTextEntryTapWitnessIsBoundToTargetIdentity() { let witness = TextEntryTapWitness( - element: app, + target: .element(app), bundleId: "com.example.app", processIdentifier: 42 ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 2de0dc186..d671fd140 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -91,7 +91,12 @@ extension RunnerTests { } struct TextEntryTapWitness { - let element: XCUIElement + enum Target { + case element(XCUIElement) + case unresolvedFirstResponder + } + + let target: Target let bundleId: String? let processIdentifier: Int? @@ -151,12 +156,43 @@ extension RunnerTests { return } textEntryTapWitness = TextEntryTapWitness( - element: element, + target: .element(element), bundleId: currentBundleId, processIdentifier: currentAppProcessIdentifier ) } + func rememberCoordinateTextEntryTap( + _ element: XCUIElement?, + xCTestProbeSkipped: Bool + ) { + if let element { + rememberTextEntryTap(element) + return + } + guard xCTestProbeSkipped else { + clearRememberedTextEntryTap() + return + } + guard + let bundleId = currentBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty, + let processIdentifier = currentAppProcessIdentifier, + processIdentifier > 0 + else { + clearRememberedTextEntryTap() + return + } + // An unhealthy XCTest channel cannot safely resolve the element under a coordinate, but the + // successful tap still scopes the immediately following bare `type` to the current process. + // Keep that evidence one-shot and distinguish it from a failed text-input probe. + textEntryTapWitness = TextEntryTapWitness( + target: .unresolvedFirstResponder, + bundleId: bundleId, + processIdentifier: processIdentifier + ) + } + func clearRememberedTextEntryTap() { textEntryTapWitness = nil } @@ -174,23 +210,32 @@ extension RunnerTests { ) else { return nil } - let element = witness.element - // XCUIElement is query-backed rather than a stable node identity. A same-identifier field - // introduced by app-side navigation between tap and this immediate type can therefore - // re-resolve here; keep the witness one-shot and fail closed on every observable identity - // boundary instead of using frame equality, which would reject legitimate layout changes. - guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { - return nil + switch witness.target { + case .element(let element): + // XCUIElement is query-backed rather than a stable node identity. A same-identifier field + // introduced by app-side navigation between tap and this immediate type can therefore + // re-resolve here; keep the witness one-shot and fail closed on every observable identity + // boundary instead of using frame equality, which would reject legitimate layout changes. + guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { + return nil + } + // Keep the target scoped to the element that the preceding tap actually selected. Do not + // attach a refresh point: if that element disappeared, bare type must fail closed rather + // than rediscovering a different field or dispatching unscoped app.typeText. + return TextEntryTarget( + element: element, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ) + case .unresolvedFirstResponder: + return TextEntryTarget( + element: nil, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ) } - // Keep the target scoped to the element that the preceding tap actually selected. Do not - // attach a refresh point: if that element disappeared, bare type must fail closed rather - // than rediscovering a different field or dispatching unscoped app.typeText. - return TextEntryTarget( - element: element, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ) } func stabilizeTextInputBeforeTyping( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index fde9ad91c..528784e5b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -68,7 +68,10 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else { + guard initialTarget != nil + || shouldUseSynthesizedFirstResponderType + || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) + else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -106,27 +109,28 @@ extension RunnerTests { func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) { #if os(iOS) - if shouldUseSynthesizedFirstResponderType, + if shouldUseSynthesizedFirstResponderType { let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) - { textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true) switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) { case .continueTyping: - // No refresh point: like the tap-witness target itself, the commit wait must observe - // only the element the tap selected, never rediscover a different field. - awaitSynthesizedFirstResponderCommit( - app: app, - target: TextEntryTarget( - element: currentTarget, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ), - textBefore: textBefore, - typedText: value - ) + if let currentTarget { + // No refresh point: like the tap-witness target itself, the commit wait must observe + // only the element the tap selected, never rediscover a different field. + awaitSynthesizedFirstResponderCommit( + app: app, + target: TextEntryTarget( + element: currentTarget, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ), + textBefore: textBefore, + typedText: value + ) + } return (currentTarget, true, nil) case .fallback: return (nil, false, .synthesisUnavailable) From 218bb8ba99a184c653cbbb4e4c9ad78b22856902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 16:37:41 +0200 Subject: [PATCH 6/9] fix: preserve thin parent touch targets --- .../test-utils/property-arbitraries.ts | 9 ++- .../__tests__/cli-help-overview.test.ts | 20 +++++++ .../interaction/runtime/resolution.ts | 1 + src/core/interaction-touch-point.test.ts | 56 ++++++++++++++++++ src/core/interaction-touch-point.ts | 58 +++++++++++++++---- .../interaction-contract/fixtures.ts | 30 ++++------ .../runtime-selector.contract.test.ts | 3 +- 7 files changed, 143 insertions(+), 34 deletions(-) diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 12354d082..97a662ee1 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -109,13 +109,18 @@ export type InteractionTouchPointScenario = { }; const halfPixel = (value: number): number => value / 2; +const touchPointAxisStepsArb = fc.oneof( + fc.integer({ min: 4, max: 46 }), + fc.integer({ min: 48, max: 800 }), +); const touchPointTargetRectArb = fc .record({ x: fc.integer({ min: -200, max: 200 }), y: fc.integer({ min: -200, max: 200 }), - width: fc.integer({ min: 48, max: 800 }), - height: fc.integer({ min: 48, max: 800 }), + // Exercise dense desktop rows as well as standard mobile touch targets. + width: touchPointAxisStepsArb, + height: touchPointAxisStepsArb, }) .map(({ x, y, width, height }) => ({ x: halfPixel(x), diff --git a/src/cli/parser/__tests__/cli-help-overview.test.ts b/src/cli/parser/__tests__/cli-help-overview.test.ts index 8f41ec6f6..9aa99d4bf 100644 --- a/src/cli/parser/__tests__/cli-help-overview.test.ts +++ b/src/cli/parser/__tests__/cli-help-overview.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { listCliCommandNames } from '../../../command-catalog.ts'; import { usage, usageForCommand } from '../args.ts'; test('root help is a bounded first-success decision card', async () => { @@ -20,6 +21,25 @@ test('root help is a bounded first-success decision card', async () => { assert.doesNotMatch(help, /^Global Flags:/m); }); +test('root help overview names only commands from the derived catalog', async () => { + const help = await usage(); + const commandsSection = help.match( + /More commands \(exact shapes: agent-device help \):\n(?[\s\S]+?)\n\nGuides/, + )?.groups?.lines; + assert.ok(commandsSection, 'expected a More commands section'); + + const catalog = new Set(listCliCommandNames()); + const overviewCommands = commandsSection.split('\n').flatMap((line) => + line + .trim() + .split(/\s{2,}/, 1)[0]! + .split(/\s+/), + ); + for (const command of overviewCommands) { + assert.ok(catalog.has(command), `root help names unknown command: ${command}`); + } +}); + test('help commands preserves the complete reference displaced from root help', async () => { const help = await usageForCommand('commands'); if (help === null) throw new Error('Expected commands help text'); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 1d4cb5b91..82f0634c7 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -795,6 +795,7 @@ function resolveNodeTouchPoint( reason: 'covered_by_interactive_descendants', ...failure.blockedTargetDetails, competitorRefs: resolution.competitorRefs.slice(0, 5).map((ref) => `@${ref}`), + competitorCount: resolution.competitorRefs.length, hint: 'Tap the specific interactive child you intend, or use a more specific selector. Every safely tappable region of the parent belongs to one of its child controls.', }, ); diff --git a/src/core/interaction-touch-point.test.ts b/src/core/interaction-touch-point.test.ts index 1f14f340a..a2b57ee60 100644 --- a/src/core/interaction-touch-point.test.ts +++ b/src/core/interaction-touch-point.test.ts @@ -115,6 +115,34 @@ test('keeps the exact center when there are no competing interactive descendants }); }); +test('keeps a parent-owned center in a thin row when a child is safely off to the side', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Dense row', + rect: { x: 0, y: 0, width: 200, height: 22 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'More', + rect: { x: 178, y: 0, width: 22, height: 22 }, + hittable: true, + }, + ]).nodes; + + assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { + kind: 'resolved', + point: { x: 100, y: 11 }, + strategy: 'parent-owned', + }); +}); + test('fails closed when interactive descendants tile the parent', () => { const nodes = makeSnapshotState([ { @@ -218,6 +246,34 @@ test('keeps a parent-owned point inside the supplied viewport bounds', () => { if (resolution.kind === 'resolved') assert.ok(resolution.point.x >= 0); }); +test('keeps rounded parent-owned points inside half-pixel viewport bounds', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Link', + label: 'Thin clipped row', + rect: { x: 37.5, y: 0, width: 2, height: 398 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Leading action', + rect: { x: 37.5, y: 0, width: 1, height: 1 }, + hittable: true, + }, + ]).nodes; + const bound = { x: 37.5, y: 0, width: 1, height: 80 }; + + const resolution = resolveInteractionTouchPoint(nodes, nodes[0]!, { bounds: [bound] }); + + assert.equal(resolution.kind, 'resolved'); + if (resolution.kind === 'resolved') assert.ok(containsPoint(bound, resolution.point)); +}); + test('property: resolved parent-owned points stay inside the target and supplied bounds', () => { fc.assert( fc.property(interactionTouchPointScenarioArb, ({ nodes, target, bound }) => { diff --git a/src/core/interaction-touch-point.ts b/src/core/interaction-touch-point.ts index 7963509ce..2e7ce5cd2 100644 --- a/src/core/interaction-touch-point.ts +++ b/src/core/interaction-touch-point.ts @@ -11,10 +11,12 @@ export type InteractionTouchPointResolution = | { kind: 'blocked'; competitorRefs: string[] } | { kind: 'invalid' }; -const MIN_PARENT_OWNED_CLEARANCE = 12; +const MIN_COMPETITOR_CLEARANCE = 12; +const MAX_PARENT_EDGE_CLEARANCE = 12; type InteractiveDescendantRect = { ref: string; index: number; rect: Rect }; type RankedPoint = { point: Point; distanceSquared: number; clearance: number }; +type PointClearance = { parentEdge: number; competitor: number }; export function resolveInteractionTouchPoint( nodes: readonly SnapshotNode[], @@ -111,26 +113,32 @@ function findNearestParentOwnedPoint( center: Point, competitors: readonly InteractiveDescendantRect[], ): Point | null { + const parentEdgeClearance = resolveParentEdgeClearance(targetRect); const xCandidates = candidateAxisCoordinates( searchRect.x, searchRect.width, center.x, competitors.flatMap(({ rect }) => [rect.x, rect.x + rect.width]), + parentEdgeClearance, ); const yCandidates = candidateAxisCoordinates( searchRect.y, searchRect.height, center.y, competitors.flatMap(({ rect }) => [rect.y, rect.y + rect.height]), + parentEdgeClearance, ); const rankedPoints = xCandidates.flatMap((x) => yCandidates.flatMap((y) => { + const point = { x: Math.round(x), y: Math.round(y) }; + if (!containsPoint(searchRect, point)) return []; const ranked = rankParentOwnedPoint( - { x: Math.round(x), y: Math.round(y) }, + point, targetRect, center, competitors, + parentEdgeClearance, ); return ranked ? [ranked] : []; }), @@ -138,16 +146,35 @@ function findNearestParentOwnedPoint( return rankedPoints.sort(compareRankedPoints)[0]?.point ?? null; } +function containsPoint(rect: Rect, point: Point): boolean { + return ( + point.x >= rect.x && + point.x <= rect.x + rect.width && + point.y >= rect.y && + point.y <= rect.y + rect.height + ); +} + function rankParentOwnedPoint( point: Point, targetRect: Rect, center: Point, competitors: readonly InteractiveDescendantRect[], + parentEdgeClearance: number, ): RankedPoint | null { const clearance = pointClearance(point, targetRect, competitors); - if (clearance < MIN_PARENT_OWNED_CLEARANCE) return null; + if ( + clearance.parentEdge < parentEdgeClearance || + clearance.competitor < MIN_COMPETITOR_CLEARANCE + ) { + return null; + } const distanceSquared = (point.x - center.x) ** 2 + (point.y - center.y) ** 2; - return { point, distanceSquared, clearance }; + return { + point, + distanceSquared, + clearance: Math.min(clearance.parentEdge, clearance.competitor), + }; } function compareRankedPoints(left: RankedPoint, right: RankedPoint): number { @@ -164,15 +191,16 @@ function candidateAxisCoordinates( size: number, center: number, obstacleEdges: readonly number[], + parentEdgeClearance: number, ): number[] { - const min = origin + MIN_PARENT_OWNED_CLEARANCE; - const max = origin + size - MIN_PARENT_OWNED_CLEARANCE; + const min = origin + parentEdgeClearance; + const max = origin + size - parentEdgeClearance; if (min > max) return []; const boundaries = [...new Set([origin, origin + size, ...obstacleEdges])].sort((a, b) => a - b); const candidates = new Set([center, min, max]); for (const edge of obstacleEdges) { - candidates.add(edge - MIN_PARENT_OWNED_CLEARANCE); - candidates.add(edge + MIN_PARENT_OWNED_CLEARANCE); + candidates.add(edge - MIN_COMPETITOR_CLEARANCE); + candidates.add(edge + MIN_COMPETITOR_CLEARANCE); } for (let index = 1; index < boundaries.length; index += 1) { candidates.add((boundaries[index - 1]! + boundaries[index]!) / 2); @@ -184,17 +212,23 @@ function pointClearance( point: Point, targetRect: Rect, competitors: readonly InteractiveDescendantRect[], -): number { - let clearance = Math.min( +): PointClearance { + const parentEdge = Math.min( point.x - targetRect.x, targetRect.x + targetRect.width - point.x, point.y - targetRect.y, targetRect.y + targetRect.height - point.y, ); + let competitor = Number.POSITIVE_INFINITY; for (const { rect } of competitors) { const dx = point.x < rect.x ? rect.x - point.x : Math.max(0, point.x - (rect.x + rect.width)); const dy = point.y < rect.y ? rect.y - point.y : Math.max(0, point.y - (rect.y + rect.height)); - clearance = Math.min(clearance, Math.max(dx, dy)); + competitor = Math.min(competitor, Math.max(dx, dy)); } - return clearance; + return { parentEdge, competitor }; +} + +function resolveParentEdgeClearance(targetRect: Rect): number { + const shortAxis = Math.min(targetRect.width, targetRect.height); + return Math.max(0, Math.min(MAX_PARENT_EDGE_CLEARANCE, Math.floor(shortAxis / 2) - 1)); } diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index f1be81cbc..1d9c697cf 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -169,8 +169,17 @@ export function coveredButtonSnapshot(): SnapshotState { // A semantic parent whose entire surface belongs to independently interactive // children. Parent-targeted coordinate paths must fail closed and preserve the -// caller's selector/ref context instead of silently activating either child. +// caller's selector/ref context instead of silently activating any child. export function fullyTiledParentSnapshot(): SnapshotState { + const children = Array.from({ length: 10 }, (_, childIndex) => ({ + index: childIndex + 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: `Action ${childIndex + 1}`, + rect: { x: 20, y: 100 + childIndex * 20, width: 360, height: 20 }, + hittable: true, + })); return makeSnapshotState([ { index: 0, @@ -189,24 +198,7 @@ export function fullyTiledParentSnapshot(): SnapshotState { rect: { x: 20, y: 100, width: 360, height: 200 }, hittable: true, }, - { - index: 2, - depth: 2, - parentIndex: 1, - type: 'Button', - label: 'Top action', - rect: { x: 20, y: 100, width: 360, height: 100 }, - hittable: true, - }, - { - index: 3, - depth: 2, - parentIndex: 1, - type: 'Button', - label: 'Bottom action', - rect: { x: 20, y: 200, width: 360, height: 100 }, - hittable: true, - }, + ...children, ]); } diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index e4ac5bbf1..9cde3a1f3 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -146,7 +146,8 @@ test(scenario('parentOwnedTouchPoint'), async () => { assert.equal(details?.reason, 'covered_by_interactive_descendants'); assert.equal(details?.selector, 'label=Card'); assert.equal(details?.ref, undefined); - assert.deepEqual(details?.competitorRefs, ['@e3', '@e4']); + assert.deepEqual(details?.competitorRefs, ['@e3', '@e4', '@e5', '@e6', '@e7']); + assert.equal(details?.competitorCount, 10); assert.match(String(details?.hint), /more specific selector/); return true; }, From 30ee703c92f037a037ab886a768f22fd348b82c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 17:59:29 +0200 Subject: [PATCH 7/9] fix: fail closed for unscoped iOS typing --- .github/workflows/ios.yml | 2 +- .../RunnerTests+CommandExecution.swift | 11 +-- ...RunnerTests+CoordinateTextEntryTests.swift | 44 +++++----- .../RunnerTests+LifecycleCacheTests.swift | 4 +- .../RunnerTests+TextEntry.swift | 81 +++++-------------- .../RunnerTests+TextTyping.swift | 37 ++++----- .../contracts/src/interaction-guarantees.ts | 5 +- 7 files changed, 65 insertions(+), 119 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 7e5084cb3..9f86efd5b 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -94,7 +94,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapPreservesBareTypeWitnessWhenSoftwareKeyboardIsHidden \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextCommitProgressWalksExpectedPrefixOnly \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 87212b995..9386b0538 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1808,6 +1808,7 @@ extension RunnerTests { if !xCTestTextInputProbeSkipped { textInput = textInputAt(app: activeApp, x: x, y: y) } else { + // A process-scoped tap cannot authorize later typing without concrete element identity. textInput = nil NSLog( "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@", @@ -1826,10 +1827,7 @@ extension RunnerTests { } if case .performed = outcome { logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) - rememberCoordinateTextEntryTap( - textInput, - xCTestProbeSkipped: xCTestTextInputProbeSkipped - ) + rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) } logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) @@ -1841,10 +1839,7 @@ extension RunnerTests { clearRememberedTextEntryTap() return response } - rememberCoordinateTextEntryTap( - textInput, - xCTestProbeSkipped: xCTestTextInputProbeSkipped - ) + rememberTextEntryTap(textInput) return gestureResponse( message: "tapped", timing: timing, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift index a6465ebdb..ed2b0001c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift @@ -2,7 +2,7 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) - func testPenalizedCoordinateTapPreservesBareTypeWitnessWhenSoftwareKeyboardIsHidden() throws { + func testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType() throws { app.launchArguments = ["--agent-device-text-entry-regression"] app.launch() defer { @@ -14,44 +14,42 @@ extension RunnerTests { let textField = app.textFields["agent-device-hardware-keyboard-input"] XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let frame = textField.frame - XCTAssertFalse(frame.isEmpty) + let nonTextTarget = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) + let nonTextFrame = nonTextTarget.frame + XCTAssertFalse(nonTextFrame.isEmpty) currentApp = app currentBundleId = "com.callstack.agentdevice.runner" currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + + let focusCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-stale-responder-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let focusResponse = try executeOnMainPrepared(command: focusCommand, activeApp: app) + XCTAssertTrue(focusResponse.ok, String(describing: focusResponse.error)) + XCTAssertFalse( + isKeyboardVisible(app: app), + "the test must leave a responder focused while the software keyboard is hidden" + ) + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-penalized-coordinate-input","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true}"# + #"{"command":"tap","commandId":"tap-penalized-non-text-target","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY),"synthesized":true}"# ) let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - XCTAssertFalse( - isKeyboardVisible(app: app), - "the test must exercise a focused responder with the software keyboard hidden" - ) let failureCountBefore = currentXCTestFailureCount() let typeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-after-penalized-coordinate","text":"coordinate-witness"}"# + #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type"}"# ) let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) - XCTAssertTrue(typeResponse.ok, String(describing: typeResponse.error)) + XCTAssertFalse(typeResponse.ok) + XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) - XCTAssertEqual(typeResponse.data?.textEntryRoute, "synthesized-first-responder") - XCTAssertEqual(String(describing: textField.value ?? ""), "coordinate-witness") - - let secondFailureCountBefore = currentXCTestFailureCount() - let secondTypeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-after-consumed-coordinate-witness","text":"-again"}"# - ) - let secondTypeResponse = executeTypeCommand(activeApp: app, command: secondTypeCommand) - - XCTAssertFalse(secondTypeResponse.ok) - XCTAssertEqual(secondTypeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") - XCTAssertFalse(didRecordXCTestFailure(since: secondFailureCountBefore)) - XCTAssertEqual(String(describing: textField.value ?? ""), "coordinate-witness") + XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift index f1738b762..6d3216bce 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift @@ -50,7 +50,7 @@ extension RunnerTests { RunnerTargetActivationSpy.state = .runningForeground textEntryTapWitness = TextEntryTapWitness( - target: .element(app), + element: app, bundleId: "com.example.previous", processIdentifier: 41 ) @@ -118,7 +118,7 @@ extension RunnerTests { func testTextEntryTapWitnessIsBoundToTargetIdentity() { let witness = TextEntryTapWitness( - target: .element(app), + element: app, bundleId: "com.example.app", processIdentifier: 42 ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index d671fd140..2de0dc186 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -91,12 +91,7 @@ extension RunnerTests { } struct TextEntryTapWitness { - enum Target { - case element(XCUIElement) - case unresolvedFirstResponder - } - - let target: Target + let element: XCUIElement let bundleId: String? let processIdentifier: Int? @@ -156,43 +151,12 @@ extension RunnerTests { return } textEntryTapWitness = TextEntryTapWitness( - target: .element(element), + element: element, bundleId: currentBundleId, processIdentifier: currentAppProcessIdentifier ) } - func rememberCoordinateTextEntryTap( - _ element: XCUIElement?, - xCTestProbeSkipped: Bool - ) { - if let element { - rememberTextEntryTap(element) - return - } - guard xCTestProbeSkipped else { - clearRememberedTextEntryTap() - return - } - guard - let bundleId = currentBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), - !bundleId.isEmpty, - let processIdentifier = currentAppProcessIdentifier, - processIdentifier > 0 - else { - clearRememberedTextEntryTap() - return - } - // An unhealthy XCTest channel cannot safely resolve the element under a coordinate, but the - // successful tap still scopes the immediately following bare `type` to the current process. - // Keep that evidence one-shot and distinguish it from a failed text-input probe. - textEntryTapWitness = TextEntryTapWitness( - target: .unresolvedFirstResponder, - bundleId: bundleId, - processIdentifier: processIdentifier - ) - } - func clearRememberedTextEntryTap() { textEntryTapWitness = nil } @@ -210,32 +174,23 @@ extension RunnerTests { ) else { return nil } - switch witness.target { - case .element(let element): - // XCUIElement is query-backed rather than a stable node identity. A same-identifier field - // introduced by app-side navigation between tap and this immediate type can therefore - // re-resolve here; keep the witness one-shot and fail closed on every observable identity - // boundary instead of using frame equality, which would reject legitimate layout changes. - guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { - return nil - } - // Keep the target scoped to the element that the preceding tap actually selected. Do not - // attach a refresh point: if that element disappeared, bare type must fail closed rather - // than rediscovering a different field or dispatching unscoped app.typeText. - return TextEntryTarget( - element: element, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ) - case .unresolvedFirstResponder: - return TextEntryTarget( - element: nil, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ) + let element = witness.element + // XCUIElement is query-backed rather than a stable node identity. A same-identifier field + // introduced by app-side navigation between tap and this immediate type can therefore + // re-resolve here; keep the witness one-shot and fail closed on every observable identity + // boundary instead of using frame equality, which would reject legitimate layout changes. + guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { + return nil } + // Keep the target scoped to the element that the preceding tap actually selected. Do not + // attach a refresh point: if that element disappeared, bare type must fail closed rather + // than rediscovering a different field or dispatching unscoped app.typeText. + return TextEntryTarget( + element: element, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ) } func stabilizeTextInputBeforeTyping( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 528784e5b..98714982d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -68,10 +68,7 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil - || shouldUseSynthesizedFirstResponderType - || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) - else { + guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -110,27 +107,27 @@ extension RunnerTests { func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) { #if os(iOS) if shouldUseSynthesizedFirstResponderType { - let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) + guard let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) else { + return (nil, false, .notFocused) + } textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true) switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) { case .continueTyping: - if let currentTarget { - // No refresh point: like the tap-witness target itself, the commit wait must observe - // only the element the tap selected, never rediscover a different field. - awaitSynthesizedFirstResponderCommit( - app: app, - target: TextEntryTarget( - element: currentTarget, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ), - textBefore: textBefore, - typedText: value - ) - } + // No refresh point: like the tap-witness target itself, the commit wait must observe + // only the element the tap selected, never rediscover a different field. + awaitSynthesizedFirstResponderCommit( + app: app, + target: TextEntryTarget( + element: currentTarget, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ), + textBefore: textBefore, + typedText: value + ) return (currentTarget, true, nil) case .fallback: return (nil, false, .synthesisUnavailable) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index f76612cd1..38a968132 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -136,6 +136,7 @@ export type InteractionPathContract = { }; const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081'; +const PARENT_OWNED_TOUCH_POINT_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/1718'; // Every path shares the SAME cell by construction: response payloads have one // construction site (ADR 0011 Layer 2), and the hand-rolled-literal guard test @@ -309,7 +310,7 @@ export const INTERACTION_DISPATCH_PATHS: Record Date: Mon, 10 Aug 2026 19:11:07 +0200 Subject: [PATCH 8/9] test: isolate replay lock fixture --- src/daemon/__tests__/request-router-replay-scope.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/daemon/__tests__/request-router-replay-scope.test.ts b/src/daemon/__tests__/request-router-replay-scope.test.ts index d15081622..64e63662c 100644 --- a/src/daemon/__tests__/request-router-replay-scope.test.ts +++ b/src/daemon/__tests__/request-router-replay-scope.test.ts @@ -21,7 +21,11 @@ vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOrigi vi.mock('../../platforms/apple/core/apps.ts', async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, resolveIosApp: vi.fn(async () => 'com.example.app') }; + return { + ...actual, + resolveIosApp: vi.fn(async () => 'com.example.app'), + resolveIosSimulatorDeepLinkBundleId: vi.fn(async () => undefined), + }; }); vi.mock('../handlers/session-device-utils.ts', async (importOriginal) => { @@ -204,7 +208,7 @@ test('fresh replay retains a dynamically selected device through finalization', const externalResponse = handler({ token: 'test-token', session: 'external', - command: 'snapshot', + command: 'home', positionals: [], meta: { cwd: root, From df7002ca1823a2784e40caabab4b1d1743b46e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 10 Aug 2026 19:40:42 +0200 Subject: [PATCH 9/9] test: share node integration process --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index affa22731..eec255a26 100644 --- a/package.json +++ b/package.json @@ -173,7 +173,7 @@ "test:output-economy": "vitest run --project output-economy", "test:smoke:web": "pnpm build && node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-web-platform.test.ts", "test:smoke": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-*.test.ts", - "test:integration:node": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-concurrency=1 test/integration/*.test.ts", + "test:integration:node": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test --experimental-test-isolation=none test/integration/*.test.ts", "test:integration": "pnpm test:integration:node && pnpm test:integration:provider", "test:concurrency-torture": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/nightly/concurrency-torture.test.ts", "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator",