perf: speed up iOS agent recovery and streamline CLI guidance

.github/workflows/ios.yml

@@ -94,9 +94,11 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextCommitProgressWalksExpectedPrefixOnly \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \
@@ -112,6 +114,7 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \

CONTEXT.md

@@ -113,6 +113,10 @@ task touches:
   center coordinate when a frame is available. This keeps target selection semantic while avoiding
   `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains
   focus/remote-driven.
+- Parent-owned touch point: runtime ref/selector activation keeps the resolved parent identity but,
+  when its center belongs to an independently interactive descendant, moves the coordinate to the
+  nearest region with bounded clearance from those child controls. The exact center remains the
+  zero-cost default when no child competes; a fully tiled parent fails closed so a child must be named.
 - Guarantee cell: one (dispatch path, guarantee) entry in `packages/contracts/src/interaction-guarantees.ts`,
   classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error;
   honesty is gate-tested.

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -1798,7 +1798,23 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
       }
       if let x = command.x, let y = command.y {
-        let textInput = textInputAt(app: activeApp, x: x, y: y)
+        let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(
+          bundleId: currentBundleId
+        )
+        let xCTestTextInputProbeSkipped = !shouldProbeCoordinateTapTextInput(
+          xCTestChannelPenalized: xCTestChannelPenalized
+        )
+        let textInput: XCUIElement?
+        if !xCTestTextInputProbeSkipped {
+          textInput = textInputAt(app: activeApp, x: x, y: y)
+        } else {
+          // A process-scoped tap cannot authorize later typing without concrete element identity.
+          textInput = nil
+          NSLog(
+            "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@",
+            currentBundleId ?? ""
+          )
+        }
         var fallback: GestureFallback?
         if command.synthesized == true {
           let policyKind = SynthesizedGesturePolicyKind.coordinateTap
@@ -2492,11 +2508,11 @@ extension RunnerTests {
     return message == "scrolled" ? "scroll" : "drag"
   }
 
-  private func currentXCTestFailureCount() -> Int {
+  func currentXCTestFailureCount() -> Int {
     return testRun?.failureCount ?? 0
   }
 
-  private func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
+  func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
     return currentXCTestFailureCount() > failureCountBefore
   }
 
@@ -2518,7 +2534,7 @@ extension RunnerTests {
     )
   }
 
-  private func runnerCommandFixture(_ json: String) throws -> Command {
+  func runnerCommandFixture(_ json: String) throws -> Command {
     try JSONDecoder().decode(Command.self, from: Data(json.utf8))
   }
 
@@ -2602,7 +2618,7 @@ extension RunnerTests {
     return XCUIApplication(bundleIdentifier: bundleId)
   }
 
-  private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
+  func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
     guard let text = command.text else {
       return Response(ok: false, error: ErrorPayload(message: "type requires text"))
     }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift

@@ -0,0 +1,55 @@
+import XCTest
+
+extension RunnerTests {
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+  func testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType() throws {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      clearSnapshotXCTestChannelPenalty(reason: "test-cleanup")
+      invalidateCachedTarget(reason: "unit_test_cleanup")
+      app.terminate()
+    }
+    XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout))
+
+    let textField = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout))
+    let nonTextTarget = app.staticTexts["Agent Device Runner"]
+    XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout))
+    let nonTextFrame = nonTextTarget.frame
+    XCTAssertFalse(nonTextFrame.isEmpty)
+    currentApp = app
+    currentBundleId = "com.callstack.agentdevice.runner"
+    currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app))
+
+    let focusCommand = try runnerCommandFixture(
+      #"{"command":"tap","commandId":"tap-stale-responder-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"#
+    )
+    let focusResponse = try executeOnMainPrepared(command: focusCommand, activeApp: app)
+    XCTAssertTrue(focusResponse.ok, String(describing: focusResponse.error))
+    XCTAssertFalse(
+      isKeyboardVisible(app: app),
+      "the test must leave a responder focused while the software keyboard is hidden"
+    )
+
+    penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test")
+
+    let tapCommand = try runnerCommandFixture(
+      #"{"command":"tap","commandId":"tap-penalized-non-text-target","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY),"synthesized":true}"#
+    )
+    let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app)
+    XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error))
+
+    let failureCountBefore = currentXCTestFailureCount()
+    let typeCommand = try runnerCommandFixture(
+      #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type"}"#
+    )
+    let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand)
+
+    XCTAssertFalse(typeResponse.ok)
+    XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED")
+    XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
+    XCTAssertEqual(String(describing: textField.value ?? ""), "")
+  }
+#endif
+}

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -500,7 +500,7 @@ extension RunnerTests {
       return ("snapshot returned no nodes", "no-nodes")
     }
     if isSparseApplicationWindowTree(nodes) {
-      return ("snapshot returned only structural application/window nodes", "sparse-tree")
+      return ("snapshot returned no semantic controls or content", "sparse-tree")
     }
     if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold {
       return ("snapshot was cut off by its budget with almost nothing collected", "budget")
@@ -531,19 +531,36 @@ extension RunnerTests {
 
   static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
     guard !nodes.isEmpty else { return false }
+    let rootRects = nodes.compactMap { node in
+      node.type == "Application" || node.type == "Window" ? node.rect : nil
+    }
     return nodes.allSatisfy { node in
       // Application/Window labels are just the app/window name, and full-screen roots
       // compute as hittable; neither says anything about tree health.
       let isRootContainer = node.type == "Application" || node.type == "Window"
-      let hasContent = (!isRootContainer && node.label?.isEmpty == false)
-        || node.identifier?.isEmpty == false
-        || node.value?.isEmpty == false
-      return !hasContent
-        && (isRootContainer || !node.hittable)
-        && Self.structuralOnlyNodeTypes.contains(node.type)
+      guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false }
+      guard !isRootContainer else { return true }
+
+      let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in
+        rootRect.x == node.rect.x && rootRect.y == node.rect.y
+          && rootRect.width == node.rect.width && rootRect.height == node.rect.height
+      }
+      let hasAddressableIdentifier = node.identifier?.isEmpty == false && !isFullScreenContainer
+      return !Self.isSemanticSnapshotText(node.label)
+        && !Self.isSemanticSnapshotText(node.value)
+        && !hasAddressableIdentifier
     }
   }
 
+  /// Private AX can stringify an unserializable accessibility label as the JavaScript object
+  /// placeholder. It is transport residue, not UI content, and must not make a shell-only tree
+  /// look healthy.
+  static func isSemanticSnapshotText(_ text: String?) -> Bool {
+    guard let text else { return false }
+    let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines)
+    return !normalized.isEmpty && normalized.lowercased() != "[object object]"
+  }
+
   /// A leaf whose label joins many short segments is a container marked as an accessibility
   /// element: the platform folds every descendant into one merged node. Nothing below it can
   /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
@@ -686,13 +703,41 @@ extension RunnerTests {
     let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true)
     let window = planTestNode(index: 1, type: "Window", parentIndex: 0)
     let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0)
+    let shell = planTestNode(
+      index: 1,
+      type: "Other",
+      identifier: "appShell",
+      parentIndex: 0
+    )
+    let serializationPlaceholder = planTestNode(
+      index: 2,
+      type: "Other",
+      label: "[object Object]",
+      parentIndex: 1
+    )
 
     // Labeled, hittable root over a bare window is still sparse.
     XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false)))
     // Deadline-truncated near-empty sweep needs recovery even with one real control.
     XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true)))
     // The same tiny tree from a completed sweep is a legitimately minimal screen.
     XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false)))
+    // Container metadata plus a stringified serialization placeholder is not readable UI.
+    XCTAssertNotNil(
+      Self.sparsePayloadReason(
+        DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false)
+      )
+    )
+    let actionableShell = planTestNode(
+      index: 1,
+      type: "Other",
+      identifier: "checkout",
+      hittable: true,
+      parentIndex: 0
+    )
+    XCTAssertNil(
+      Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false))
+    )
     // Empty payloads are degraded.
     XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false)))
   }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift

@@ -101,6 +101,10 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz
   }
 }
 
+func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool {
+  !xCTestChannelPenalized
+}
+
 func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
   steps.contains { step in
     step.synthesized == true && step.kind == "tap"
@@ -217,5 +221,10 @@ extension RunnerTests {
     )
   }
 
+  func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() {
+    XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false))
+    XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true))
+  }
+
 }
 #endif

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -106,9 +106,10 @@ extension RunnerTests {
 
     func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) {
 #if os(iOS)
-      if shouldUseSynthesizedFirstResponderType,
-        let currentTarget = resolveTextEntryElement(app: app, target: activeTarget)
-      {
+      if shouldUseSynthesizedFirstResponderType {
+        guard let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) else {
+          return (nil, false, .notFocused)
+        }
         textEntryRoute = "synthesized-first-responder"
         NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
         let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true)

package.json

@@ -173,7 +173,7 @@
     "test:output-economy": "vitest run --project output-economy",
     "test:smoke:web": "pnpm build && node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-web-platform.test.ts",
     "test:smoke": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-*.test.ts",
-    "test:integration:node": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-concurrency=1 test/integration/*.test.ts",
+    "test:integration:node": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test --experimental-test-isolation=none test/integration/*.test.ts",
     "test:integration": "pnpm test:integration:node && pnpm test:integration:provider",
     "test:concurrency-torture": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/nightly/concurrency-torture.test.ts",
     "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator",

packages/contracts/src/interaction-guarantees.ts

@@ -34,6 +34,10 @@ export const INTERACTION_GUARANTEES = [
   'disambiguation',
   // Targets covered by another visible element are refused.
   'occlusion',
+  // Element-targeted coordinate paths keep the resolved parent identity while
+  // choosing a point outside independently interactive descendants. A parent
+  // whose safe region is fully tiled fails closed instead of activating a child.
+  'parentOwnedTouchPoint',
   // The tap point (rect center) must lie inside the root viewport; closed
   // drawers / off-viewport carousels are refused, not silently no-op tapped.
   'offscreen',
@@ -132,6 +136,7 @@ export type InteractionPathContract = {
 };
 
 const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081';
+const PARENT_OWNED_TOUCH_POINT_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/1718';
 
 // Every path shares the SAME cell by construction: response payloads have one
 // construction site (ADR 0011 Layer 2), and the hand-rolled-literal guard test
@@ -149,6 +154,10 @@ const RUNTIME_TREE_SHARED_GUARANTEES = {
     kind: 'runtime',
     via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
   },
+  parentOwnedTouchPoint: {
+    kind: 'runtime',
+    via: 'src/core/interaction-touch-point.ts#resolveInteractionTouchPoint',
+  },
   // #1542: the base decision is isNodeVisibleOnScreen (bulk accessibility
   // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end
   // enforcement point — on iOS (local, non-provider sessions only) a would-be
@@ -242,6 +251,10 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         kind: 'runtime',
         via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
       },
+      parentOwnedTouchPoint: {
+        kind: 'runtime',
+        via: 'src/core/interaction-touch-point.ts#resolveInteractionTouchPoint',
+      },
       offscreen: {
         kind: 'runtime',
         via: 'src/commands/interaction/runtime/resolution.ts#throwIfOffscreenInteractionTarget',
@@ -293,6 +306,12 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         to: 'runtime-selector',
         via: 'runner ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH fall back to tree-based resolution (isDirectIosSelectorFallbackError delegateSemanticFailures; non-maestro dispatches only) — XCTest skips covered/non-hittable matches, so the runtime path raises the covered-element refusal with its hint',
       },
+      parentOwnedTouchPoint: {
+        kind: 'waived',
+        reason:
+          'gap: the direct runner path has the matched element but no daemon snapshot tree from which to classify independently interactive descendants.',
+        trackingIssue: PARENT_OWNED_TOUCH_POINT_GAP_ISSUE,
+      },
       offscreen: {
         // Decision: TapPointPolicy (pure geometry, parity-tested against the
         // TS twin isTapPointInsideViewport). onScreenWindowFrame stays the
@@ -353,6 +372,11 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         kind: 'runtime',
         via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
       },
+      parentOwnedTouchPoint: {
+        kind: 'inapplicable',
+        reason:
+          'The native web ref path activates a provider-owned semantic handle rather than choosing a coordinate from the daemon snapshot.',
+      },
       // Same enforcement point as the runtime-tree paths (#1542): the
       // preflight guard IS throwIfOffscreenInteractionTarget, which can
       // rescue via the optional iOS confirmOffscreenTargetVisible hook — see
@@ -409,6 +433,10 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         kind: 'inapplicable',
         reason: 'Coordinates bypass element semantics by design (escape hatch).',
       },
+      parentOwnedTouchPoint: {
+        kind: 'inapplicable',
+        reason: 'Coordinates name the exact point to activate; no parent element is resolved.',
+      },
       offscreen: {
         kind: 'runtime',
         via: 'src/commands/interaction/runtime/resolution.ts#resolveInteractionTarget',
@@ -455,6 +483,12 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         kind: 'waived',
         reason: 'Intentional: Maestro taps resolved bounds regardless of overlay state.',
       },
+      parentOwnedTouchPoint: {
+        kind: 'waived',
+        reason:
+          'gap: Maestro compatibility executes the matched element center runner-side without a daemon snapshot tree of independently interactive descendants.',
+        trackingIssue: PARENT_OWNED_TOUCH_POINT_GAP_ISSUE,
+      },
       offscreen: {
         // hasTappableFrame keeps two path-specific choices (empty element
         // frames are refused; app.frame is the frame source, Maestro-style)