From 816a3ea04c3bc02c1442fed7b75370120cfa2634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Sat, 10 Oct 2026 10:18:56 +0200 Subject: [PATCH] Detect gestures on the main thread, in order with their timeouts On Android 13 and later, Backtalk recognizes gestures itself. The touch interaction controller passed each motion event to a thread of its own, but the gesture matchers and the touch exploration delay counted their timeouts on the main thread, so the same state changed on both threads without locking: - A timeout that completes or cancels a gesture, such as the hold of a double tap and hold, could run on the main thread while the gesture thread handled the lift that ended it, and both could report a gesture. - When the touch exploration delay ended, it handed its request to the gesture thread, where cancelling the delay no longer stopped it, so a swipe that started just then could still start touch exploration. - Following a held touch with the circle menu and two-finger rotation kept state that either thread changed. - Cancelling the phonetic letter hint ran the feedback pipeline on the gesture thread, and turning touch exploration off once a touch ended changed the service info there. - Each restart of gesture detection started a new thread without stopping the old one. Android passes motion events and touch state changes to the service through its main thread before the controller hands them on, so the gesture thread never got them sooner. The controller now calls the monitor on the main thread as each event comes in, with no executor, and the monitor no longer hands requests to another thread; the requestStateChangeInSameThread flag, which only chose that, is gone. Events and timeouts wait in one queue, in the order they came in, and cancelling a timeout is exact. An executor that posts to the main thread would not do: a timeout that ran out while the main thread was busy would then run before an event that came in earlier. TouchInteractionMonitorTest runs the monitor with Android's own controller and keeps the main thread busy during a touch: a swipe stays a swipe instead of starting touch exploration, a double tap is not taken for a double tap and hold, touch exploration turned off during a touch changes when the touch ends, and a finger held still starts touch exploration after the delay. Registering the monitor with an executor that posts to the main thread fails two of these tests, and with a thread of its own, all four. On an Android 16 emulator, swipes, double taps and touch exploration work, also after turning the service off and on, and gesture detection logs come from the main thread. Limitations: gestures still wait while the main thread is busy, as before, since Android passes them through it. Following a held touch still posts each move to the circle menu, all displays still share the finger-down state, and stopping detection still goes through the connected displays rather than the registered monitors. --- .../talkback/TalkBackService.java | 8 +- .../talkback/TouchInteractionMonitor.java | 110 ++------ .../talkback/flags/FeatureFlagReader.java | 4 - .../talkback/TouchInteractionMonitorTest.kt | 247 ++++++++++++++++++ 4 files changed, 268 insertions(+), 101 deletions(-) create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 5c5827b4..36ca847e 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -324,8 +324,6 @@ import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import java.util.stream.Stream; import kotlin.Unit; import org.checkerframework.checker.nullness.qual.NonNull; @@ -4251,7 +4249,6 @@ private void registerGestureDetection() { } List displays = WindowUtils.getAllDisplays(getApplicationContext()); - Executor gestureExecutor = Executors.newSingleThreadExecutor(); for (Display display : displays) { @Nullable TouchInteractionController touchInteractionController = getTouchInteractionController(display.getDisplayId()); @@ -4263,7 +4260,6 @@ private void registerGestureDetection() { display, prefs, touchInteractionController, - gestureExecutor, this, primesController, new TouchExplorationModeFailureReporter(analytics), @@ -4272,7 +4268,9 @@ private void registerGestureDetection() { touchInteractionMonitor.setMultiFingerGesturesEnabled(true); touchInteractionMonitor.setTwoFingerPassthroughEnabled(true); touchInteractionMonitor.setServiceHandlesDoubleTap(true); - touchInteractionController.registerCallback(gestureExecutor, touchInteractionMonitor); + // No executor: the controller calls the monitor on the main thread as each event comes in, in + // order with the timeouts that gesture detection counts there. + touchInteractionController.registerCallback(/* executor= */ null, touchInteractionMonitor); displayIdToTouchInteractionMonitors.put(display.getDisplayId(), touchInteractionMonitor); userInterface.registerListener(touchInteractionMonitor); LogUtils.i(TAG, "Enabling service gesture detection on display %d", display.getDisplayId()); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java index 88bbfcc6..a46916f2 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java @@ -27,7 +27,6 @@ import static android.view.MotionEvent.ACTION_MOVE; import static android.view.MotionEvent.ACTION_POINTER_DOWN; import static android.view.MotionEvent.INVALID_POINTER_ID; -import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_CONTROLLER_STATE_CHANGE_LATENCY; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_100; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_150; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_200; @@ -61,10 +60,10 @@ import android.view.MotionEvent; import android.view.ViewConfiguration; import android.view.accessibility.AccessibilityNodeInfo; +import androidx.annotation.MainThread; import androidx.annotation.NonNull; import androidx.annotation.RequiresApi; import androidx.annotation.VisibleForTesting; -import androidx.annotation.WorkerThread; import androidx.core.view.accessibility.AccessibilityNodeInfoCompat; import com.google.android.accessibility.talkback.PrimesController.TimerAction; import com.google.android.accessibility.talkback.contextmenu.ListMenuManager; @@ -92,7 +91,6 @@ import java.util.List; import java.util.Queue; import java.util.Set; -import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -100,6 +98,12 @@ * interaction is a gesture, touch exploration, or passthrough . If the gesture detector clasifies * an interaction as a gesture this class will relay that back to the service. If an interaction * qualifies as touch exploration or a passthrough this class will relay that to the framework. + * + *

All of this runs on the service's main thread: the controller calls this class there as each + * motion event and state change comes in, and the gesture matchers and the touch exploration delay + * count their timeouts there too. Android passes motion events to the service through its main + * thread anyway, so another thread would not get them sooner, and a timeout counted there could run + * out before an event that came in earlier but was still waiting for the busy main thread. */ @RequiresApi(Build.VERSION_CODES.TIRAMISU) public class TouchInteractionMonitor @@ -148,10 +152,6 @@ public class TouchInteractionMonitor private int draggingPointerId = INVALID_POINTER_ID; private final TalkBackService service; private final Handler mainHandler; - private final Executor executor; - // Cache the executor thread for ensuring the request of touch controller's state is running with - // the same executor thread. - private Thread executorThread; private final GestureManifold gestureDetector; private boolean gestureStarted = false; // Whether a gesture was performed while fingers were still down, such as a double tap and hold, @@ -215,7 +215,6 @@ public static boolean isFingerDown() { private boolean waitFirstMotionEvent; private EventId eventId; private final int displayId; - private final boolean requestStateChangeInSameThread; private final PrimesController primesController; private final Queue callerInfos; private int latestDensityDpi; @@ -311,7 +310,6 @@ public TouchInteractionMonitor( Display display, SharedPreferences prefs, TouchInteractionController controller, - Executor executor, TalkBackService service, PrimesController primesController, TouchExplorationModeFailureReporter touchExplorationModeFailureReporter, @@ -322,14 +320,11 @@ public TouchInteractionMonitor( this.display = display; context.registerComponentCallbacks(this); this.controller = controller; - this.executor = executor; - executor.execute(() -> executorThread = Thread.currentThread()); receivedPointerTracker = new ReceivedPointerTracker(); this.service = service; this.primesController = primesController; this.idlePerformer = idlePerformer; this.logger = logger; - requestStateChangeInSameThread = FeatureFlagReader.requestStateChangeInSameThread(context); displayId = context.getDisplay().getDisplayId(); mainHandler = new Handler(context.getMainLooper()); this.touchExplorationModeFailureReporter = touchExplorationModeFailureReporter; @@ -484,7 +479,7 @@ private void updateMultiTapTimeout(SharedPreferences prefs) { } }; - @WorkerThread + @MainThread @SuppressWarnings("Override") @Override public void onMotionEvent(MotionEvent event) { @@ -658,7 +653,7 @@ public void handleMotionEventStateDragging(MotionEvent event) { } } - @WorkerThread + @MainThread @SuppressWarnings("Override") @Override public void onStateChanged(int state) { @@ -1246,18 +1241,7 @@ private IllegalStateException packExceptionWithCallerInfo(Exception e) { return new IllegalStateException(stringBuilder.toString(), e); } - private IllegalStateException genNewException(Exception e) { - return packExceptionWithCallerInfo(e); - } - - private IllegalStateException genException(Exception e) { - return packExceptionWithCallerInfo(e); - } - - protected void requestTouchExplorationInternal(String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + void requestTouchExploration(String caller) { try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_TOUCH_EXPLORING, caller); @@ -1265,38 +1249,13 @@ protected void requestTouchExplorationInternal(String caller, long requestStartT controller.requestTouchExploration(); } } catch (IllegalStateException e) { - // The P/H flag determines which exception packer will be shown in the exception stack. - // TODO: When the solution is verified OK, only one packer left for monitor. - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - void requestTouchExploration(String caller) { - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestTouchExplorationInternal(caller, requestStartTime)); - } else { - requestTouchExplorationInternal(caller, 0L); - } - } - - protected void reportStateChangeLatency(long requestStartTime) { - if (requestStateChangeInSameThread) { - long currentTime = SystemClock.uptimeMillis(); - if (currentTime >= requestStartTime) { - primesController.recordDuration( - TOUCH_CONTROLLER_STATE_CHANGE_LATENCY, requestStartTime, currentTime); - } - } - } - - protected void requestDraggingInternal(int pointerId, String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + protected void requestDragging(int pointerId, String caller) { + LogUtils.v(LOG_TAG, "requestDragging"); try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_DRAGGING, caller); @@ -1305,27 +1264,13 @@ protected void requestDraggingInternal(int pointerId, String caller, long reques gestureDetector.clear(); } catch (IllegalStateException e) { - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - protected void requestDragging(int pointerId, String caller) { - LogUtils.v(LOG_TAG, "requestDragging"); - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestDraggingInternal(pointerId, caller, requestStartTime)); - } else { - requestDraggingInternal(pointerId, caller, 0L); - } - } - - protected void requestDelegatingInternal(String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + protected void requestDelegating(String caller) { + LogUtils.v(LOG_TAG, "requestDelegating"); try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_DELEGATING, caller); @@ -1333,23 +1278,11 @@ protected void requestDelegatingInternal(String caller, long requestStartTime) { } gestureDetector.clear(); } catch (IllegalStateException e) { - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - protected void requestDelegating(String caller) { - LogUtils.v(LOG_TAG, "requestDelegating"); - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestDelegatingInternal(caller, requestStartTime)); - } else { - requestDelegatingInternal(caller, 0L); - } - } - private boolean shouldPerformGestureDetection() { if (this.state == STATE_TOUCH_INTERACTING || this.state == STATE_TOUCH_EXPLORING) { return true; @@ -1461,14 +1394,7 @@ public void run() { if (timerAction != null) { primesController.recordDuration(timerAction, startTime, SystemClock.uptimeMillis()); } - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute( - () -> requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L)); - } else { - requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L); - } + requestTouchExploration("RequestTouchExplorationDelayed"); int savedTimeMs = (int) (SystemClock.uptimeMillis() - postTime); if (savedTimeMs > 0) { logger.logAnalyticsEvent( diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java b/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java index 4542bc71..6876a910 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java @@ -253,10 +253,6 @@ public static boolean enableSplitTapAndHold(Context context) { return false; } - public static boolean requestStateChangeInSameThread(Context context) { - return true; - } - public static boolean logEventBasedLatency(Context context) { return false; } diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt new file mode 100644 index 00000000..14edb800 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt @@ -0,0 +1,247 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback + +import android.accessibilityservice.AccessibilityGestureEvent +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP +import android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT +import android.accessibilityservice.TouchInteractionController +import android.accessibilityservice.TouchInteractionController.STATE_CLEAR +import android.accessibilityservice.TouchInteractionController.STATE_TOUCH_INTERACTING +import android.hardware.display.DisplayManager +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.view.Display +import android.view.MotionEvent +import android.view.MotionEvent.ACTION_DOWN +import android.view.MotionEvent.ACTION_MOVE +import android.view.MotionEvent.ACTION_UP +import android.view.ViewConfiguration +import com.google.android.accessibility.talkback.analytics.TalkBackAnalytics +import com.google.android.accessibility.utils.SharedPreferencesUtils +import java.time.Duration +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowLooper +import org.robolectric.shadows.ShadowSystemClock + +/** + * Runs the monitor with Android's own TouchInteractionController passing it touch state changes and + * motion events on the main thread, as Android does. The main looper is paused, so each test + * decides when it runs, and can keep it busy while the user's finger moves on. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class TouchInteractionMonitorTest { + /** Records the gestures and other results that reach the service, and their threads. */ + class RecordingService : TalkBackService() { + val gestures = mutableListOf() + val threads = mutableSetOf() + + override fun onGesture(accessibilityGestureEvent: AccessibilityGestureEvent): Boolean { + threads.add(Thread.currentThread()) + gestures.add(accessibilityGestureEvent.gestureId) + return true + } + + override fun onGestureDetectionStarted() { + threads.add(Thread.currentThread()) + } + } + + /** Records the threads that ask the controller for touch exploration. */ + private class RecordingReporter : + TouchExplorationModeFailureReporter(object : TalkBackAnalytics {}) { + val requests = mutableListOf() + + override fun onRequestTouchExploration() { + requests.add(Thread.currentThread()) + super.onRequestTouchExploration() + } + } + + private val mainThread: Thread = Looper.getMainLooper().thread + private val main = Handler(Looper.getMainLooper()) + private val mainLooper: ShadowLooper = shadowOf(Looper.getMainLooper()) + private val reporter = RecordingReporter() + private val provisioned = mutableListOf() + private val provisionThreads = mutableListOf() + private lateinit var service: RecordingService + private lateinit var controller: TouchInteractionController + private lateinit var monitor: TouchInteractionMonitor + private var start = 0L + private var downTime = 0L + + @Before + fun setUp() { + service = Robolectric.buildService(RecordingService::class.java).get() + controller = + TouchInteractionController::class + .java + .getDeclaredConstructor( + AccessibilityService::class.java, + Any::class.java, + Int::class.javaPrimitiveType, + ) + .apply { isAccessible = true } + .newInstance(service, Any(), Display.DEFAULT_DISPLAY) + val display = + service.getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY) + // As TalkBackService.registerGestureDetection() sets the monitor up and registers it: with no + // executor, the controller calls the monitor on the main thread as each event comes in. + monitor = + TouchInteractionMonitor( + display, + SharedPreferencesUtils.getSharedPreferences(service), + controller, + service, + PrimesController(), + reporter, + { enabled -> + provisionThreads.add(Thread.currentThread()) + provisioned.add(enabled) + }, + {}, + ) + monitor.setMultiFingerGesturesEnabled(true) + monitor.setTwoFingerPassthroughEnabled(true) + monitor.setServiceHandlesDoubleTap(true) + controller.registerCallback(/* executor= */ null, monitor) + // Without a connection to Android, the controller would refuse every state change. + TouchInteractionController::class.java.getDeclaredField("mServiceDetectsGestures").apply { + isAccessible = true + setBoolean(controller, true) + } + start = SystemClock.uptimeMillis() + } + + @After + fun tearDown() { + monitor.stop() + controller.unregisterCallback(monitor) + } + + private fun callController(name: String, type: Class<*>, argument: Any) { + TouchInteractionController::class + .java + .getDeclaredMethod(name, type) + .apply { isAccessible = true } + .invoke(controller, argument) + } + + /** Android changes the touch state [timeMs] after the start, through the main thread. */ + private fun stateChangeAt(state: Int, timeMs: Long) { + main.postAtTime( + { callController("onStateChanged", Int::class.javaPrimitiveType!!, state) }, + start + timeMs, + ) + } + + /** The finger touches at ([x], [y]) [timeMs] after the start; Android passes it on through the main thread. */ + private fun touchAt(action: Int, x: Float, y: Float, timeMs: Long) { + val eventTime = start + timeMs + if (action == ACTION_DOWN) { + downTime = eventTime + } + val event = MotionEvent.obtain(downTime, eventTime, action, x, y, 0) + main.postAtTime({ callController("onMotionEvent", MotionEvent::class.java, event) }, eventTime) + } + + /** Runs the main thread until [timeMs] after the start. */ + private fun runUntil(timeMs: Long) { + mainLooper.idleFor(Duration.ofMillis(start + timeMs - SystemClock.uptimeMillis())) + } + + /** Lets time pass until [timeMs] after the start while the main thread is busy with other work. */ + private fun mainThreadBusyUntil(timeMs: Long) { + ShadowSystemClock.advanceBy(Duration.ofMillis(start + timeMs - SystemClock.uptimeMillis())) + } + + @Test + fun fingerHeldStillAsksForTouchExplorationOnTheMainThreadAfterTheDelay() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + + runUntil(140) + assertEquals(listOf(), reporter.requests) + runUntil(200) + assertEquals(listOf(mainThread), reporter.requests) + } + + @Test + fun swipeThatWaitedOnTheBusyMainThreadIsAGestureRatherThanTouchExploration() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 40f, 200f, 0) + runUntil(10) + // The finger swipes right while the main thread is busy for longer than the touch exploration + // delay. The delay ends after the moves came in, so the moves come first. + for (step in 1..6) { + touchAt(ACTION_MOVE, 40f + 40f * step, 200f, 10L + 15 * step) + } + touchAt(ACTION_UP, 280f, 200f, 110) + mainThreadBusyUntil(400) + runUntil(450) + + assertEquals(listOf(), reporter.requests) + assertEquals(listOf(GESTURE_SWIPE_RIGHT), service.gestures) + assertEquals(setOf(mainThread), service.threads) + } + + @Test + fun doubleTapThatWaitedOnTheBusyMainThreadIsNotTakenForAHold() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + touchAt(ACTION_UP, 160f, 200f, 50) + touchAt(ACTION_DOWN, 160f, 200f, 150) + runUntil(160) + // The finger lifts 50 ms after it went down again, while the main thread is busy until after + // the hold would have completed. A hold goes to the controller, not to the service. + touchAt(ACTION_UP, 160f, 200f, 200) + val holdEnds = 150 + ViewConfiguration.getLongPressTimeout().toLong() + mainThreadBusyUntil(holdEnds + 100) + runUntil(holdEnds + 110) + + assertEquals(listOf(GESTURE_DOUBLE_TAP), service.gestures) + assertEquals(setOf(mainThread), service.threads) + } + + @Test + fun touchExplorationTurnedOffDuringATouchChangesWhenTheTouchEnds() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + runUntil(10) + + monitor.requestA11yTouchExploreState(false) + runUntil(20) + assertEquals(listOf(), provisioned) + + touchAt(ACTION_UP, 160f, 200f, 50) + stateChangeAt(STATE_CLEAR, 50) + runUntil(60) + assertEquals(listOf(false), provisioned) + assertEquals(listOf(mainThread), provisionThreads) + } +}