diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 5c5827b4..36ca847e 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -324,8 +324,6 @@ import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import java.util.stream.Stream; import kotlin.Unit; import org.checkerframework.checker.nullness.qual.NonNull; @@ -4251,7 +4249,6 @@ private void registerGestureDetection() { } List displays = WindowUtils.getAllDisplays(getApplicationContext()); - Executor gestureExecutor = Executors.newSingleThreadExecutor(); for (Display display : displays) { @Nullable TouchInteractionController touchInteractionController = getTouchInteractionController(display.getDisplayId()); @@ -4263,7 +4260,6 @@ private void registerGestureDetection() { display, prefs, touchInteractionController, - gestureExecutor, this, primesController, new TouchExplorationModeFailureReporter(analytics), @@ -4272,7 +4268,9 @@ private void registerGestureDetection() { touchInteractionMonitor.setMultiFingerGesturesEnabled(true); touchInteractionMonitor.setTwoFingerPassthroughEnabled(true); touchInteractionMonitor.setServiceHandlesDoubleTap(true); - touchInteractionController.registerCallback(gestureExecutor, touchInteractionMonitor); + // No executor: the controller calls the monitor on the main thread as each event comes in, in + // order with the timeouts that gesture detection counts there. + touchInteractionController.registerCallback(/* executor= */ null, touchInteractionMonitor); displayIdToTouchInteractionMonitors.put(display.getDisplayId(), touchInteractionMonitor); userInterface.registerListener(touchInteractionMonitor); LogUtils.i(TAG, "Enabling service gesture detection on display %d", display.getDisplayId()); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java index 88bbfcc6..a46916f2 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java @@ -27,7 +27,6 @@ import static android.view.MotionEvent.ACTION_MOVE; import static android.view.MotionEvent.ACTION_POINTER_DOWN; import static android.view.MotionEvent.INVALID_POINTER_ID; -import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_CONTROLLER_STATE_CHANGE_LATENCY; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_100; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_150; import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_200; @@ -61,10 +60,10 @@ import android.view.MotionEvent; import android.view.ViewConfiguration; import android.view.accessibility.AccessibilityNodeInfo; +import androidx.annotation.MainThread; import androidx.annotation.NonNull; import androidx.annotation.RequiresApi; import androidx.annotation.VisibleForTesting; -import androidx.annotation.WorkerThread; import androidx.core.view.accessibility.AccessibilityNodeInfoCompat; import com.google.android.accessibility.talkback.PrimesController.TimerAction; import com.google.android.accessibility.talkback.contextmenu.ListMenuManager; @@ -92,7 +91,6 @@ import java.util.List; import java.util.Queue; import java.util.Set; -import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -100,6 +98,12 @@ * interaction is a gesture, touch exploration, or passthrough . If the gesture detector clasifies * an interaction as a gesture this class will relay that back to the service. If an interaction * qualifies as touch exploration or a passthrough this class will relay that to the framework. + * + *

All of this runs on the service's main thread: the controller calls this class there as each + * motion event and state change comes in, and the gesture matchers and the touch exploration delay + * count their timeouts there too. Android passes motion events to the service through its main + * thread anyway, so another thread would not get them sooner, and a timeout counted there could run + * out before an event that came in earlier but was still waiting for the busy main thread. */ @RequiresApi(Build.VERSION_CODES.TIRAMISU) public class TouchInteractionMonitor @@ -148,10 +152,6 @@ public class TouchInteractionMonitor private int draggingPointerId = INVALID_POINTER_ID; private final TalkBackService service; private final Handler mainHandler; - private final Executor executor; - // Cache the executor thread for ensuring the request of touch controller's state is running with - // the same executor thread. - private Thread executorThread; private final GestureManifold gestureDetector; private boolean gestureStarted = false; // Whether a gesture was performed while fingers were still down, such as a double tap and hold, @@ -215,7 +215,6 @@ public static boolean isFingerDown() { private boolean waitFirstMotionEvent; private EventId eventId; private final int displayId; - private final boolean requestStateChangeInSameThread; private final PrimesController primesController; private final Queue callerInfos; private int latestDensityDpi; @@ -311,7 +310,6 @@ public TouchInteractionMonitor( Display display, SharedPreferences prefs, TouchInteractionController controller, - Executor executor, TalkBackService service, PrimesController primesController, TouchExplorationModeFailureReporter touchExplorationModeFailureReporter, @@ -322,14 +320,11 @@ public TouchInteractionMonitor( this.display = display; context.registerComponentCallbacks(this); this.controller = controller; - this.executor = executor; - executor.execute(() -> executorThread = Thread.currentThread()); receivedPointerTracker = new ReceivedPointerTracker(); this.service = service; this.primesController = primesController; this.idlePerformer = idlePerformer; this.logger = logger; - requestStateChangeInSameThread = FeatureFlagReader.requestStateChangeInSameThread(context); displayId = context.getDisplay().getDisplayId(); mainHandler = new Handler(context.getMainLooper()); this.touchExplorationModeFailureReporter = touchExplorationModeFailureReporter; @@ -484,7 +479,7 @@ private void updateMultiTapTimeout(SharedPreferences prefs) { } }; - @WorkerThread + @MainThread @SuppressWarnings("Override") @Override public void onMotionEvent(MotionEvent event) { @@ -658,7 +653,7 @@ public void handleMotionEventStateDragging(MotionEvent event) { } } - @WorkerThread + @MainThread @SuppressWarnings("Override") @Override public void onStateChanged(int state) { @@ -1246,18 +1241,7 @@ private IllegalStateException packExceptionWithCallerInfo(Exception e) { return new IllegalStateException(stringBuilder.toString(), e); } - private IllegalStateException genNewException(Exception e) { - return packExceptionWithCallerInfo(e); - } - - private IllegalStateException genException(Exception e) { - return packExceptionWithCallerInfo(e); - } - - protected void requestTouchExplorationInternal(String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + void requestTouchExploration(String caller) { try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_TOUCH_EXPLORING, caller); @@ -1265,38 +1249,13 @@ protected void requestTouchExplorationInternal(String caller, long requestStartT controller.requestTouchExploration(); } } catch (IllegalStateException e) { - // The P/H flag determines which exception packer will be shown in the exception stack. - // TODO: When the solution is verified OK, only one packer left for monitor. - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - void requestTouchExploration(String caller) { - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestTouchExplorationInternal(caller, requestStartTime)); - } else { - requestTouchExplorationInternal(caller, 0L); - } - } - - protected void reportStateChangeLatency(long requestStartTime) { - if (requestStateChangeInSameThread) { - long currentTime = SystemClock.uptimeMillis(); - if (currentTime >= requestStartTime) { - primesController.recordDuration( - TOUCH_CONTROLLER_STATE_CHANGE_LATENCY, requestStartTime, currentTime); - } - } - } - - protected void requestDraggingInternal(int pointerId, String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + protected void requestDragging(int pointerId, String caller) { + LogUtils.v(LOG_TAG, "requestDragging"); try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_DRAGGING, caller); @@ -1305,27 +1264,13 @@ protected void requestDraggingInternal(int pointerId, String caller, long reques gestureDetector.clear(); } catch (IllegalStateException e) { - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - protected void requestDragging(int pointerId, String caller) { - LogUtils.v(LOG_TAG, "requestDragging"); - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestDraggingInternal(pointerId, caller, requestStartTime)); - } else { - requestDraggingInternal(pointerId, caller, 0L); - } - } - - protected void requestDelegatingInternal(String caller, long requestStartTime) { - if (requestStartTime != 0L) { - reportStateChangeLatency(requestStartTime); - } + protected void requestDelegating(String caller) { + LogUtils.v(LOG_TAG, "requestDelegating"); try { if (isStateTransitionAllowed()) { trackStateChangeRequest(STATE_DELEGATING, caller); @@ -1333,23 +1278,11 @@ protected void requestDelegatingInternal(String caller, long requestStartTime) { } gestureDetector.clear(); } catch (IllegalStateException e) { - throw requestStateChangeInSameThread ? genNewException(e) : genException(e); + throw packExceptionWithCallerInfo(e); } stateChangeRequested = true; } - protected void requestDelegating(String caller) { - LogUtils.v(LOG_TAG, "requestDelegating"); - long requestStartTime = SystemClock.uptimeMillis(); - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute(() -> requestDelegatingInternal(caller, requestStartTime)); - } else { - requestDelegatingInternal(caller, 0L); - } - } - private boolean shouldPerformGestureDetection() { if (this.state == STATE_TOUCH_INTERACTING || this.state == STATE_TOUCH_EXPLORING) { return true; @@ -1461,14 +1394,7 @@ public void run() { if (timerAction != null) { primesController.recordDuration(timerAction, startTime, SystemClock.uptimeMillis()); } - if (requestStateChangeInSameThread - && executorThread != null - && !executorThread.equals(Thread.currentThread())) { - executor.execute( - () -> requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L)); - } else { - requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L); - } + requestTouchExploration("RequestTouchExplorationDelayed"); int savedTimeMs = (int) (SystemClock.uptimeMillis() - postTime); if (savedTimeMs > 0) { logger.logAnalyticsEvent( diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java b/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java index 4542bc71..6876a910 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java @@ -253,10 +253,6 @@ public static boolean enableSplitTapAndHold(Context context) { return false; } - public static boolean requestStateChangeInSameThread(Context context) { - return true; - } - public static boolean logEventBasedLatency(Context context) { return false; } diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt new file mode 100644 index 00000000..14edb800 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/TouchInteractionMonitorTest.kt @@ -0,0 +1,247 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback + +import android.accessibilityservice.AccessibilityGestureEvent +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP +import android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT +import android.accessibilityservice.TouchInteractionController +import android.accessibilityservice.TouchInteractionController.STATE_CLEAR +import android.accessibilityservice.TouchInteractionController.STATE_TOUCH_INTERACTING +import android.hardware.display.DisplayManager +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.view.Display +import android.view.MotionEvent +import android.view.MotionEvent.ACTION_DOWN +import android.view.MotionEvent.ACTION_MOVE +import android.view.MotionEvent.ACTION_UP +import android.view.ViewConfiguration +import com.google.android.accessibility.talkback.analytics.TalkBackAnalytics +import com.google.android.accessibility.utils.SharedPreferencesUtils +import java.time.Duration +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowLooper +import org.robolectric.shadows.ShadowSystemClock + +/** + * Runs the monitor with Android's own TouchInteractionController passing it touch state changes and + * motion events on the main thread, as Android does. The main looper is paused, so each test + * decides when it runs, and can keep it busy while the user's finger moves on. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class TouchInteractionMonitorTest { + /** Records the gestures and other results that reach the service, and their threads. */ + class RecordingService : TalkBackService() { + val gestures = mutableListOf() + val threads = mutableSetOf() + + override fun onGesture(accessibilityGestureEvent: AccessibilityGestureEvent): Boolean { + threads.add(Thread.currentThread()) + gestures.add(accessibilityGestureEvent.gestureId) + return true + } + + override fun onGestureDetectionStarted() { + threads.add(Thread.currentThread()) + } + } + + /** Records the threads that ask the controller for touch exploration. */ + private class RecordingReporter : + TouchExplorationModeFailureReporter(object : TalkBackAnalytics {}) { + val requests = mutableListOf() + + override fun onRequestTouchExploration() { + requests.add(Thread.currentThread()) + super.onRequestTouchExploration() + } + } + + private val mainThread: Thread = Looper.getMainLooper().thread + private val main = Handler(Looper.getMainLooper()) + private val mainLooper: ShadowLooper = shadowOf(Looper.getMainLooper()) + private val reporter = RecordingReporter() + private val provisioned = mutableListOf() + private val provisionThreads = mutableListOf() + private lateinit var service: RecordingService + private lateinit var controller: TouchInteractionController + private lateinit var monitor: TouchInteractionMonitor + private var start = 0L + private var downTime = 0L + + @Before + fun setUp() { + service = Robolectric.buildService(RecordingService::class.java).get() + controller = + TouchInteractionController::class + .java + .getDeclaredConstructor( + AccessibilityService::class.java, + Any::class.java, + Int::class.javaPrimitiveType, + ) + .apply { isAccessible = true } + .newInstance(service, Any(), Display.DEFAULT_DISPLAY) + val display = + service.getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY) + // As TalkBackService.registerGestureDetection() sets the monitor up and registers it: with no + // executor, the controller calls the monitor on the main thread as each event comes in. + monitor = + TouchInteractionMonitor( + display, + SharedPreferencesUtils.getSharedPreferences(service), + controller, + service, + PrimesController(), + reporter, + { enabled -> + provisionThreads.add(Thread.currentThread()) + provisioned.add(enabled) + }, + {}, + ) + monitor.setMultiFingerGesturesEnabled(true) + monitor.setTwoFingerPassthroughEnabled(true) + monitor.setServiceHandlesDoubleTap(true) + controller.registerCallback(/* executor= */ null, monitor) + // Without a connection to Android, the controller would refuse every state change. + TouchInteractionController::class.java.getDeclaredField("mServiceDetectsGestures").apply { + isAccessible = true + setBoolean(controller, true) + } + start = SystemClock.uptimeMillis() + } + + @After + fun tearDown() { + monitor.stop() + controller.unregisterCallback(monitor) + } + + private fun callController(name: String, type: Class<*>, argument: Any) { + TouchInteractionController::class + .java + .getDeclaredMethod(name, type) + .apply { isAccessible = true } + .invoke(controller, argument) + } + + /** Android changes the touch state [timeMs] after the start, through the main thread. */ + private fun stateChangeAt(state: Int, timeMs: Long) { + main.postAtTime( + { callController("onStateChanged", Int::class.javaPrimitiveType!!, state) }, + start + timeMs, + ) + } + + /** The finger touches at ([x], [y]) [timeMs] after the start; Android passes it on through the main thread. */ + private fun touchAt(action: Int, x: Float, y: Float, timeMs: Long) { + val eventTime = start + timeMs + if (action == ACTION_DOWN) { + downTime = eventTime + } + val event = MotionEvent.obtain(downTime, eventTime, action, x, y, 0) + main.postAtTime({ callController("onMotionEvent", MotionEvent::class.java, event) }, eventTime) + } + + /** Runs the main thread until [timeMs] after the start. */ + private fun runUntil(timeMs: Long) { + mainLooper.idleFor(Duration.ofMillis(start + timeMs - SystemClock.uptimeMillis())) + } + + /** Lets time pass until [timeMs] after the start while the main thread is busy with other work. */ + private fun mainThreadBusyUntil(timeMs: Long) { + ShadowSystemClock.advanceBy(Duration.ofMillis(start + timeMs - SystemClock.uptimeMillis())) + } + + @Test + fun fingerHeldStillAsksForTouchExplorationOnTheMainThreadAfterTheDelay() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + + runUntil(140) + assertEquals(listOf(), reporter.requests) + runUntil(200) + assertEquals(listOf(mainThread), reporter.requests) + } + + @Test + fun swipeThatWaitedOnTheBusyMainThreadIsAGestureRatherThanTouchExploration() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 40f, 200f, 0) + runUntil(10) + // The finger swipes right while the main thread is busy for longer than the touch exploration + // delay. The delay ends after the moves came in, so the moves come first. + for (step in 1..6) { + touchAt(ACTION_MOVE, 40f + 40f * step, 200f, 10L + 15 * step) + } + touchAt(ACTION_UP, 280f, 200f, 110) + mainThreadBusyUntil(400) + runUntil(450) + + assertEquals(listOf(), reporter.requests) + assertEquals(listOf(GESTURE_SWIPE_RIGHT), service.gestures) + assertEquals(setOf(mainThread), service.threads) + } + + @Test + fun doubleTapThatWaitedOnTheBusyMainThreadIsNotTakenForAHold() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + touchAt(ACTION_UP, 160f, 200f, 50) + touchAt(ACTION_DOWN, 160f, 200f, 150) + runUntil(160) + // The finger lifts 50 ms after it went down again, while the main thread is busy until after + // the hold would have completed. A hold goes to the controller, not to the service. + touchAt(ACTION_UP, 160f, 200f, 200) + val holdEnds = 150 + ViewConfiguration.getLongPressTimeout().toLong() + mainThreadBusyUntil(holdEnds + 100) + runUntil(holdEnds + 110) + + assertEquals(listOf(GESTURE_DOUBLE_TAP), service.gestures) + assertEquals(setOf(mainThread), service.threads) + } + + @Test + fun touchExplorationTurnedOffDuringATouchChangesWhenTheTouchEnds() { + stateChangeAt(STATE_TOUCH_INTERACTING, 0) + touchAt(ACTION_DOWN, 160f, 200f, 0) + runUntil(10) + + monitor.requestA11yTouchExploreState(false) + runUntil(20) + assertEquals(listOf(), provisioned) + + touchAt(ACTION_UP, 160f, 200f, 50) + stateChangeAt(STATE_CLEAR, 50) + runUntil(60) + assertEquals(listOf(false), provisioned) + assertEquals(listOf(mainThread), provisionThreads) + } +}