Skip to content

Detect gestures on the main thread, in order with their timeouts #144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
trypsynth merged 1 commit into from
Oct 10, 2026
Merged

Detect gestures on the main thread, in order with their timeouts #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,8 +324,6 @@
import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.stream.Stream;
import kotlin.Unit;
import org.checkerframework.checker.nullness.qual.NonNull;
Expand Down Expand Up @@ -4251,7 +4249,6 @@ private void registerGestureDetection() {
}

List<Display> displays = WindowUtils.getAllDisplays(getApplicationContext());
Executor gestureExecutor = Executors.newSingleThreadExecutor();
for (Display display : displays) {
@Nullable TouchInteractionController touchInteractionController =
getTouchInteractionController(display.getDisplayId());
Expand All @@ -4263,7 +4260,6 @@ private void registerGestureDetection() {
display,
prefs,
touchInteractionController,
gestureExecutor,
this,
primesController,
new TouchExplorationModeFailureReporter(analytics),
Expand All @@ -4272,7 +4268,9 @@ private void registerGestureDetection() {
touchInteractionMonitor.setMultiFingerGesturesEnabled(true);
touchInteractionMonitor.setTwoFingerPassthroughEnabled(true);
touchInteractionMonitor.setServiceHandlesDoubleTap(true);
touchInteractionController.registerCallback(gestureExecutor, touchInteractionMonitor);
// No executor: the controller calls the monitor on the main thread as each event comes in, in
// order with the timeouts that gesture detection counts there.
touchInteractionController.registerCallback(/* executor= */ null, touchInteractionMonitor);
displayIdToTouchInteractionMonitors.put(display.getDisplayId(), touchInteractionMonitor);
userInterface.registerListener(touchInteractionMonitor);
LogUtils.i(TAG, "Enabling service gesture detection on display %d", display.getDisplayId());
Expand Down
110 changes: 18 additions & 92 deletions ...back/src/main/java/com/google/android/accessibility/talkback/TouchInteractionMonitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import static android.view.MotionEvent.ACTION_MOVE;
import static android.view.MotionEvent.ACTION_POINTER_DOWN;
import static android.view.MotionEvent.INVALID_POINTER_ID;
import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_CONTROLLER_STATE_CHANGE_LATENCY;
import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_100;
import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_150;
import static com.google.android.accessibility.talkback.PrimesController.TimerAction.TOUCH_EXPLORE_DELAY_200;
Expand Down Expand Up @@ -61,10 +60,10 @@
import android.view.MotionEvent;
import android.view.ViewConfiguration;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.annotation.MainThread;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.annotation.WorkerThread;
import androidx.core.view.accessibility.AccessibilityNodeInfoCompat;
import com.google.android.accessibility.talkback.PrimesController.TimerAction;
import com.google.android.accessibility.talkback.contextmenu.ListMenuManager;
Expand Down Expand Up @@ -92,14 +91,19 @@
import java.util.List;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.Nullable;

/**
* This class receives motion events from the framework for the purposes of figuring out whether an
* interaction is a gesture, touch exploration, or passthrough . If the gesture detector clasifies
* an interaction as a gesture this class will relay that back to the service. If an interaction
* qualifies as touch exploration or a passthrough this class will relay that to the framework.
*
* <p>All of this runs on the service's main thread: the controller calls this class there as each
* motion event and state change comes in, and the gesture matchers and the touch exploration delay
* count their timeouts there too. Android passes motion events to the service through its main
* thread anyway, so another thread would not get them sooner, and a timeout counted there could run
* out before an event that came in earlier but was still waiting for the busy main thread.
*/
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
public class TouchInteractionMonitor
Expand Down Expand Up @@ -148,10 +152,6 @@ public class TouchInteractionMonitor
private int draggingPointerId = INVALID_POINTER_ID;
private final TalkBackService service;
private final Handler mainHandler;
private final Executor executor;
// Cache the executor thread for ensuring the request of touch controller's state is running with
// the same executor thread.
private Thread executorThread;
private final GestureManifold gestureDetector;
private boolean gestureStarted = false;
// Whether a gesture was performed while fingers were still down, such as a double tap and hold,
Expand Down Expand Up @@ -215,7 +215,6 @@ public static boolean isFingerDown() {
private boolean waitFirstMotionEvent;
private EventId eventId;
private final int displayId;
private final boolean requestStateChangeInSameThread;
private final PrimesController primesController;
private final Queue<CallerInfo> callerInfos;
private int latestDensityDpi;
Expand Down Expand Up @@ -311,7 +310,6 @@ public TouchInteractionMonitor(
Display display,
SharedPreferences prefs,
TouchInteractionController controller,
Executor executor,
TalkBackService service,
PrimesController primesController,
TouchExplorationModeFailureReporter touchExplorationModeFailureReporter,
Expand All @@ -322,14 +320,11 @@ public TouchInteractionMonitor(
this.display = display;
context.registerComponentCallbacks(this);
this.controller = controller;
this.executor = executor;
executor.execute(() -> executorThread = Thread.currentThread());
receivedPointerTracker = new ReceivedPointerTracker();
this.service = service;
this.primesController = primesController;
this.idlePerformer = idlePerformer;
this.logger = logger;
requestStateChangeInSameThread = FeatureFlagReader.requestStateChangeInSameThread(context);
displayId = context.getDisplay().getDisplayId();
mainHandler = new Handler(context.getMainLooper());
this.touchExplorationModeFailureReporter = touchExplorationModeFailureReporter;
Expand Down Expand Up @@ -484,7 +479,7 @@ private void updateMultiTapTimeout(SharedPreferences prefs) {
}
};

@WorkerThread
@MainThread
@SuppressWarnings("Override")
@Override
public void onMotionEvent(MotionEvent event) {
Expand Down Expand Up @@ -658,7 +653,7 @@ public void handleMotionEventStateDragging(MotionEvent event) {
}
}

@WorkerThread
@MainThread
@SuppressWarnings("Override")
@Override
public void onStateChanged(int state) {
Expand Down Expand Up @@ -1246,57 +1241,21 @@ private IllegalStateException packExceptionWithCallerInfo(Exception e) {
return new IllegalStateException(stringBuilder.toString(), e);
}

private IllegalStateException genNewException(Exception e) {
return packExceptionWithCallerInfo(e);
}

private IllegalStateException genException(Exception e) {
return packExceptionWithCallerInfo(e);
}

protected void requestTouchExplorationInternal(String caller, long requestStartTime) {
if (requestStartTime != 0L) {
reportStateChangeLatency(requestStartTime);
}
void requestTouchExploration(String caller) {
try {
if (isStateTransitionAllowed()) {
trackStateChangeRequest(STATE_TOUCH_EXPLORING, caller);
touchExplorationModeFailureReporter.onRequestTouchExploration();
controller.requestTouchExploration();
}
} catch (IllegalStateException e) {
// The P/H flag determines which exception packer will be shown in the exception stack.
// TODO: When the solution is verified OK, only one packer left for monitor.
throw requestStateChangeInSameThread ? genNewException(e) : genException(e);
throw packExceptionWithCallerInfo(e);
}
stateChangeRequested = true;
}

void requestTouchExploration(String caller) {
long requestStartTime = SystemClock.uptimeMillis();
if (requestStateChangeInSameThread
&& executorThread != null
&& !executorThread.equals(Thread.currentThread())) {
executor.execute(() -> requestTouchExplorationInternal(caller, requestStartTime));
} else {
requestTouchExplorationInternal(caller, 0L);
}
}

protected void reportStateChangeLatency(long requestStartTime) {
if (requestStateChangeInSameThread) {
long currentTime = SystemClock.uptimeMillis();
if (currentTime >= requestStartTime) {
primesController.recordDuration(
TOUCH_CONTROLLER_STATE_CHANGE_LATENCY, requestStartTime, currentTime);
}
}
}

protected void requestDraggingInternal(int pointerId, String caller, long requestStartTime) {
if (requestStartTime != 0L) {
reportStateChangeLatency(requestStartTime);
}
protected void requestDragging(int pointerId, String caller) {
LogUtils.v(LOG_TAG, "requestDragging");
try {
if (isStateTransitionAllowed()) {
trackStateChangeRequest(STATE_DRAGGING, caller);
Expand All @@ -1305,51 +1264,25 @@ protected void requestDraggingInternal(int pointerId, String caller, long reques
gestureDetector.clear();

} catch (IllegalStateException e) {
throw requestStateChangeInSameThread ? genNewException(e) : genException(e);
throw packExceptionWithCallerInfo(e);
}
stateChangeRequested = true;
}

protected void requestDragging(int pointerId, String caller) {
LogUtils.v(LOG_TAG, "requestDragging");
long requestStartTime = SystemClock.uptimeMillis();
if (requestStateChangeInSameThread
&& executorThread != null
&& !executorThread.equals(Thread.currentThread())) {
executor.execute(() -> requestDraggingInternal(pointerId, caller, requestStartTime));
} else {
requestDraggingInternal(pointerId, caller, 0L);
}
}

protected void requestDelegatingInternal(String caller, long requestStartTime) {
if (requestStartTime != 0L) {
reportStateChangeLatency(requestStartTime);
}
protected void requestDelegating(String caller) {
LogUtils.v(LOG_TAG, "requestDelegating");
try {
if (isStateTransitionAllowed()) {
trackStateChangeRequest(STATE_DELEGATING, caller);
controller.requestDelegating();
}
gestureDetector.clear();
} catch (IllegalStateException e) {
throw requestStateChangeInSameThread ? genNewException(e) : genException(e);
throw packExceptionWithCallerInfo(e);
}
stateChangeRequested = true;
}

protected void requestDelegating(String caller) {
LogUtils.v(LOG_TAG, "requestDelegating");
long requestStartTime = SystemClock.uptimeMillis();
if (requestStateChangeInSameThread
&& executorThread != null
&& !executorThread.equals(Thread.currentThread())) {
executor.execute(() -> requestDelegatingInternal(caller, requestStartTime));
} else {
requestDelegatingInternal(caller, 0L);
}
}

private boolean shouldPerformGestureDetection() {
if (this.state == STATE_TOUCH_INTERACTING || this.state == STATE_TOUCH_EXPLORING) {
return true;
Expand Down Expand Up @@ -1461,14 +1394,7 @@ public void run() {
if (timerAction != null) {
primesController.recordDuration(timerAction, startTime, SystemClock.uptimeMillis());
}
if (requestStateChangeInSameThread
&& executorThread != null
&& !executorThread.equals(Thread.currentThread())) {
executor.execute(
() -> requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L));
} else {
requestTouchExplorationInternal("RequestTouchExplorationDelayed", 0L);
}
requestTouchExploration("RequestTouchExplorationDelayed");
int savedTimeMs = (int) (SystemClock.uptimeMillis() - postTime);
if (savedTimeMs > 0) {
logger.logAnalyticsEvent(
Expand Down
4 changes: 0 additions & 4 deletions ...back/src/main/java/com/google/android/accessibility/talkback/flags/FeatureFlagReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,10 +253,6 @@ public static boolean enableSplitTapAndHold(Context context) {
return false;
}

public static boolean requestStateChangeInSameThread(Context context) {
return true;
}

public static boolean logEventBasedLatency(Context context) {
return false;
}
Expand Down
Loading
Loading