From 7817ec4b51832a8852c3f14dbb9df2e591d44505 Mon Sep 17 00:00:00 2001 From: willbilec Date: Sat, 10 Oct 2026 17:24:53 -0600 Subject: [PATCH 1/2] Restore previous accessibility focus when going Back --- differences.md | 8 + lint-baseline.xml | 20 + .../talkback/TalkBackService.java | 63 +- .../talkback/actor/FocusActor.java | 16 + .../talkback/actor/SystemActionPerformer.java | 9 + .../focusmanagement/ScreenFocusBookmarks.kt | 127 ++ .../focusmanagement/ScreenFocusRestorer.kt | 728 ++++++++++ .../preference/TalkBackPreferenceFilter.java | 1 + .../src/main/res/values/donottranslate.xml | 1 + talkback/src/main/res/values/strings.xml | 4 + .../src/main/res/xml/advanced_preferences.xml | 5 + .../ScreenFocusBookmarksTest.kt | 165 +++ .../ScreenFocusRestorerTest.kt | 1288 +++++++++++++++++ 13 files changed, 2426 insertions(+), 9 deletions(-) create mode 100644 talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusBookmarks.kt create mode 100644 talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusBookmarksTest.kt create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorerTest.kt diff --git a/differences.md b/differences.md index 92de7d536..cd4b59e81 100644 --- a/differences.md +++ b/differences.md @@ -1,5 +1,13 @@ # Differences from TalkBack +## Returning to a screen + +Backtalk remembers the last item you read on up to 20 recent app screens and browser pages while the service is running. Restoration starts only when you use Back, including the Back gesture/key or a recognized app Back button. It tries to focus the same item, including when an app recreates its list or automatically focuses the toolbar. This also works when an app opens a post or link in another app: using Back from the browser or other app restores the originating item. Forward navigation, switching apps, and opening or reading the status bar do not trigger restoration. Input on the returned screen or focus in another window cancels pending restoration. This memory clears when Backtalk stops. + +Visible items are restored without scrolling. **Advanced settings** > **Scroll to restore previous focus** is off by default. Turn it on to let Backtalk bring an offscreen item back into view. Backtalk uses the app's supported scroll actions and checks that it found the remembered item; it stops after at most ten scroll actions or three seconds. It waits for rebuilding content instead of searching by scrolling when the saved row position is already visible. Touching, navigating, typing, or scrolling cancels pending restoration. A browser restoring its own page scroll position or updating its address during Back does not cancel the attempt. + +Apps and pages must expose enough accessibility information to distinguish the screen and item. In native app rows and cards, a single exposed title or heading can identify the item even when its read-state description, counts, or subtitle change. This applies across apps; uncertain or duplicate titles still need an unambiguous match. If the item was removed, labels are ambiguous, or the app does not expose the necessary information, Backtalk keeps its usual focus behavior. It remembers elements such as headings, links, and posts, rather than character positions within text. TV keeps its usual input-focus behavior. + Backtalk is built from Google's TalkBack source releases. This document describes how Backtalk differs from TalkBack: features that TalkBack doesn't have, defaults that Backtalk changes, and problems in TalkBack that Backtalk fixes. To learn how to install Backtalk, see the [README](README.md). Each section names the settings involved as they appear in Backtalk. Unless a section says otherwise, each setting is in Backtalk settings. diff --git a/lint-baseline.xml b/lint-baseline.xml index 999ca12d0..f2cf64d6a 100644 --- a/lint-baseline.xml +++ b/lint-baseline.xml @@ -25520,4 +25520,24 @@ + + + + + + diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 12aa46c7d..25a2363ea 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -173,6 +173,7 @@ import com.google.android.accessibility.talkback.flags.Flags; import com.google.android.accessibility.talkback.focusmanagement.AccessibilityFocusMonitor; import com.google.android.accessibility.talkback.focusmanagement.FocusProcessorForLogicalNavigation; +import com.google.android.accessibility.talkback.focusmanagement.ScreenFocusRestorer; import com.google.android.accessibility.talkback.focusmanagement.TraversalTreeCache; import com.google.android.accessibility.talkback.focusmanagement.interpreter.ScreenStateMonitor; import com.google.android.accessibility.talkback.focusmanagement.interpreter.TouchExplorationInterpreter; @@ -821,6 +822,7 @@ public interface GestureDetectionState { private EventLatencyLogger eventLatencyLogger; private UserInterface userInterface; + private @Nullable ScreenFocusRestorer screenFocusRestorer; private DisableTalkBackDialog disableTalkbackDialog; private Configuration lastConfiguration; @@ -896,6 +898,9 @@ private float userSpeechVolume() { @Override public boolean onUnbind(Intent intent) { + if (screenFocusRestorer != null) { + screenFocusRestorer.shutdown(); + } LogUtils.d(TAG, "onUnbind start"); final long turningOffTime = System.currentTimeMillis(); interruptAllFeedback(/* stopTtsSpeechCompletely= */ false); @@ -933,6 +938,10 @@ public boolean onUnbind(Intent intent) { @Override public void onDestroy() { + if (screenFocusRestorer != null) { + screenFocusRestorer.shutdown(); + screenFocusRestorer = null; + } if (userInterface != null) { userInterface.unregisterAllListeners(); } @@ -1099,6 +1108,9 @@ public void onAccessibilityEvent(AccessibilityEvent event) { } Performance perf = Performance.getInstance(); EventId eventId = perf.onEventReceived(event); + if (screenFocusRestorer != null) { + screenFocusRestorer.onAccessibilityEvent(event); + } int eventType = event.getEventType(); TraversalTreeCache.onAccessibilityEvent(event); PreparedFocusSpeech.onAccessibilityEvent(event); @@ -1314,6 +1326,9 @@ && switch (physicalKeyboardEcho) { */ @Override protected final boolean onKeyEvent(KeyEvent keyEvent) { + if (screenFocusRestorer != null) { + screenFocusRestorer.onKeyEvent(keyEvent); + } boolean result = onKeyEventInternal(keyEvent); if (primesController != null) { @@ -1498,6 +1513,9 @@ private void onGestureDebug(GestureAnalyticsEvent analyticsEvent) { /** Called by {@link TouchInteractionMonitor} when gesture detection started. */ public void onGestureDetectionStarted() { + if (screenFocusRestorer != null) { + screenFocusRestorer.onUserInteraction(); + } if (processorPhoneticLetters != null) { processorPhoneticLetters.cancelPhoneticLetter(EVENT_ID_UNTRACKED); } @@ -1507,6 +1525,9 @@ private boolean handleOnGestureById(int displayId, int gestureId) { if (!isServiceActive() || PauseController.isPaused()) { return false; } + if (screenFocusRestorer != null) { + screenFocusRestorer.onUserInteraction(); + } // Return before feedback, training, menus and gesture recording. Both callback overloads // use this entry point; ignoring an action in GestureController still reports it as handled. if (VibrationWatchGestureSettings.shouldReserve( @@ -1938,6 +1959,9 @@ private void initializeFingerprintGestureCallback() { @Override public void onGestureDetected(int gesture) { if (isServiceActive() && !PauseController.isPaused() && gestureController != null) { + if (screenFocusRestorer != null) { + screenFocusRestorer.onUserInteraction(); + } Performance perf = Performance.getInstance(); EventId eventId = perf.onFingerprintGestureEventReceived(gesture); @@ -2198,6 +2222,17 @@ public long getSpeechCacheWindowCounts() { KeyboardActor keyboardActor = new KeyboardActor(this); // Construct pipeline. + SystemActionPerformer systemActionPerformer = new SystemActionPerformer(this); + systemActionPerformer.setBeforeActionListener( + action -> { + if (screenFocusRestorer != null) { + if (action == GLOBAL_ACTION_BACK) { + screenFocusRestorer.requestBack(); + } else { + screenFocusRestorer.onUserInteraction(); + } + } + }); pipeline = new Pipeline( this, @@ -2250,7 +2285,7 @@ public long getSpeechCacheWindowCounts() { editor, labelManager, new NodeActionPerformer(), - new SystemActionPerformer(this), + systemActionPerformer, languageActor, passThroughModeActor, new TalkBackUIActor(this), @@ -2461,15 +2496,22 @@ public void onPausedChanged(boolean paused) { // in the order they are added. eventFilter = new EventFilter(this, compositor, touchMonitor, globalVariables, earlyFocusSpeech); + screenFocusRestorer = + new ScreenFocusRestorer( + this, prefs, pipeline.getFeedbackReturner(), pipeline.getActorState().getFocusHistory()); + userInterface.registerListener(screenFocusRestorer); + focuser.setBeforeActivationListener(screenFocusRestorer::rememberBeforeActivation); focuser.setFocusSetListener( - (node, info, eventId, actionTime) -> - eventFilter.onAccessibilityFocusSet( - node, - info, - eventId, - actionTime, - /* continuousReading= */ fullScreenReadActor != null - && fullScreenReadActor.isActive())); + (node, info, eventId, actionTime) -> { + eventFilter.onAccessibilityFocusSet( + node, + info, + eventId, + actionTime, + /* continuousReading= */ fullScreenReadActor != null + && fullScreenReadActor.isActive()); + screenFocusRestorer.onFocusSet(node, info, actionTime); + }); eventFilter.setTargetPredictor(directionNavigationActor::predictTarget); eventFilter.setFingerDownSupplier( () -> @@ -3808,6 +3850,9 @@ private void applyTouchExplorationPreference() { /** Turns off, or back on, what pausing Backtalk turns off besides events and feedback. */ private void onPauseChanged(boolean paused) { + if (paused && screenFocusRestorer != null) { + screenFocusRestorer.onUserInteraction(); + } if (paused) { if (supportsTouchScreen) { requestTouchExploration(false); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActor.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActor.java index f92c5b9ff..56023bb59 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActor.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActor.java @@ -61,6 +61,7 @@ import com.google.android.accessibility.utils.WebInterfaceUtils; import com.google.android.libraries.accessibility.utils.log.LogUtils; import java.util.List; +import java.util.function.Consumer; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -86,6 +87,7 @@ public class FocusActor implements UserInputEventListener { private Pipeline.FeedbackReturner pipeline; private ListMenuManager menuManager; private GlobalVariables globalVariables; + private @Nullable Consumer beforeActivationListener; ////////////////////////////////////////////////////////////////////////////////////////////////// // Construction @@ -124,6 +126,11 @@ public void setFocusSetListener(FocusManagerInternal.@Nullable FocusSetListener focusManagerInternal.setFocusSetListener(listener); } + public void setBeforeActivationListener( + @Nullable Consumer listener) { + beforeActivationListener = listener; + } + public void setMenuManager(ListMenuManager menuManager) { this.menuManager = menuManager; } @@ -145,6 +152,9 @@ public boolean clickNode(AccessibilityNodeInfoCompat node, EventId eventId) { if (node == null || pipeline == null) { return false; } + if (beforeActivationListener != null) { + beforeActivationListener.accept(node); + } node = Scripts.actionTarget(node, false); if (globalVariables.supportClickableLinks() && tryClickLinks(node, eventId)) { @@ -189,6 +199,9 @@ public boolean longClickNode(AccessibilityNodeInfoCompat node, EventId eventId) if (node == null || pipeline == null) { return false; } + if (beforeActivationListener != null) { + beforeActivationListener.accept(node); + } return pipeline.returnFeedback( eventId, Feedback.nodeAction(Scripts.actionTarget(node, true), ACTION_LONG_CLICK.getId())); @@ -200,6 +213,9 @@ public boolean clickCurrentHierarchical(@Nullable EventId eventId) { if (currentFocus == null) { return false; } + if (beforeActivationListener != null) { + beforeActivationListener.accept(currentFocus); + } AccessibilityNodeInfoCompat nodeToClick = AccessibilityNodeInfoUtils.getSelfOrMatchingAncestor( currentFocus, AccessibilityNodeInfoUtils.FILTER_CLICKABLE); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/SystemActionPerformer.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/SystemActionPerformer.java index c31b91645..31233b822 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/SystemActionPerformer.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/SystemActionPerformer.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; +import java.util.function.IntConsumer; /** System/global action performer */ public final class SystemActionPerformer { @@ -50,12 +51,16 @@ public final class SystemActionPerformer { /** Actor-state passed in from pipeline. */ private ActorStateWritable actorState; + private IntConsumer beforeActionListener; public SystemActionPerformer(AccessibilityService service) { this.service = service; } public boolean performAction(int id) { + if (beforeActionListener != null) { + beforeActionListener.accept(id); + } if (actorState != null) { actorState.setLastSystemAction(id); } @@ -83,4 +88,8 @@ public boolean performAction(int id) { public void setActorState(ActorStateWritable actorState) { this.actorState = actorState; } + + public void setBeforeActionListener(IntConsumer listener) { + beforeActionListener = listener; + } } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusBookmarks.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusBookmarks.kt new file mode 100644 index 000000000..63ee6128e --- /dev/null +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusBookmarks.kt @@ -0,0 +1,127 @@ +/* Copyright 2026 Backtalk contributors. Licensed under the Apache License, Version 2.0. */ +package com.google.android.accessibility.talkback.focusmanagement + +/** Immutable values only: bookmarks must survive recycled views and never keep old node trees. */ +internal data class RememberedScreen( + val packageName: String, + val title: String = "", + val pane: String = "", + val document: String = "", + val structure: List = emptyList(), + val context: List = emptyList(), +) + +internal data class RememberedItem( + val uniqueId: String = "", + val viewId: String = "", + val className: String = "", + val role: Int = 0, + val text: String = "", + val description: String = "", + val ancestors: List = emptyList(), + val previous: String = "", + val next: String = "", + val container: String = "", + val row: Int = -1, + val column: Int = -1, +) { + val hasContent: Boolean get() = text.isNotEmpty() || description.isNotEmpty() + + /** Indices, bounds, and source node IDs are deliberately not evidence of item identity. */ + fun matchScore(other: RememberedItem): Int { + if (className != other.className || role != other.role) return 0 + if (viewId.isNotEmpty() && other.viewId.isNotEmpty() && viewId != other.viewId) return 0 + if (hasContent && (text != other.text || description != other.description)) return 0 + if (uniqueId.isNotEmpty() && uniqueId == other.uniqueId) return 100 + if (!hasContent || !other.hasContent) return 0 + val context = ancestors.isNotEmpty() && ancestors == other.ancestors + // An adjacent duplicate of the same title cannot distinguish which duplicate this is. + val before = previous.isNotEmpty() && previous != text && previous == other.previous + val after = next.isNotEmpty() && next != text && next == other.next + if (!context && !before && !after) return 0 + return 1 + (if (context) 2 else 0) + (if (before) 2 else 0) + (if (after) 2 else 0) + } +} + +internal data class ScreenBookmark( + val screen: RememberedScreen, + val anchors: Set, + val item: RememberedItem, +) + +/** Session-only history and cancellation/deadline policy, independent of Android node lifetimes. */ +internal class ScreenFocusBookmarks { + companion object { + const val MAX_BOOKMARKS = 20 + const val RESTORE_TIMEOUT_MS = 3000L + const val MAX_SCROLL_ACTIONS = 10 + } + + private val bookmarks = LinkedHashMap(16, 0.75f, true) + var activeScreen: RememberedScreen? = null + private set + private var pendingSince: Long? = null + private var scrollActions = 0 + private var lastViewport: List? = null + val size: Int get() = bookmarks.size + fun contains(screen: RememberedScreen): Boolean = bookmarks.containsKey(screen) + + fun remember(screen: RememberedScreen, anchors: Set, item: RememberedItem) { + if (!item.hasContent && item.uniqueId.isEmpty()) return + bookmarks[screen] = ScreenBookmark(screen, anchors.toSet(), item) + while (bookmarks.size > MAX_BOOKMARKS) bookmarks.remove(bookmarks.keys.first()) + } + + fun enter(screen: RememberedScreen?, now: Long, explicitTransition: Boolean = false, + allowRestore: Boolean = true) { + if (screen == activeScreen && !explicitTransition) return + activeScreen = screen + pendingSince = if (allowRestore && screen != null && bookmarks.containsKey(screen)) now else null + scrollActions = 0 + lastViewport = null + } + + fun pending(now: Long): ScreenBookmark? { + val since = pendingSince ?: return null + if (now < since || now - since >= RESTORE_TIMEOUT_MS) { + cancel() + return null + } + return bookmarks[activeScreen] + } + + fun remaining(now: Long): Long = + pendingSince?.let { (RESTORE_TIMEOUT_MS - (now - it)).coerceAtLeast(0) } ?: 0 + + fun find(bookmark: ScreenBookmark, items: List): Int? { + val scored = items.mapIndexed { index, item -> index to bookmark.item.matchScore(item) } + val best = scored.maxOfOrNull { it.second } ?: return null + if (best == 0) return null + return scored.filter { it.second == best }.singleOrNull()?.first + } + + /** Scrolling needs independent screen evidence because the target is not yet available. */ + fun mayScroll(bookmark: ScreenBookmark, anchors: Set): Boolean = + bookmark.anchors.intersect(anchors).isNotEmpty() + + fun beginScroll(now: Long, viewport: List): Boolean { + if (pending(now) == null || scrollActions >= MAX_SCROLL_ACTIONS || lastViewport == viewport) { + cancel() + return false + } + scrollActions++ + lastViewport = viewport.toList() + return true + } + + fun cancel() { + pendingSince = null + } + + fun clear() { + cancel() + bookmarks.clear() + activeScreen = null + lastViewport = null + } +} diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt new file mode 100644 index 000000000..68752a828 --- /dev/null +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt @@ -0,0 +1,728 @@ +/* Copyright 2026 Backtalk contributors. Licensed under the Apache License, Version 2.0. */ +package com.google.android.accessibility.talkback.focusmanagement + +import android.content.SharedPreferences +import android.graphics.Rect +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.view.KeyEvent +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat.AccessibilityActionCompat +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.talkback.TalkBackService +import com.google.android.accessibility.talkback.UserInterface.UserInputEventListener +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory +import com.google.android.accessibility.talkback.focusmanagement.record.FocusActionInfo +import com.google.android.accessibility.talkback.pause.PauseController +import com.google.android.accessibility.utils.AccessibilityNodeInfoUtils +import com.google.android.accessibility.utils.FormFactorUtils +import com.google.android.accessibility.utils.Performance +import com.google.android.accessibility.utils.Role +import com.google.android.accessibility.utils.WebInterfaceUtils +import com.google.android.libraries.accessibility.utils.log.LogUtils +import java.security.MessageDigest + +/** + * Complements window history with content bookmarks. Runs after normal initial-focus processing, + * so a returning screen can replace automatic toolbar focus without changing dialog/IME policy. + * Session bookmarks retain values only; a pending scroll briefly retains its action target. + */ +class ScreenFocusRestorer internal constructor( + private val service: TalkBackService, + private val prefs: SharedPreferences, + private val pipeline: Pipeline.FeedbackReturner, + private val focusHistory: AccessibilityFocusActionHistory.Reader, + private val history: ScreenFocusBookmarks, + private val available: () -> Boolean, + private val readContext: (() -> Context?)?, +) : UserInputEventListener { + constructor(service: TalkBackService, prefs: SharedPreferences, + pipeline: Pipeline.FeedbackReturner, focusHistory: AccessibilityFocusActionHistory.Reader) : + this(service, prefs, pipeline, focusHistory, ScreenFocusBookmarks(), + { TalkBackService.isServiceActive() && !service.actorState.continuousRead.isActive }, null) + companion object { + const val SCROLL_PREFERENCE = "pref_scroll_to_restore_previous_focus" + private const val TAG = "ScreenFocusRestorer" + private const val SETTLE_DELAY_MS = 120L + private const val MAX_SEARCH_NODES = 1000 + // Native address controls already recognized by WebInterfaceUtils; search only the skeleton, + // never an entire document just because its browser toolbar is temporarily hidden. + private val ADDRESS_IDS = setOf("url_bar", "url", "url_bar_title", "url_edit_text", + "mozac_browser_toolbar_url_view", "mozac_browser_toolbar_edit_url_view", + "location_bar_edit_text", "url_field") + } + + private val handler = Handler(Looper.getMainLooper()) + private var transitionQueued = false + private var restoring = false + private var scrollNode: AccessibilityNodeInfoCompat? = null + private var scrollTime = 0L + private var triedPosition = false + private var waitingForScroll = false + private var stopped = false + private var cancelledBeforeEnter = false + private var cancelledScreen: RememberedScreen? = null + private var lastWindowId: Int? = null + private var lastRecordedFocusTime = Long.MIN_VALUE + private var backRequestedAt: Long? = null + private var backFrom: RememberedScreen? = null + private var backWindowId: Int? = null + private var activationOrigin: RememberedScreen? = null + private var retryAt = Long.MAX_VALUE + private val retry = Runnable { retryAt = Long.MAX_VALUE; attemptRestore() } + private val releaseScroll = Runnable { scrollNode = null; waitingForScroll = false } + + internal data class Context( + val root: AccessibilityNodeInfoCompat, + val screen: RememberedScreen, + val anchors: Set, + ) + + override fun touchInteractionState(active: Boolean) { + if (active) onUserInteraction() + } + + fun onKeyEvent(event: KeyEvent) { + if (event.action == KeyEvent.ACTION_DOWN && event.keyCode == KeyEvent.KEYCODE_BACK) { + requestBack() + return + } + if (event.action == KeyEvent.ACTION_DOWN && event.keyCode != KeyEvent.KEYCODE_POWER && + event.keyCode != KeyEvent.KEYCODE_WAKEUP && event.keyCode != KeyEvent.KEYCODE_SLEEP) { + onUserInteraction() + } + } + + fun onUserInteraction() { + backRequestedAt = null + if (transitionQueued || retryAt != Long.MAX_VALUE) { + cancelledBeforeEnter = true + cancelledScreen = context()?.screen + } + history.cancel() + handler.removeCallbacks(retry) + handler.removeCallbacks(releaseScroll) + retryAt = Long.MAX_VALUE + transitionQueued = false + scrollNode = null + waitingForScroll = false + } + + /** Also called synchronously before the app's event for a successful focus action arrives. */ + fun onFocusSet(node: AccessibilityNodeInfoCompat, info: FocusActionInfo, + actionTime: Long = SystemClock.uptimeMillis()) { + if (stopped || restoring) return + val active = context() + if (active == null || node.packageName != active.root.packageName || + node.windowId != active.root.windowId) { + onUserInteraction() + return + } + if (info.sourceAction == FocusActionInfo.SCREEN_STATE_CHANGE) { + // Normal initial focus may run again after our successful restore during the first load. + // Keep the original bounded attempt alive to replace that late automatic toolbar focus. + if (history.pending(SystemClock.uptimeMillis()) != null) schedule() + return + } + if (!isUserFocus(info)) return + // Direct show/scroll actions can be followed by the manual-scroll interpreter's focus action. + // Real input clears scrollNode first, so this exemption cannot override a user's gesture. + if (info.sourceAction == FocusActionInfo.MANUAL_SCROLL && + (scrollNode != null || isReturningBrowserContent(node))) { + lastRecordedFocusTime = maxOf(lastRecordedFocusTime, actionTime) + if (scrollNode == null) schedule() + return + } + // A focus action can precede attachment/reconstruction of its node. Only deduplicate its + // later event after recording actually succeeds, allowing that event to finish the first save. + if (recordPosition(node)) { + lastRecordedFocusTime = maxOf(lastRecordedFocusTime, actionTime) + activationOrigin = null + } + } + + /** Capture initial/app-assigned focus too when the user activates it without first swiping. */ + fun rememberBeforeActivation(node: AccessibilityNodeInfoCompat) { + if (isBackControl(node)) { + requestBack() + return + } + val recorded = recordPosition(node) + if (!recorded && !isDocument(node)) return + if (recorded) focusHistory.lastFocusActionRecord?.let { record -> + if (record.focusedNode == node) { + lastRecordedFocusTime = maxOf(lastRecordedFocusTime, record.actionTime) + } + } + activationOrigin = context()?.screen + } + + private fun isBackControl(node: AccessibilityNodeInfoCompat): Boolean { + if (Role.getRole(node) != Role.ROLE_BUTTON && Role.getRole(node) != Role.ROLE_IMAGE_BUTTON) return false + return node.viewIdResourceName == "android:id/home" || + node.viewIdResourceName?.substringAfterLast('/') in setOf("back", "back_button", "navigate_up") || + label(node) in setOf("Back", "Navigate up") + } + + private fun recordPosition(node: AccessibilityNodeInfoCompat): Boolean { + if (stopped) return false + val context = context() ?: return false + if (node.windowId != context.root.windowId || node.packageName != context.root.packageName) return false + if (!node.refresh() || (node != context.root && + !AccessibilityNodeInfoUtils.hasAncestor(node, context.root))) { + return false + } + if (node.isPassword || node.isEditable || + (isDocument(node) && (Role.getRole(node) == Role.ROLE_WEB_VIEW || + (!node.isHeading && !node.isClickable && node.childCount > 0)))) { + return false + } + val item = describe(node) + if (!item.hasContent && item.uniqueId.isEmpty()) return false + onUserInteraction() + history.enter(context.screen, SystemClock.uptimeMillis(), allowRestore = false) + history.remember(context.screen, context.anchors, item) + lastWindowId = context.root.windowId + return true + } + + /** Only explicit Back (gesture, key, or app Back button) authorizes a restoration attempt. */ + fun requestBack() { + val departing = context() + // User focus and activation already record positions. Reading native focus here can return + // a WebView container instead of the actual HTML heading and erase the browser bookmark. + onUserInteraction() + backFrom = if (departing?.screen == activationOrigin) null else departing?.screen + backWindowId = if (backFrom != null) departing?.root?.windowId else null + activationOrigin = null + backRequestedAt = SystemClock.uptimeMillis() + cancelledBeforeEnter = false + cancelledScreen = null + schedule() + } + + private fun isUserFocus(info: FocusActionInfo): Boolean = when (info.sourceAction) { + FocusActionInfo.TOUCH_EXPLORATION, FocusActionInfo.LOGICAL_NAVIGATION, + FocusActionInfo.MANUAL_SCROLL, FocusActionInfo.KEYBOARD_SHORTCUT_REFOCUS -> true + else -> false + } + + fun onAccessibilityEvent(event: AccessibilityEvent) { + if (stopped) return + when (event.eventType) { + AccessibilityEvent.TYPE_TOUCH_INTERACTION_START, + AccessibilityEvent.TYPE_VIEW_HOVER_ENTER -> onUserInteraction() + AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED, AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED -> { + val source = event.source?.let(AccessibilityNodeInfoCompat::wrap) + // Browsers update their address/title controls while reconstructing a history entry. + // These notifications are not typing. Touch, keys and navigation cancel separately; + // an editor receiving input still cancels immediately. + if (source != null && !source.isFocused && isReturningBrowserContent(source, address = true)) { + schedule() + } else onUserInteraction() + } + AccessibilityEvent.TYPE_VIEW_CLICKED -> { + val source = event.source?.let(AccessibilityNodeInfoCompat::wrap) + // This is the event produced by the Back button whose activation armed the request. + if (backRequestedAt == null || source == null || !isBackControl(source)) onUserInteraction() + } + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED -> { + val focused = event.source?.let(AccessibilityNodeInfoCompat::wrap) + val active = context() + if (focused != null && (active == null || focused.packageName != active.root.packageName || + focused.windowId != active.root.windowId)) { + onUserInteraction() + return + } + // Firefox's HTML navigation is asynchronous and bypasses the synchronous focus-set hook. + val record = focusHistory.matchFocusActionRecordFromEvent(event) ?: run { + // Apps can assign focus without going through our focus actor. Genuine touch/key/ + // navigation input already cancels this bounded attempt through the input hooks. + if (history.pending(SystemClock.uptimeMillis()) != null) schedule() + return + } + // Native actions were recorded synchronously. Their delayed events must not cancel a + // return attempt or save the old item against a destination in the same window. + if (record.actionTime <= lastRecordedFocusTime) return + val source = event.source ?: return + onFocusSet(AccessibilityNodeInfoCompat.wrap(source), record.extraInfo, record.actionTime) + } + AccessibilityEvent.TYPE_WINDOWS_CHANGED -> { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + val navigationChanges = AccessibilityEvent.WINDOWS_CHANGE_ADDED or + AccessibilityEvent.WINDOWS_CHANGE_REMOVED or AccessibilityEvent.WINDOWS_CHANGE_ACTIVE or + AccessibilityEvent.WINDOWS_CHANGE_TITLE + if (event.windowChanges and navigationChanges != 0) queueTransition() else schedule() + } else schedule() + } + AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED -> queueTransition() + AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED -> schedule() + AccessibilityEvent.TYPE_VIEW_SCROLLED -> { + val source = event.source?.let(AccessibilityNodeInfoCompat::wrap) + if (scrollNode != null && source != null && + (source == scrollNode || AccessibilityNodeInfoUtils.hasAncestor(scrollNode, source)) && + SystemClock.uptimeMillis() - scrollTime < 1000L) { + waitingForScroll = false + schedule() + } else if (isReturningBrowserScroll(event, source)) { + // Browser Back restores the document's viewport itself, even with our optional scroll + // setting disabled. Do not lose the Back authorization to that automatic scroll event. + // A user's scroll gesture/key already cancels through the input hooks before this event. + schedule() + } else { + // App/manual scrolls must not be mistaken for restoration scrolls. + onUserInteraction() + } + } + else -> Unit // Other event types remain with the existing focus and announcement policies. + } + } + + private fun isReturningBrowserContent(node: AccessibilityNodeInfoCompat, + address: Boolean = false): Boolean { + val active = returningBrowserContext() ?: return false + if (node.packageName != active.root.packageName || node.windowId != active.root.windowId) return false + if (address && node.viewIdResourceName?.substringAfterLast('/') in ADDRESS_IDS) return true + // Edge reports automatic history-entry scrolling on native containers above the WebView, + // as well as on HTML nodes. The existing Back grant and document identity scope this to the + // browser window; actual user input has already revoked that grant through the input hooks. + if (!address && (node == active.root || AccessibilityNodeInfoUtils.hasAncestor(node, active.root))) { + return true + } + var ancestor: AccessibilityNodeInfoCompat? = node + repeat(20) { + val current = ancestor ?: return false + if (isDocument(current)) return true + ancestor = current.parent + } + return false + } + + private fun isReturningBrowserScroll(event: AccessibilityEvent, + source: AccessibilityNodeInfoCompat?): Boolean { + if (source != null) return isReturningBrowserContent(source) + val active = returningBrowserContext() ?: return false + // A browser may clear the source while rebuilding its virtual tree. Window and package must + // still agree; source-less events from System UI or another app continue to cancel the attempt. + return event.packageName == active.root.packageName && event.windowId == active.root.windowId + } + + private fun returningBrowserContext(): Context? { + val now = SystemClock.uptimeMillis() + if (history.pending(now) == null && + backRequestedAt?.let { now - it < ScreenFocusBookmarks.RESTORE_TIMEOUT_MS } != true) return null + val active = context() ?: return null + if (active.screen.document.isEmpty() && (backFrom?.packageName != active.screen.packageName || + backFrom?.document.isNullOrEmpty())) return null + return active + } + + private fun queueTransition() { + transitionQueued = true + // Window/title events may repeat throughout reconstruction. Only a changed screen or + // recreated window starts a new attempt; duplicate events cannot reset its deadline or input veto. + // Observe departures immediately, even if the other app/page is only open for less than + // the settling delay. Restoration itself still waits for normal initial focus to finish. + observeScreen(context(), SystemClock.uptimeMillis()) + schedule() + } + + private fun observeScreen(context: Context?, now: Long) { + if (activationOrigin != null && context?.screen != activationOrigin) activationOrigin = null + val requestedAt = backRequestedAt + val validBack = requestedAt != null && now - requestedAt < ScreenFocusBookmarks.RESTORE_TIMEOUT_MS + if (!validBack) backRequestedAt = null + val returning = validBack && context != null && + (context.screen != backFrom || context.root.windowId != backWindowId) && + history.contains(context.screen) + val changed = context?.screen != history.activeScreen || + (transitionQueued && lastWindowId != null && context?.root?.windowId != lastWindowId) + history.enter(context?.screen, if (returning) requestedAt else now, + changed || returning, allowRestore = returning) + if (returning) backRequestedAt = null + lastWindowId = context?.root?.windowId + if (changed) { + // Input on the departing screen must not veto a later return. Input already performed + // on the destination still cancels restoration, including content-only transitions. + if (cancelledBeforeEnter && cancelledScreen != context?.screen) { + cancelledBeforeEnter = false + cancelledScreen = null + } + triedPosition = false + scrollNode = null + waitingForScroll = false + } + } + + private fun schedule(delay: Long = SETTLE_DELAY_MS) { + // Do not debounce forever when pages continuously update. + val at = SystemClock.uptimeMillis() + delay + if (at >= retryAt) return + handler.removeCallbacks(retry) + retryAt = at + handler.postDelayed(retry, delay) + } + + private fun attemptRestore() { + if (stopped || PauseController.isPaused() || !available()) { + onUserInteraction() + return + } + val now = SystemClock.uptimeMillis() + val context = context() + observeScreen(context, now) + transitionQueued = false + if (cancelledBeforeEnter) { + history.cancel() + cancelledBeforeEnter = false + cancelledScreen = null + } + if (context == null) { + scrollNode = null + waitingForScroll = false + return + } + val bookmark = history.pending(now) ?: run { + scrollNode = null + waitingForScroll = false + backRequestedAt?.let { schedule((ScreenFocusBookmarks.RESTORE_TIMEOUT_MS - (now - it)).coerceAtLeast(0)) } + return + } + val nodes = walk(context.root, MAX_SEARCH_NODES) + if (nodes.any { it.isFocused && it.isEditable }) { + history.cancel() + scrollNode = null + waitingForScroll = false + return + } + if (waitingForScroll && now - scrollTime >= SETTLE_DELAY_MS * 2) waitingForScroll = false + val candidates = nodes.filter { !it.isPassword && !it.isEditable && + it.className?.toString().orEmpty() == bookmark.item.className && + Role.getRole(it) == bookmark.item.role && + (it.uniqueId?.let { id -> id.isNotEmpty() && id == bookmark.item.uniqueId } == true || + (bookmark.item.hasContent && itemText(it) == bookmark.item.text)) } + val items = candidates.map(::describe) + val viewport = nodes.filter { visibleOnScreen(it, context.root) && + it.parent?.let(::containerId) == bookmark.item.container } + .map { RememberedItem(text = itemText(it), row = it.collectionItemInfo?.rowIndex ?: -1, + column = it.collectionItemInfo?.columnIndex ?: -1) } + val match = history.find(bookmark, items) + var matchedOffscreen = false + if (match != null) { + val node = candidates[match] + if (node.refresh() && bookmark.item.matchScore(describe(node)) > 0 && + (bookmark.item.uniqueId.isNotEmpty() || bookmark.screen.document.isNotEmpty() || + bookmark.anchors.isEmpty() || history.mayScroll(bookmark, context.anchors)) && + node.windowId == context.root.windowId && + AccessibilityNodeInfoUtils.hasAncestor(node, context.root)) { + if (history.pending(SystemClock.uptimeMillis()) == null || + context()?.screen != context.screen) { + history.cancel() + return + } + matchedOffscreen = !visibleOnScreen(node, context.root) + if (visibleOnScreen(node, context.root) && + AccessibilityNodeInfoUtils.shouldFocusNode(node)) { + restoring = true + try { + val info = FocusActionInfo.builder() + .setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE) + .setInitialFocusType(FocusActionInfo.RESTORED_LAST_FOCUS) + .build() + if (node.isAccessibilityFocused || pipeline.returnFeedback( + Performance.EVENT_ID_UNTRACKED, Feedback.focus(node, info))) { + scrollNode = null + schedule(history.remaining(SystemClock.uptimeMillis())) + LogUtils.d(TAG, "Restored remembered item in %s window %d", + context.screen.packageName, node.windowId) + return + } + } finally { + restoring = false + } + } else if (matchedOffscreen && scrollEnabled() && history.mayScroll(bookmark, context.anchors) && + supports(node, AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN.id) && + !waitingForScroll && history.beginScroll(now, viewport)) { + if (requestScroll(node, AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN.id, null, + context)) return + history.cancel() + return + } + } + } + val rows = viewport.filter { it.row >= 0 } + val savedPositionOutsideViewport = bookmark.item.row >= 0 && rows.isNotEmpty() && + (bookmark.item.row < rows.minOf { it.row } || bookmark.item.row > rows.maxOf { it.row }) + // A missing/changed label does not prove the item is offscreen. Apps can rebuild a visible + // row after marking an item read. Position is only evidence against scrolling, + // never enough evidence to focus an item. A visible candidate also rules out moving the list + // after focus fails, or while duplicate labels/context prevent selecting a safe target. + val visibleCandidate = candidates.any { visibleOnScreen(it, context.root) } + if (scrollEnabled() && !visibleCandidate && (matchedOffscreen || savedPositionOutsideViewport) && + !waitingForScroll && history.mayScroll(bookmark, context.anchors)) { + val containers = nodes.filter { containerId(it) == bookmark.item.container && + bookmark.item.container.isNotEmpty() && (it.isScrollable || it.collectionInfo != null) } + val container = containers.singleOrNull() + if (container != null && history.beginScroll(now, viewport)) { + if (!triedPosition && bookmark.item.row >= 0 && supports(container, + AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION.id)) { + triedPosition = true + val args = Bundle().apply { + putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_ROW_INT, bookmark.item.row) + if (bookmark.item.column >= 0) { + putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_COLUMN_INT, bookmark.item.column) + } + } + if (requestScroll(container, AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION.id, + args, context)) return + // A rejected action terminates this attempt instead of issuing a second action against + // the same viewport (which would also bypass the ten-action budget). + history.cancel() + return + } + val backwards = bookmark.item.row >= 0 && rows.isNotEmpty() && + bookmark.item.row < rows.minOf { it.row } + val action = if (backwards) AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD + else AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD + if (requestScroll(container, action, null, context)) return + history.cancel() + } + } + // Content events trigger retries; this one deadline callback releases pending state even if + // the app never sends another event. Normal initial focus already remains available. + val finishedAt = SystemClock.uptimeMillis() + if (history.pending(finishedAt) != null) schedule(history.remaining(finishedAt)) + } + + private fun scrollEnabled(): Boolean = prefs.getBoolean(SCROLL_PREFERENCE, false) + + private fun visibleOnScreen(node: AccessibilityNodeInfoCompat, + root: AccessibilityNodeInfoCompat): Boolean { + if (!node.isVisibleToUser || !AccessibilityNodeInfoUtils.isVisible(node)) return false + val bounds = Rect() + val viewport = Rect() + node.getBoundsInScreen(bounds) + val window = root.window + if (window != null) window.getBoundsInScreen(viewport) else root.getBoundsInScreen(viewport) + return !bounds.isEmpty && Rect.intersects(bounds, viewport) + } + + private fun supports(node: AccessibilityNodeInfoCompat, action: Int): Boolean = + node.actionList.any { it.id == action } + + private fun requestScroll(node: AccessibilityNodeInfoCompat, action: Int, args: Bundle?, + expected: Context): Boolean { + if (history.pending(SystemClock.uptimeMillis()) == null) return false + if (context()?.screen != expected.screen || !node.refresh() || + (node != expected.root && !AccessibilityNodeInfoUtils.hasAncestor(node, expected.root))) { + history.cancel() + return false + } + if (!supports(node, action)) return false + if (history.pending(SystemClock.uptimeMillis()) == null) return false + scrollNode = AccessibilityNodeInfoCompat.obtain(node) + scrollTime = SystemClock.uptimeMillis() + waitingForScroll = true + // A no-progress retry can cancel history before the deadline callback is scheduled. Release + // the temporary node independently, while recognizing late automatic scroll focus until then. + handler.removeCallbacks(releaseScroll) + handler.postDelayed(releaseScroll, history.remaining(scrollTime)) + val success = pipeline.returnFeedback(Performance.EVENT_ID_UNTRACKED, + Feedback.nodeAction(node, action, args)) + if (!success) { + handler.removeCallbacks(releaseScroll) + waitingForScroll = false + scrollNode = null + } else { + // Some apps perform the action without a scroll event. Re-read the tree, then stop if it + // has not progressed. Never focus a guessed row after requesting a positional scroll. + schedule(SETTLE_DELAY_MS * 2) + } + return success + } + + private fun context(): Context? { + if (readContext != null) return readContext.invoke() + if (FormFactorUtils.isAndroidTv()) return null + val root = service.rootInActiveWindow?.let(AccessibilityNodeInfoCompat::wrap) ?: return null + val window = root.window ?: return null + if (window.type != AccessibilityWindowInfo.TYPE_APPLICATION) return null + val packageName = root.packageName?.toString().orEmpty() + if (packageName.isEmpty() || packageName == service.packageName) return null + return captureContext(root, window.title?.toString().orEmpty()) + } + + internal fun captureContext(root: AccessibilityNodeInfoCompat, title: String): Context { + // Stop at content containers: identity must not depend on which recycled rows are visible. + val skeleton = walk(root, 100, stopAtContent = true) + val documents = skeleton.filter(::isDocument) + val documentTitle = documents.firstOrNull()?.let(::label).orEmpty() + val url = skeleton.firstOrNull { + it.viewIdResourceName?.substringAfterLast('/') in ADDRESS_IDS + }?.takeUnless { it.isPassword }?.text?.toString().orEmpty().let(::boundedIdentity) + val document = if (documentTitle.isNotEmpty() || url.isNotEmpty()) "$documentTitle|$url" else "" + val pane = skeleton.mapNotNull { it.paneTitle?.toString()?.takeIf(String::isNotEmpty) } + .distinct().joinToString("|") + val structure = skeleton.filter { it.collectionInfo != null || it.isScrollable || + isDocument(it) }.map(::containerId).distinct().sorted() + val headings = skeleton.filter { node -> node.isHeading || + node.viewIdResourceName?.substringAfterLast('/') in + setOf("title", "action_bar_title", "toolbar_title") } + .map(::label).filter(String::isNotEmpty).distinct() + val anchors = skeleton.mapNotNull { node -> + val text = label(node) + if (text.isNotEmpty() && !node.isEditable && !node.isPassword) { + "${containerId(node)}:$text" + } else null + }.toSet() + return Context(root, RememberedScreen(root.packageName?.toString().orEmpty(), title, + pane, document, structure, headings), anchors) + } + + private fun walk(root: AccessibilityNodeInfoCompat, limit: Int, + stopAtContent: Boolean = false): List { + val queue = ArrayDeque() + val visited = HashSet() + val result = ArrayList() + queue.add(root) + while (queue.isNotEmpty() && result.size < limit) { + val node = queue.removeFirst() + if (!visited.add(node)) continue + result.add(node) + if (node.isPassword || node.isEditable) continue + if (stopAtContent && (node.collectionInfo != null || node.isScrollable || + isDocument(node))) continue + for (i in 0 until minOf(node.childCount, limit - result.size - queue.size)) { + node.getChild(i)?.let(queue::addLast) + } + } + return result + } + + private fun label(node: AccessibilityNodeInfoCompat): String = + (node.text?.toString()?.takeIf(String::isNotBlank) + ?: node.contentDescription?.toString().orEmpty()).trim().let(::boundedIdentity) + + // A shared prefix is not an item identity. Retain a digest of long labels instead of silently + // conflating posts/documents that differ only beyond the memory bound. + private fun boundedIdentity(value: String): String { + if (value.length <= 512) return value + val digest = MessageDigest.getInstance("SHA-256").digest(value.toByteArray(Charsets.UTF_8)) + .joinToString("") { "%02x".format(it) } + return value.take(440) + "#" + digest + } + + private fun isDocument(node: AccessibilityNodeInfoCompat): Boolean = + Role.getRole(node) == Role.ROLE_WEB_VIEW || + (WebInterfaceUtils.supportsWebActions(node) && + !WebInterfaceUtils.supportsWebActions(node.parent)) + + private fun containerId(node: AccessibilityNodeInfoCompat): String = + "${node.className}|${node.viewIdResourceName.orEmpty()}" + + internal fun describe(node: AccessibilityNodeInfoCompat): RememberedItem { + val ancestors = ArrayList() + var container = "" + var row = node.collectionItemInfo?.rowIndex ?: -1 + var column = node.collectionItemInfo?.columnIndex ?: -1 + var ancestor = node.parent + var child = node + var previous = "" + var next = "" + repeat(8) { + val parent = ancestor ?: return@repeat + ancestors.add(containerId(parent)) + if (container.isEmpty() && row < 0 && parent.collectionItemInfo != null) { + row = parent.collectionItemInfo.rowIndex + column = parent.collectionItemInfo.columnIndex + } + val nearestContainer = container.isEmpty() && + (parent.collectionInfo != null || parent.isScrollable) + if (nearestContainer) { + container = containerId(parent) + } + // Use siblings of the item wrapper as well as siblings of its focused child. + if (it == 0 || nearestContainer) { + for (index in 0 until minOf(parent.childCount, 100)) { + if (parent.getChild(index) == child) { + if (index > 0) previous = parent.getChild(index - 1)?.let(::itemText).orEmpty() + if (index + 1 < parent.childCount) { + next = parent.getChild(index + 1)?.let(::itemText).orEmpty() + } + break + } + } + } + child = parent + ancestor = parent.parent + } + val primaryTitle = primaryItemTitle(node) + return RememberedItem(node.uniqueId.orEmpty(), node.viewIdResourceName.orEmpty(), + node.className?.toString().orEmpty(), Role.getRole(node), primaryTitle ?: itemText(node), + if (primaryTitle != null) "" else boundedIdentity(node.contentDescription?.toString().orEmpty()), + ancestors, previous, next, + container, row, column) + } + + private fun primaryItemTitle(node: AccessibilityNodeInfoCompat): String? { + if (node.isPassword || node.isEditable || node.isScrollable || node.collectionInfo != null || + isDocument(node) || WebInterfaceUtils.supportsWebActions(node)) return null + // Only normalize native rows/cards, never a screen or toolbar containing many items. + var ancestor = node.parent + var inRow = node.collectionItemInfo != null + repeat(8) { + val parent = ancestor ?: return@repeat + if (isDocument(parent) || WebInterfaceUtils.supportsWebActions(parent) || + parent.packageName != node.packageName) return null + if (parent.collectionItemInfo != null || parent.collectionInfo != null || parent.isScrollable) { + inRow = true + ancestor = null + } else { + ancestor = parent.parent + } + } + val isItem = inRow || + (Role.getRole(node) == Role.ROLE_VIEW_GROUP && + (node.isClickable || node.isScreenReaderFocusable)) + if (!isItem) return null + val fields = walk(node, 32) + // An incomplete walk cannot establish that the title is unique. + if (fields.size >= 32 || fields.drop(1).any { + it.collectionItemInfo != null || it.collectionInfo != null || it.isScrollable + }) return null + val title = fields.filter { + val id = it.viewIdResourceName?.substringAfterLast('/').orEmpty().lowercase(java.util.Locale.ROOT) + it.packageName == node.packageName && !it.isPassword && !it.isEditable && + !WebInterfaceUtils.supportsWebActions(it) && !it.text.isNullOrBlank() && + !id.contains("toolbar") && !id.contains("action_bar") && !id.contains("actionbar") && + (it.isHeading || id == "title" || id.endsWith("_title")) + }.singleOrNull() ?: return null + // Keep the exposed title verbatim. Read-state descriptions and count/age subtitles may + // change; stripping words such as "read" would corrupt real titles and localized labels. + return boundedIdentity(title.text.toString()) + } + + private fun itemText(node: AccessibilityNodeInfoCompat): String { + if (node.isPassword || node.isEditable) return "" + primaryItemTitle(node)?.let { return it } + val own = label(node) + if (own.isNotEmpty()) return own + return walk(node, 8).drop(1).filter { !it.isPassword && !it.isEditable } + .map(::label).filter(String::isNotEmpty).joinToString(" ").let(::boundedIdentity) + } + + fun shutdown() { + stopped = true + handler.removeCallbacksAndMessages(null) + scrollNode = null + lastWindowId = null + history.clear() + } +} diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java b/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java index c6044db72..8581fe169 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java @@ -192,6 +192,7 @@ enum TalkBackPreference { // A TV has no touchscreen for direct touch, and no gesture for the status readout. DIRECT_TOUCH(R.string.pref_direct_touch_key, HIDDEN_ON_WATCH | HIDDEN_ON_TV), STATUS_READOUT(R.string.pref_status_readout_key, HIDDEN_ON_TV), + RESTORE_FOCUS_SCROLL(R.string.pref_scroll_to_restore_previous_focus_key, HIDDEN_ON_TV), SPEECH_VOLUME(R.string.pref_speech_volume_key, HIDE_HAS_VOLUME_KEY), BRAILLE_KEYBOARD( R.string.pref_brailleime_key, diff --git a/talkback/src/main/res/values/donottranslate.xml b/talkback/src/main/res/values/donottranslate.xml index 9ee93b430..b20d6dd21 100644 --- a/talkback/src/main/res/values/donottranslate.xml +++ b/talkback/src/main/res/values/donottranslate.xml @@ -2002,4 +2002,5 @@ false + pref_scroll_to_restore_previous_focus diff --git a/talkback/src/main/res/values/strings.xml b/talkback/src/main/res/values/strings.xml index 6770c666a..56d26b413 100644 --- a/talkback/src/main/res/values/strings.xml +++ b/talkback/src/main/res/values/strings.xml @@ -4324,4 +4324,8 @@ Do not use any markdown in your response." To hear notifications when your screen is off, turn on \"Speak notifications when screen is off\" in Backtalk verbosity settings + + Scroll to restore previous focus + + When returning to a screen, allow Backtalk to scroll to the last focused item. Off by default. diff --git a/talkback/src/main/res/xml/advanced_preferences.xml b/talkback/src/main/res/xml/advanced_preferences.xml index 810637cc8..efae689b5 100644 --- a/talkback/src/main/res/xml/advanced_preferences.xml +++ b/talkback/src/main/res/xml/advanced_preferences.xml @@ -10,6 +10,11 @@ android:summary="@string/talkback_latency_reduction_summary" android:fragment="com.google.android.accessibility.talkback.preference.base.TalkBackLatencyReductionPreferenceFragment" > + () + private val createdNodes = ArrayList() + private var feedbackSucceeds = true + private var currentContext: ScreenFocusRestorer.Context? = null + + @Before fun setUp() { + service = Robolectric.buildService(TalkBackService::class.java).get() + com.google.android.accessibility.utils.FormFactorUtils.initialize(service) + val prefs = service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE) + focusHistory = AccessibilityFocusActionHistory(service) + restorer = ScreenFocusRestorer(service, prefs, Pipeline.FeedbackReturner { + feedback.add(it) + if (feedbackSucceeds) it.failovers().single().focus()?.target()?.let { target -> + // Feedback targets are framework snapshots. Update the backing tree as the real focus + // actor does, so repeated events can observe that restoration already succeeded. + createdNodes.forEach { node -> node.isAccessibilityFocused = node == target } + } + feedbackSucceeds + }, focusHistory.reader, bookmarks, { true }, { currentContext }) + } + + @After fun tearDown() { restorer.shutdown() } + + private fun node(text: String = "", className: String = "android.widget.TextView", + viewId: String = "", parent: AccessibilityNodeInfoCompat? = null): AccessibilityNodeInfoCompat { + val raw = AccessibilityNodeInfo.obtain() + raw.packageName = "org.quantumbadger.redreader" + raw.text = text + raw.className = className + raw.viewIdResourceName = viewId + raw.isVisibleToUser = true + raw.isImportantForAccessibility = true + raw.setBoundsInScreen(Rect(0, 0, 100, 100)) + val result = AccessibilityNodeInfoCompat.wrap(raw) + createdNodes.add(result) + if (parent != null) { + shadowOf(parent.unwrap()).addChild(raw) + } + return result + } + + @Test fun recreatedWindowsProduceSameContentIdentity() { + val first = node(className = "android.widget.FrameLayout") + val second = node(className = "android.widget.FrameLayout") + node(className = "androidx.recyclerview.widget.RecyclerView", viewId = "posts", parent = first) + .isScrollable = true + node(className = "androidx.recyclerview.widget.RecyclerView", viewId = "posts", parent = second) + .isScrollable = true + assertEquals(restorer.captureContext(first, "RedReader").screen, + restorer.captureContext(second, "RedReader").screen) + } + + @Test fun screenIdentityDoesNotDependOnVisibleRecycledRows() { + val root = node(className = "android.widget.FrameLayout") + val list = node(className = "androidx.recyclerview.widget.RecyclerView", viewId = "posts", + parent = root) + list.isScrollable = true + val row = node("First visible post", parent = list) + val initial = restorer.captureContext(root, "RedReader") + row.text = "Different visible post" + val changed = restorer.captureContext(root, "RedReader") + assertEquals(initial.screen, changed.screen) + assertEquals(initial.anchors, changed.anchors) + } + + @Test fun changedListContainerDistinguishesDetailFromFrontPage() { + val root = node(className = "android.widget.FrameLayout") + val list = node(className = "androidx.recyclerview.widget.RecyclerView", viewId = "posts", + parent = root) + list.isScrollable = true + val initial = restorer.captureContext(root, "RedReader") + list.viewIdResourceName = "comments" + assertNotEquals(initial.screen, restorer.captureContext(root, "RedReader").screen) + } + + @Test fun rootListIdentityDoesNotIncludeVisibleHeadings() { + val root = node(className = "android.widget.ListView", viewId = "posts") + root.isScrollable = true + val row = node("First post", parent = root) + row.isHeading = true + val initial = restorer.captureContext(root, "RedReader") + row.text = "Different post" + val changed = restorer.captureContext(root, "RedReader") + assertEquals(initial.screen, changed.screen) + assertEquals(initial.anchors, changed.anchors) + } + + @Test fun labelsThatDifferAfterMemoryBoundDoNotMatch() { + val root = node(className = "android.widget.FrameLayout") + val row = node("a".repeat(600) + "Original", parent = root) + val saved = restorer.describe(row) + row.text = "a".repeat(600) + "Replacement" + assertEquals(0, saved.matchScore(restorer.describe(row))) + assertTrue(saved.text.length <= 512) + } + + @Test fun detachedNodeCannotBecomeDestinationBookmark() { + returningList() + restorer.rememberBeforeActivation(node("Stale departing item")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + // Recording a detached focus event must not overwrite the valid bookmark. + val saved = bookmarks.pending(android.os.SystemClock.uptimeMillis()) + assertTrue(saved == null || saved.item.text == "Remember this post") + currentContext = currentContext!!.copy(screen = currentContext!!.screen.copy(pane = "Details")) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + currentContext = currentContext!!.copy(screen = currentContext!!.screen.copy(pane = "")) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals("Remember this post", feedback.last().failovers().single().focus()!!.target()!!.text) + } + + @Test fun detachedFocusDoesNotCancelPendingReturn() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.onFocusSet(node("Detached item"), + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + settle() + assertEquals("Remember this post", feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun itemSnapshotDoesNotChangeWhenViewIsRecycled() { + val root = node(className = "android.widget.FrameLayout") + val row = node("Original post", parent = root) + val saved = restorer.describe(row) + row.text = "Replacement post" + assertEquals("Original post", saved.text) + assertEquals(0, saved.matchScore(restorer.describe(row))) + } + + @Test fun itemIncludesWrapperCollectionPositionAndNeighborText() { + val root = node(className = "android.widget.ListView", viewId = "posts") + root.isScrollable = true + node("Previous post", parent = root) + val wrapper = node(className = "android.widget.LinearLayout", parent = root) + wrapper.setCollectionItemInfo(AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain( + 30, 1, 0, 1, false)) + val title = node("Post title", parent = wrapper) + node("Next post", parent = root) + val saved = restorer.describe(title) + assertEquals(30, saved.row) + assertEquals(0, saved.column) + assertEquals("Previous post", saved.previous) + assertEquals("Next post", saved.next) + assertEquals("android.widget.ListView|posts", saved.container) + } + + @Test fun nestedListsKeepNearestContainerNeighborsAndPosition() { + val outer = node(className = "android.widget.ListView", viewId = "outer") + outer.isScrollable = true + node("Outer previous", parent = outer) + val wrapper = node(className = "android.widget.LinearLayout", parent = outer) + wrapper.setCollectionItemInfo(AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain( + 50, 1, 0, 1, false)) + val inner = node(className = "android.widget.ListView", viewId = "inner", parent = wrapper) + inner.isScrollable = true + node("Inner previous", parent = inner) + val row = node("Target", parent = inner) + node("Inner next", parent = inner) + node("Outer next", parent = outer) + val saved = restorer.describe(row) + assertEquals("android.widget.ListView|inner", saved.container) + assertEquals("Inner previous", saved.previous) + assertEquals("Inner next", saved.next) + assertEquals(-1, saved.row) + } + + @Test fun wrapperTextDoesNotIncludeDescendantsOfEditableControls() { + val wrapper = node(className = "android.widget.LinearLayout") + val editor = node(className = "android.widget.EditText", parent = wrapper) + editor.isEditable = true + node("Private editing content", parent = editor) + node("Public label", parent = wrapper) + assertEquals("Public label", restorer.describe(wrapper).text) + } + + @Test fun webDocumentTitlesKeepSeparatePositionsInSameWindow() { + val root = node(className = "android.widget.FrameLayout") + val web = node("Page A", "android.webkit.WebView", parent = root) + val initial = restorer.captureContext(root, "Browser").screen + web.text = "Page B" + assertNotEquals(initial, restorer.captureContext(root, "Browser").screen) + } + + @Test fun nativeHtmlDocumentDoesNotNeedWebViewClassToKeepPagesSeparate() { + val root = node(className = "android.widget.FrameLayout") + val html = node("Page A", "android.view.View", parent = root) + html.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + node("A heading within the page", parent = html) + val initial = restorer.captureContext(root, "Browser").screen + html.text = "Page B" + assertNotEquals(initial, restorer.captureContext(root, "Browser").screen) + } + + @Test fun nativeBrowserAddressAlsoDistinguishesDocumentsWithNoTitle() { + val root = node(className = "android.widget.FrameLayout") + val address = node("example.org/a", viewId = "org.mozilla.firefox:id/mozac_browser_toolbar_url_view", + parent = root) + val initial = restorer.captureContext(root, "Browser").screen + address.text = "example.org/b" + assertNotEquals(initial, restorer.captureContext(root, "Browser").screen) + } + + @Test fun scrollSettingIsOffAndKeyMatchesResource() { + val prefs = service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE) + assertFalse(prefs.getBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, false)) + assertEquals(ScreenFocusRestorer.SCROLL_PREFERENCE, + service.getString(R.string.pref_scroll_to_restore_previous_focus_key)) + } + + private fun returningList(scrollAction: AccessibilityNodeInfoCompat.AccessibilityActionCompat? = null, + position: Int = -1, prepareBack: Boolean = true): + AccessibilityNodeInfoCompat { + val root = node(className = "android.widget.FrameLayout") + node("Front page", viewId = "toolbar_title", parent = root) + val list = node(className = "android.widget.ListView", viewId = "posts", parent = root) + list.isScrollable = true + scrollAction?.let(list::addAction) + val row = node("Remember this post", parent = list) + if (position >= 0) row.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain(position, 1, 0, 1, false)) + currentContext = restorer.captureContext(root, "RedReader") + bookmarks.remember(currentContext!!.screen, currentContext!!.anchors, restorer.describe(row)) + if (prepareBack) { + val destination = currentContext!! + currentContext = destination.copy(screen = destination.screen.copy(pane = "Departing details")) + restorer.requestBack() + currentContext = destination + } + return row + } + + private fun event(type: Int) { + restorer.onAccessibilityEvent(AccessibilityEvent.obtain(type)) + } + + private fun browserContext(): ScreenFocusRestorer.Context { + val root = node(className = "android.widget.FrameLayout") + root.packageName = "com.android.chrome" + val document = node("External post", "android.webkit.WebView", parent = root) + document.packageName = "com.android.chrome" + return restorer.captureContext(root, "Chrome") + } + + @Test fun firstFocusEventCanFinishSaveAfterNodeAttaches() { + val root = node(className = "android.widget.FrameLayout") + node("Front page", viewId = "toolbar_title", parent = root) + val list = node(className = "android.widget.ListView", viewId = "posts", parent = root) + list.isScrollable = true + val post = node("First opened post") + currentContext = restorer.captureContext(root, "RedReader") + val info = FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + val time = android.os.SystemClock.uptimeMillis() + focusHistory.onAccessibilityFocusAction(post, info, time, null) + restorer.onFocusSet(post, info, time) + assertEquals(0, bookmarks.size) + shadowOf(list.unwrap()).addChild(post.unwrap()) + val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(focused).setSourceNode(post.unwrap()) + restorer.onAccessibilityEvent(focused) + assertEquals(1, bookmarks.size) + val front = currentContext!! + currentContext = front.copy(screen = front.screen.copy(pane = "Comments")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + currentContext = front + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(post.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun quickFirstPostRoundTripDoesNotCarryDepartureInputVeto() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.rememberBeforeActivation(row) + val front = currentContext!! + currentContext = front.copy(screen = front.screen.copy(pane = "Comments")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + currentContext = front + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun externalBrowserRoundTripRestoresOriginatingPostWithoutSettlingInBrowser() { + val row = returningList() + restorer.rememberBeforeActivation(row) + val origin = currentContext!! + currentContext = browserContext() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + // The browser can close before the 120 ms restoration callback runs. + restorer.requestBack() + currentContext = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun browserInputDoesNotCancelRestorationAfterReturningToOrigin() { + val row = returningList() + restorer.rememberBeforeActivation(row) + val origin = currentContext!! + currentContext = browserContext() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.onUserInteraction() + restorer.requestBack() + currentContext = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun activationPreservesReturnEvenWhenDestinationRootWasNeverAvailable() { + val row = returningList() + restorer.rememberBeforeActivation(row) + // The app redirected and returned while rootInActiveWindow was stale. Only the returning + // screen's event is observable; the saved activation must still allow a return attempt. + restorer.requestBack() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun inputOnOriginAfterBrowserReturnStillPreventsOverride() { + val row = returningList() + restorer.rememberBeforeActivation(row) + val origin = currentContext!! + currentContext = browserContext() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + currentContext = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.onUserInteraction() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun lateAutomaticBackButtonFocusCannotDisplaceRestoredFirstPost() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + val back = node("Back", "android.widget.Button", parent = currentContext!!.root) + row.isAccessibilityFocused = false + back.isAccessibilityFocused = true + restorer.onFocusSet(back, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build()) + settle() + assertEquals(2, feedback.size) + assertEquals(row.text, feedback.last().failovers().single().focus()!!.target()!!.text) + } + + @Test fun userInputAfterRestorePreventsCorrectionOfLaterAutomaticFocus() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + restorer.onUserInteraction() + row.isAccessibilityFocused = false + restorer.onFocusSet(node("Back", "android.widget.Button", parent = currentContext!!.root), + FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build()) + settle() + assertEquals(1, feedback.size) + } + + @Test fun appAssignedBackFocusWithoutFocusHistoryIsAlsoCorrected() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + row.isAccessibilityFocused = false + val back = node("Back", "android.widget.Button", parent = currentContext!!.root) + back.isAccessibilityFocused = true + val assigned = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(assigned).setSourceNode(back.unwrap()) + restorer.onAccessibilityEvent(assigned) + settle() + assertEquals(2, feedback.size) + assertEquals(row.text, feedback.last().failovers().single().focus()!!.target()!!.text) + } + + private fun settle(ms: Long = 121L) { + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(ms)) + } + + @Test fun returningListReplacesAutomaticTopFocusWithRememberedPost() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + assertEquals("Remember this post", feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun delayedContentRetriesWithoutWaitingForDeadline() { + val row = returningList() + row.text = "Loading" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + row.text = "Remember this post" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(1, feedback.size) + } + + @Test fun touchingDuringTransitionPreventsFocusOverride() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START) + settle() + // A later content event cannot restart the cancelled transition. + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun touchingDuringContentOnlyTransitionPreventsFocusOverride() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + event(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START) + settle() + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun repeatedWindowEventCannotUndoTouchCancellation() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun repeatedWindowEventsDoNotExtendReconstructionDeadline() { + val row = returningList() + row.text = "Loading" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + repeat(4) { + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle(800) + } + row.text = "Remember this post" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun repeatedWindowEventsDoNotRepeatSuccessfulRestoration() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + } + + @Test fun keyInputCancelsPendingRestoration() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.onKeyEvent(KeyEvent(KeyEvent.ACTION_DOWN, KeyEvent.KEYCODE_DPAD_DOWN)) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun editableInputFocusKeepsNormalInputFocusPolicy() { + returningList() + val editor = node("Editing", "android.widget.EditText", parent = currentContext!!.root) + editor.isEditable = true + editor.isFocused = true + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun automaticInitialFocusDoesNotReplaceSavedPost() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val info = FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build() + restorer.onFocusSet(node("Toolbar"), info) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun delayedNativeFocusEventDoesNotOverwriteDestinationBookmark() { + val row = returningList() + val info = FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + val time = android.os.SystemClock.uptimeMillis() + focusHistory.onAccessibilityFocusAction(row, info, time, null) + restorer.onFocusSet(row, info, time) + val destination = node("Destination item", parent = currentContext!!.root) + restorer.requestBack() + currentContext = currentContext!!.copy(screen = currentContext!!.screen.copy(pane = "Details")) + bookmarks.remember(currentContext!!.screen, currentContext!!.anchors, + restorer.describe(destination)) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val delayed = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(delayed).setSourceNode(row.unwrap()) + assertNotNull(focusHistory.reader.matchFocusActionRecordFromEvent(delayed)) + restorer.onAccessibilityEvent(delayed) + settle() + assertEquals(destination.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun defaultDoesNotScrollForOffscreenItem() { + val row = returningList() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun visibleFlagOutsideWindowDoesNotCauseImplicitScrollingByDefault() { + val row = returningList() + row.setBoundsInScreen(Rect(0, 200, 100, 300)) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun activationSavesInitialFocusWithoutAnEarlierSwipe() { + val root = node(className = "android.widget.FrameLayout") + val link = node("Open a page", parent = root) + currentContext = restorer.captureContext(root, "Browser") + restorer.rememberBeforeActivation(link) + val front = currentContext!! + currentContext = front.copy(screen = front.screen.copy(document = "Another page")) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + restorer.requestBack() + currentContext = front + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(link.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun enabledSettingShowsOffscreenItemBeforeFocusing() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN.id, + feedback.single().failovers().single().nodeAction()!!.actionId()) + row.isVisibleToUser = true + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(300) + assertEquals(2, feedback.size) + assertNotNull(feedback.last().failovers().single().focus()) + } + + @Test fun showOnScreenAcceptsScrollEventFromContainingList() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + val scrolled = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scrolled).setSourceNode(row.parent.unwrap()) + row.isVisibleToUser = true + restorer.onAccessibilityEvent(scrolled) + settle() + assertEquals(2, feedback.size) + assertNotNull(feedback.last().failovers().single().focus()) + } + + @Test fun unsupportedScrollActionLeavesExistingFocus() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun unsupportedShowOnScreenFallsBackToSupportedContainerScroll() { + val row = returningList(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD, + feedback.single().failovers().single().nodeAction()!!.actionId()) + row.isVisibleToUser = true + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(300) + assertEquals(2, feedback.size) + assertNotNull(feedback.last().failovers().single().focus()) + } + + @Test fun failedPositionScrollStopsWithoutIssuingAnotherAction() { + val row = returningList( + AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION, 30) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.text = "Different visible post" + row.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain(0, 1, 0, 1, false)) + feedbackSucceeds = false + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION.id, + feedback.single().failovers().single().nodeAction()!!.actionId()) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(400) + assertEquals(1, feedback.size) + } + + @Test fun containerSearchIssuesAtMostTenActionsEvenWhenContentKeepsChanging() { + val row = returningList(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD, 100) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.text = "Loading first row" + row.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain(0, 1, 0, 1, false)) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + repeat(12) { + row.text = "Loading row $it" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(241) + } + assertEquals(10, feedback.size) + assertTrue(feedback.all { it.failovers().single().nodeAction() != null }) + } + + @Test fun changedLabelAtVisibleSavedPositionDoesNotScroll() { + val row = returningList(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION, 30) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.text = "Read state changed while returning" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + // Content reconstruction can finish without changing the viewport. + row.text = "Remember this post" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertNotNull(feedback.single().failovers().single().focus()) + } + + @Test fun missingUnindexedItemDoesNotStartBlindContainerScrolling() { + val row = returningList(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.text = "List still rebuilding" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + row.text = "Remember this post" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertNotNull(feedback.single().failovers().single().focus()) + } + + @Test fun visibleFocusFailureDoesNotFallThroughToScrolling() { + val row = returningList(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD) + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + feedbackSucceeds = false + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertNotNull(feedback.single().failovers().single().focus()) + assertNull(feedback.single().failovers().single().nodeAction()) + } + + @Test fun visibleNonFocusableMatchDoesNotRequestShowOnScreen() { + val root = node(className = "android.widget.FrameLayout") + val list = node(className = "android.widget.ListView", viewId = "posts", parent = root) + list.isScrollable = true + list.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD) + val wrapper = node(className = "android.widget.LinearLayout", parent = list) + val row = node("Remember this post", parent = wrapper) + currentContext = restorer.captureContext(root, "App") + bookmarks.remember(currentContext!!.screen, currentContext!!.anchors, restorer.describe(row)) + val destination = currentContext!! + currentContext = destination.copy(screen = destination.screen.copy(pane = "Details")) + restorer.requestBack() + currentContext = destination + // Reconstruction groups the visible label into its clickable row wrapper. + wrapper.isClickable = true + assertFalse(com.google.android.accessibility.utils.AccessibilityNodeInfoUtils.shouldFocusNode(row)) + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.none { it.failovers().single().nodeAction() != null }) + } + + @Test fun redReaderReadStateAndChangingSubtitlePreservePostIdentity() { + val root = node(className = "androidx.recyclerview.widget.RecyclerView") + root.isScrollable = true + val row = node(className = "android.widget.LinearLayout", + viewId = "org.quantumbadger.redreader:id/reddit_post_layout_outer", parent = root) + row.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain(0, 1, 0, 1, false)) + val title = node("Read this post", viewId = "org.quantumbadger.redreader:id/reddit_post_title", parent = row) + val subtitle = node("1 minute, 3 comments", viewId = "org.quantumbadger.redreader:id/reddit_post_subtitle", parent = row) + currentContext = restorer.captureContext(root, "RedReader") + restorer.rememberBeforeActivation(row) + val saved = restorer.describe(row) + currentContext = currentContext!!.copy(screen = currentContext!!.screen.copy(pane = "Post")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + currentContext = restorer.captureContext(root, "RedReader") + title.contentDescription = "Read, Read this post" + subtitle.text = "2 minutes, 4 comments" + assertEquals("Read this post", restorer.describe(row).text) + assertTrue(saved.matchScore(restorer.describe(row)) > 0) + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + root.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SCROLL_FORWARD) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row, feedback.single().failovers().single().focus()!!.target()) + } + + @Test fun redReaderRecycledPostWithDifferentTitleCannotMatchSamePosition() { + val row = node(className = "android.widget.LinearLayout", + viewId = "org.quantumbadger.redreader:id/reddit_post_layout_outer") + row.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain(0, 1, 0, 1, false)) + val title = node("Original post", viewId = "org.quantumbadger.redreader:id/reddit_post_title", parent = row) + val original = restorer.describe(row) + title.text = "Different post" + assertEquals(0, original.matchScore(restorer.describe(row))) + } + + @Test fun primaryTitleAndChangingMetadataWorkInAnyApp() { + val root = node(className = "android.widget.FrameLayout") + root.packageName = "other.app" + val row = node(className = "android.widget.LinearLayout", + viewId = "other.app:id/article_card", parent = root) + row.packageName = "other.app" + row.isClickable = true + val title = node("Post name", viewId = "other.app:id/article_title", parent = row) + title.packageName = "other.app" + val subtitle = node("Unread", parent = row) + subtitle.packageName = "other.app" + val original = restorer.describe(row) + subtitle.text = "Read" + title.contentDescription = "Read, Post name" + assertEquals("Post name", restorer.describe(row).text) + assertEquals(original.text, restorer.describe(row).text) + assertEquals(original.description, restorer.describe(row).description) + assertTrue(original.matchScore(restorer.describe(row)) > 0) + // A different main title is still a different item even on the same recycled card. + title.text = "Another post" + assertEquals(0, original.matchScore(restorer.describe(row))) + } + + @Test fun nativeRowHeadingWithoutResourceIdPreservesIdentity() { + val root = node(className = "android.widget.ListView") + root.isScrollable = true + val row = node(className = "android.widget.LinearLayout", parent = root) + val heading = node("Item heading", parent = row) + heading.isHeading = true + val status = node("Unread", parent = row) + val original = restorer.describe(row) + heading.contentDescription = "Read, Item heading" + status.text = "Read" + assertEquals("Item heading", original.text) + assertTrue(original.matchScore(restorer.describe(row)) > 0) + } + + @Test fun multipleTitleFieldsCannotHideChangedItemContent() { + val root = node(className = "android.widget.ListView") + root.isScrollable = true + val row = node(className = "android.widget.LinearLayout", parent = root) + node("Shared title", viewId = "other.app:id/title", parent = row) + val other = node("Original title", viewId = "other.app:id/secondary_title", parent = row) + val original = restorer.describe(row) + other.text = "Replacement title" + assertEquals(0, original.matchScore(restorer.describe(row))) + } + + @Test fun screenToolbarTitleCannotHideChangedContent() { + val root = node(className = "android.widget.FrameLayout") + node("Shared title", viewId = "other.app:id/toolbar_title", parent = root) + val content = node("Original item", parent = root) + val original = restorer.describe(root) + content.text = "Replacement item" + assertEquals(0, original.matchScore(restorer.describe(root))) + } + + @Test fun nestedTitleFocusUsesPrimaryTextInsteadOfChangingDescription() { + val root = node(className = "android.widget.ListView") + root.isScrollable = true + val row = node(className = "android.widget.LinearLayout", parent = root) + val wrapper = node(className = "android.widget.LinearLayout", parent = row) + val title = node("Actual title", viewId = "other.app:id/title", parent = wrapper) + val original = restorer.describe(title) + title.contentDescription = "Read, Actual title" + assertTrue(original.matchScore(restorer.describe(title)) > 0) + } + + @Test fun duplicatePrimaryTitlesWithDifferentMetadataRemainAmbiguous() { + val root = node(className = "android.widget.ListView") + root.isScrollable = true + val first = node(className = "android.widget.LinearLayout", parent = root) + node("Same title", viewId = "other.app:id/title", parent = first) + node("Unread, 1 comment", parent = first) + val second = node(className = "android.widget.LinearLayout", parent = root) + node("Same title", viewId = "other.app:id/title", parent = second) + node("Read, 2 comments", parent = second) + val screen = restorer.captureContext(root, "Other app").screen + val saved = ScreenBookmark(screen, emptySet(), restorer.describe(first)) + assertNull(bookmarks.find(saved, listOf(restorer.describe(first), restorer.describe(second)))) + } + + @Test fun browserTitleFieldRetainsExistingFullDescriptionMatching() { + val root = node(className = "android.webkit.WebView") + val title = node("Web heading", viewId = "page_title", parent = root) + title.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + val original = restorer.describe(title) + title.contentDescription = "Different web content" + assertEquals(0, original.matchScore(restorer.describe(title))) + } + + @Test fun shutdownClearsBookmarksAndQueuedCallbacks() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.shutdown() + settle() + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(0, bookmarks.size) + assertTrue(feedback.isEmpty()) + } + + @Test fun sameWindowContentChangeCanRecognizeReturningScreen() { + val row = returningList() + val front = currentContext!! + currentContext = front.copy(screen = front.screen.copy(pane = "Comments")) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + restorer.requestBack() + currentContext = front + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun contentAfterDeadlineDoesNotReclaimFocus() { + val row = returningList() + row.text = "Loading" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle(3200) + row.text = "Remember this post" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun inputDuringScrollCannotReviveDelayedCallback() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + restorer.onUserInteraction() + row.isVisibleToUser = true + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(400) + assertEquals(1, feedback.size) + } + + @Test fun lateAutomaticScrollFocusDoesNotOverwriteBookmark() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + settle(1100) + val automaticTop = node("Automatic top", parent = currentContext!!.root) + restorer.onFocusSet(automaticTop, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + val now = android.os.SystemClock.uptimeMillis() + bookmarks.enter(currentContext!!.screen, now, true) + assertEquals("Remember this post", bookmarks.pending(now)!!.item.text) + // A real interaction clears the exemption and may update the remembered position. + restorer.onUserInteraction() + restorer.onFocusSet(automaticTop, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + bookmarks.enter(currentContext!!.screen, now, true) + assertEquals("Automatic top", bookmarks.pending(now)!!.item.text) + } + + @Test fun failedSearchReleasesScrollFocusExemptionAtDeadline() { + val row = returningList() + service.getSharedPreferences("screen-focus-test", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + row.isVisibleToUser = false + row.addAction(AccessibilityNodeInfoCompat.AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle(3300) + val newFocus = node("Later focus", parent = currentContext!!.root) + restorer.onFocusSet(newFocus, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + val now = android.os.SystemClock.uptimeMillis() + bookmarks.enter(currentContext!!.screen, now, true) + assertEquals("Later focus", bookmarks.pending(now)!!.item.text) + } + + @Test fun windowAndContentEventsWithoutBackNeverRestore() { + returningList(prepareBack = false) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(1100) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun forwardActivationDoesNotAuthorizeRestoration() { + val row = returningList(prepareBack = false) + restorer.rememberBeforeActivation(row) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + val origin = currentContext!! + currentContext = browserContext() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + currentContext = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun statusBarFocusCancelsPendingBackEvenWithStaleApplicationRoot() { + returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val clock = node("Clock") + clock.packageName = "com.android.systemui" + val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(focused).setSourceNode(clock.unwrap()) + restorer.onAccessibilityEvent(focused) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(1200) + assertTrue(feedback.isEmpty()) + } + + @Test fun statusBarFocusStopsCorrectionAfterSuccessfulBackRestore() { + val row = returningList() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + row.isAccessibilityFocused = false + val clock = node("Clock") + clock.packageName = "com.android.systemui" + restorer.onFocusSet(clock, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build()) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle(1200) + assertEquals(1, feedback.size) + } + + @Test fun browserBackRestoresHeadingAfterTransientDocumentReconstruction() { + val root = node(className = "android.widget.FrameLayout") + root.packageName = "com.android.chrome" + val address = node("https://example.org/a", viewId = "url_bar", parent = root) + address.packageName = "com.android.chrome" + val document = node("Page A", "android.webkit.WebView", parent = root) + document.packageName = "com.android.chrome" + val heading = node("Original heading", "android.view.View", parent = document) + heading.packageName = "com.android.chrome" + heading.isHeading = true + heading.isFocusable = true + currentContext = restorer.captureContext(root, "Chrome") + val info = FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + restorer.onFocusSet(heading, info) + // Browsers expose the native WebView as current focus when activating an HTML link. + restorer.rememberBeforeActivation(document) + document.text = "Page B" + address.text = "https://example.org/b" + heading.text = "Other heading" + currentContext = restorer.captureContext(root, "Chrome") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + document.text = "" + address.text = "https://example.org/a" + heading.text = "Loading" + currentContext = restorer.captureContext(root, "Chrome") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + document.text = "Page A" + heading.text = "Original heading" + currentContext = restorer.captureContext(root, "Chrome") + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals("Original heading", feedback.single().failovers().single().focus()!!.target()!!.text) + } + + private fun returningBrowserHeading(): AccessibilityNodeInfoCompat { + val root = node(className = "android.widget.FrameLayout") + val document = node("Page A", "android.webkit.WebView", parent = root) + val heading = node("Remembered browser heading", "android.view.View", parent = document) + heading.isHeading = true + heading.isFocusable = true + currentContext = restorer.captureContext(root, "Browser") + restorer.onFocusSet(heading, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + document.text = "Page B" + currentContext = restorer.captureContext(root, "Browser") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + document.text = "Page A" + currentContext = restorer.captureContext(root, "Browser") + return heading + } + + @Test fun browserAutomaticViewportRestoreDoesNotCancelBackBeforeScreenEvent() { + val heading = returningBrowserHeading() + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(heading.parent.unwrap()) + restorer.onAccessibilityEvent(scroll) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(heading.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun browserAutomaticViewportRestoreDoesNotCancelPendingBookmark() { + val heading = returningBrowserHeading() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(heading.parent.unwrap()) + restorer.onAccessibilityEvent(scroll) + settle() + assertEquals(heading.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun browserViewportScrollFromNativeAncestorAllowsDelayedDomReconstruction() { + val heading = returningBrowserHeading() + val savedText = heading.text + heading.text = "Loading" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + // Edge can report the history-entry scroll on the native browser container above WebView. + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(currentContext!!.root.unwrap()) + restorer.onAccessibilityEvent(scroll) + heading.text = savedText + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(savedText, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun sourceLessBrowserViewportEventAllowsDelayedDomReconstruction() { + val heading = returningBrowserHeading() + val savedText = heading.text + heading.text = "Loading" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + scroll.packageName = currentContext!!.root.packageName + restorer.onAccessibilityEvent(scroll) + heading.text = savedText + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(savedText, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun sourceLessScrollFromSystemUiCancelsBrowserReturn() { + returningBrowserHeading() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + scroll.packageName = "com.android.systemui" + restorer.onAccessibilityEvent(scroll) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun userInputBeforeNativeBrowserScrollCancelsReturn() { + returningBrowserHeading() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.onUserInteraction() + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(currentContext!!.root.unwrap()) + restorer.onAccessibilityEvent(scroll) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun browserAddressUpdateDoesNotCancelBack() { + val heading = returningBrowserHeading() + val address = node("https://example.org/a", "android.widget.EditText", "url_bar", + currentContext!!.root) + address.isEditable = true + val changed = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED) + shadowOf(changed).setSourceNode(address.unwrap()) + restorer.onAccessibilityEvent(changed) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(heading.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun automaticBrowserScrollFocusDoesNotEraseHeadingBookmark() { + val heading = returningBrowserHeading() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val automatic = node("Automatic first item", "android.view.View", parent = heading.parent) + val scrolled = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scrolled).setSourceNode(heading.parent.unwrap()) + restorer.onAccessibilityEvent(scrolled) + // The normal scroll interpreter can refocus after the browser restores its own viewport. + restorer.onFocusSet(automatic, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + settle() + assertEquals(heading.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun typingInBrowserAddressStillCancelsBack() { + returningBrowserHeading() + val address = node("https://example.org/a", "android.widget.EditText", "url_bar", + currentContext!!.root) + address.isEditable = true + address.isFocused = true + val changed = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED) + shadowOf(changed).setSourceNode(address.unwrap()) + restorer.onAccessibilityEvent(changed) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun userBrowserScrollStillCancelsBack() { + val heading = returningBrowserHeading() + restorer.onUserInteraction() + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(heading.parent.unwrap()) + restorer.onAccessibilityEvent(scroll) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun browserAutomaticScrollWithoutBackCannotRestore() { + val heading = returningBrowserHeading() + restorer.onUserInteraction() + val scroll = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED) + shadowOf(scroll).setSourceNode(heading.parent.unwrap()) + restorer.onAccessibilityEvent(scroll) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test fun asynchronousHtmlHeadingWithNativeOnlyDocumentAncestorRemainsABookmark() { + val root = node(className = "android.widget.FrameLayout") + root.packageName = "org.mozilla.firefox" + val container = node("Firefox page", "android.webkit.WebView", parent = root) + container.packageName = "org.mozilla.firefox" + val heading = node("A web heading", "android.view.View", parent = container) + heading.packageName = "org.mozilla.firefox" + heading.isHeading = true + heading.isFocusable = true + heading.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + currentContext = restorer.captureContext(root, "Firefox") + val page = currentContext!! + val info = FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + focusHistory.onPendingAccessibilityFocusActionOnWebElement(info, + android.os.SystemClock.uptimeMillis(), null) + settle(1) + val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(focused).setSourceNode(heading.unwrap()) + restorer.onAccessibilityEvent(focused) + assertEquals(1, bookmarks.size) + currentContext = page.copy(screen = page.screen.copy(document = "Another document")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + currentContext = page + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(heading.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun htmlLinkWithNestedContentIsNotMistakenForDocumentContainer() { + val root = node(className = "android.widget.FrameLayout") + val link = node("A web link", "android.view.View", parent = root) + link.isClickable = true + link.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + node("Nested link content", "android.view.View", parent = link) + currentContext = restorer.captureContext(root, "Firefox") + restorer.onFocusSet(link, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + val now = android.os.SystemClock.uptimeMillis() + bookmarks.enter(currentContext!!.screen, now, true) + assertEquals("A web link", bookmarks.pending(now)!!.item.text) + } + + @Test fun activatingNativeDocumentDoesNotDiscardPendingHtmlFocusEvent() { + val root = node(className = "android.widget.FrameLayout") + val container = node("Page title", "android.webkit.WebView", parent = root) + val heading = node("Pending HTML heading", "android.view.View", parent = container) + heading.isHeading = true + heading.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + currentContext = restorer.captureContext(root, "Firefox") + focusHistory.onPendingAccessibilityFocusActionOnWebElement( + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build(), + android.os.SystemClock.uptimeMillis(), null) + settle(1) + focusHistory.onAccessibilityFocusAction(container, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build(), + android.os.SystemClock.uptimeMillis(), null) + restorer.rememberBeforeActivation(container) + val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(focused).setSourceNode(heading.unwrap()) + restorer.onAccessibilityEvent(focused) + val now = android.os.SystemClock.uptimeMillis() + bookmarks.enter(currentContext!!.screen, now, true) + assertEquals("Pending HTML heading", bookmarks.pending(now)!!.item.text) + } + + @Test fun appBackButtonAuthorizesRestoreAndItsClickEventDoesNotCancelIt() { + val row = returningList(prepareBack = false) + val origin = currentContext!! + currentContext = origin.copy(screen = origin.screen.copy(pane = "Comments")) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val back = node("Back", "android.widget.Button", parent = currentContext!!.root) + restorer.rememberBeforeActivation(back) + val clicked = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_CLICKED) + shadowOf(clicked).setSourceNode(back.unwrap()) + restorer.onAccessibilityEvent(clicked) + currentContext = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) + } +} From 644aa99f0d55dbd74ec5b8c08c9ba4e838392a9e Mon Sep 17 00:00:00 2001 From: willbilec Date: Sun, 11 Oct 2026 15:42:51 -0600 Subject: [PATCH 2/2] Bound Back focus restoration work and address review --- differences.md | 16 +- lint-baseline.xml | 6 +- .../talkback/TalkBackService.java | 36 +- .../focusmanagement/ScreenFocusRestorer.kt | 259 +++++++++---- .../preference/TalkBackPreferenceFilter.java | 3 +- .../main/res/values-cs/strings_backtalk.xml | 6 +- talkback/src/main/res/values/strings.xml | 2 +- .../ScreenFocusRestorerTest.kt | 37 +- .../ScreenFocusWindowAccessTest.kt | 366 ++++++++++++++++++ 9 files changed, 622 insertions(+), 109 deletions(-) create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusWindowAccessTest.kt diff --git a/differences.md b/differences.md index cd4b59e81..80bb4b46e 100644 --- a/differences.md +++ b/differences.md @@ -1,13 +1,5 @@ # Differences from TalkBack -## Returning to a screen - -Backtalk remembers the last item you read on up to 20 recent app screens and browser pages while the service is running. Restoration starts only when you use Back, including the Back gesture/key or a recognized app Back button. It tries to focus the same item, including when an app recreates its list or automatically focuses the toolbar. This also works when an app opens a post or link in another app: using Back from the browser or other app restores the originating item. Forward navigation, switching apps, and opening or reading the status bar do not trigger restoration. Input on the returned screen or focus in another window cancels pending restoration. This memory clears when Backtalk stops. - -Visible items are restored without scrolling. **Advanced settings** > **Scroll to restore previous focus** is off by default. Turn it on to let Backtalk bring an offscreen item back into view. Backtalk uses the app's supported scroll actions and checks that it found the remembered item; it stops after at most ten scroll actions or three seconds. It waits for rebuilding content instead of searching by scrolling when the saved row position is already visible. Touching, navigating, typing, or scrolling cancels pending restoration. A browser restoring its own page scroll position or updating its address during Back does not cancel the attempt. - -Apps and pages must expose enough accessibility information to distinguish the screen and item. In native app rows and cards, a single exposed title or heading can identify the item even when its read-state description, counts, or subtitle change. This applies across apps; uncertain or duplicate titles still need an unambiguous match. If the item was removed, labels are ambiguous, or the app does not expose the necessary information, Backtalk keeps its usual focus behavior. It remembers elements such as headings, links, and posts, rather than character positions within text. TV keeps its usual input-focus behavior. - Backtalk is built from Google's TalkBack source releases. This document describes how Backtalk differs from TalkBack: features that TalkBack doesn't have, defaults that Backtalk changes, and problems in TalkBack that Backtalk fixes. To learn how to install Backtalk, see the [README](README.md). Each section names the settings involved as they appear in Backtalk. Unless a section says otherwise, each setting is in Backtalk settings. @@ -317,6 +309,14 @@ To let Android recognize gestures again, turn off **Advanced settings** > **Deve ## Navigation +### Returning to a screen + +On phones, Backtalk remembers the last item you read on up to 20 recent app screens and browser pages while the service is running. Restoration starts only when you use Back, including the Back gesture/key or a recognized app Back button. It tries to focus the same item, including when an app recreates its list or automatically focuses the toolbar. This also works when an app opens a post or link in another app: using Back from the browser or other app restores the originating item. Forward navigation, switching apps, and opening or reading the status bar do not trigger restoration. User input on the returned screen cancels pending restoration. This memory clears when Backtalk stops. + +Visible items are restored without scrolling. **Advanced settings** > **Scroll to restore previous focus** is off by default. Turn it on to let Backtalk bring an offscreen item back into view. Backtalk uses the app's supported scroll actions and checks that it found the remembered item; it stops after at most ten scroll actions or three seconds. It waits for rebuilding content instead of searching by scrolling when the saved row position is already visible. Touching, navigating, typing, or scrolling cancels pending restoration. A browser restoring its own page scroll position or updating its address during Back does not cancel the attempt. + +Apps and pages must expose enough accessibility information to distinguish the screen and item. In native app rows and cards, a single exposed title or heading can identify the item even when its read-state description, counts, or subtitle change. This applies across apps; uncertain or duplicate titles still need an unambiguous match. If the item was removed, labels are ambiguous, or the app does not expose the necessary information, Backtalk keeps its usual focus behavior. It remembers elements such as headings, links, and posts, rather than character positions within text. Wear OS watches and TV keep their usual focus behavior and do not run this feature. + ### Reading place kept when an item disappears When an app removed the item that had focus, such as an ad that collapses, TalkBack jumped back to the top of the page. Backtalk carries on from where you were. diff --git a/lint-baseline.xml b/lint-baseline.xml index f2cf64d6a..9d06e6376 100644 --- a/lint-baseline.xml +++ b/lint-baseline.xml @@ -25522,7 +25522,7 @@ { - if (screenFocusRestorer != null) { - if (action == GLOBAL_ACTION_BACK) { - screenFocusRestorer.requestBack(); - } else { - screenFocusRestorer.onUserInteraction(); + if (!FormFactorUtils.isAndroidWear() && !FormFactorUtils.isAndroidTv()) { + systemActionPerformer.setBeforeActionListener( + action -> { + if (screenFocusRestorer != null) { + if (action == GLOBAL_ACTION_BACK) { + screenFocusRestorer.requestBack(); + } else { + screenFocusRestorer.onUserInteraction(); + } } - } - }); + }); + } pipeline = new Pipeline( this, @@ -2496,11 +2498,13 @@ public void onPausedChanged(boolean paused) { // in the order they are added. eventFilter = new EventFilter(this, compositor, touchMonitor, globalVariables, earlyFocusSpeech); - screenFocusRestorer = - new ScreenFocusRestorer( - this, prefs, pipeline.getFeedbackReturner(), pipeline.getActorState().getFocusHistory()); - userInterface.registerListener(screenFocusRestorer); - focuser.setBeforeActivationListener(screenFocusRestorer::rememberBeforeActivation); + if (!FormFactorUtils.isAndroidWear() && !FormFactorUtils.isAndroidTv()) { + screenFocusRestorer = + new ScreenFocusRestorer( + this, prefs, pipeline.getFeedbackReturner(), pipeline.getActorState().getFocusHistory()); + userInterface.registerListener(screenFocusRestorer); + focuser.setBeforeActivationListener(screenFocusRestorer::rememberBeforeActivation); + } focuser.setFocusSetListener( (node, info, eventId, actionTime) -> { eventFilter.onAccessibilityFocusSet( @@ -2510,7 +2514,9 @@ public void onPausedChanged(boolean paused) { actionTime, /* continuousReading= */ fullScreenReadActor != null && fullScreenReadActor.isActive()); - screenFocusRestorer.onFocusSet(node, info, actionTime); + if (screenFocusRestorer != null) { + screenFocusRestorer.onFocusSet(node, info, actionTime); + } }); eventFilter.setTargetPredictor(directionNavigationActor::predictTarget); eventFilter.setFingerDownSupplier( diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt index 68752a828..ad7f91bd3 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorer.kt @@ -32,7 +32,8 @@ import java.security.MessageDigest /** * Complements window history with content bookmarks. Runs after normal initial-focus processing, * so a returning screen can replace automatic toolbar focus without changing dialog/IME policy. - * Session bookmarks retain values only; a pending scroll briefly retains its action target. + * Bookmarks retain values only. The latest user focus is held until activation or Back, when + * its identity is checked and described; ordinary focus and content events never scan screens. */ class ScreenFocusRestorer internal constructor( private val service: TalkBackService, @@ -42,6 +43,7 @@ class ScreenFocusRestorer internal constructor( private val history: ScreenFocusBookmarks, private val available: () -> Boolean, private val readContext: (() -> Context?)?, + private val windows: WindowAccess = ServiceWindowAccess(service), ) : UserInputEventListener { constructor(service: TalkBackService, prefs: SharedPreferences, pipeline: Pipeline.FeedbackReturner, focusHistory: AccessibilityFocusActionHistory.Reader) : @@ -67,8 +69,13 @@ class ScreenFocusRestorer internal constructor( private var triedPosition = false private var waitingForScroll = false private var stopped = false - private var cancelledBeforeEnter = false - private var cancelledScreen: RememberedScreen? = null + private var generation = 0L + private var observedContext: Context? = null + private var ordinaryWindow = false + private var cachedFocus: CachedFocus? = null + private var awaitingWebFocus = false + private var awaitingWebScreen: RememberedScreen? = null + private var activationTime = 0L private var lastWindowId: Int? = null private var lastRecordedFocusTime = Long.MIN_VALUE private var backRequestedAt: Long? = null @@ -77,7 +84,8 @@ class ScreenFocusRestorer internal constructor( private var activationOrigin: RememberedScreen? = null private var retryAt = Long.MAX_VALUE private val retry = Runnable { retryAt = Long.MAX_VALUE; attemptRestore() } - private val releaseScroll = Runnable { scrollNode = null; waitingForScroll = false } + private val deadline = Runnable { onUserInteraction() } + private val releaseWebFocus = Runnable { awaitingWebFocus = false; awaitingWebScreen = null } internal data class Context( val root: AccessibilityNodeInfoCompat, @@ -85,6 +93,36 @@ class ScreenFocusRestorer internal constructor( val anchors: Set, ) + /** Boundary for framework reads; tests feed actual trees through the production capture path. */ + internal interface WindowAccess { + fun activeRoot(): AccessibilityNodeInfoCompat? + fun rootFor(node: AccessibilityNodeInfoCompat): AccessibilityNodeInfoCompat? + } + + private class ServiceWindowAccess(private val service: TalkBackService) : WindowAccess { + override fun activeRoot() = service.rootInActiveWindow?.let(AccessibilityNodeInfoCompat::wrap) + override fun rootFor(node: AccessibilityNodeInfoCompat) = node.window?.root + } + + private data class FocusIdentity(val packageName: String, val windowId: Int, + val uniqueId: String, val viewId: String, val className: String, val role: Int, + val text: String, val description: String) + + private data class CachedFocus(val node: AccessibilityNodeInfoCompat, val identity: FocusIdentity) + + private fun identity(node: AccessibilityNodeInfoCompat) = FocusIdentity( + node.packageName?.toString().orEmpty(), node.windowId, node.uniqueId.orEmpty(), + node.viewIdResourceName.orEmpty(), node.className?.toString().orEmpty(), Role.getRole(node), + boundedIdentity(node.text?.toString().orEmpty()), + boundedIdentity(node.contentDescription?.toString().orEmpty())) + + private fun enabled() = !stopped && !FormFactorUtils.isAndroidWear() && !FormFactorUtils.isAndroidTv() + + private fun authorized(now: Long = SystemClock.uptimeMillis()): Boolean = + history.pending(now) != null || backRequestedAt?.let { + now >= it && now - it < ScreenFocusBookmarks.RESTORE_TIMEOUT_MS + } == true + override fun touchInteractionState(active: Boolean) { if (active) onUserInteraction() } @@ -101,14 +139,17 @@ class ScreenFocusRestorer internal constructor( } fun onUserInteraction() { + generation++ backRequestedAt = null - if (transitionQueued || retryAt != Long.MAX_VALUE) { - cancelledBeforeEnter = true - cancelledScreen = context()?.screen - } + backFrom = null + backWindowId = null + observedContext = null + awaitingWebFocus = false + awaitingWebScreen = null history.cancel() handler.removeCallbacks(retry) - handler.removeCallbacks(releaseScroll) + handler.removeCallbacks(deadline) + handler.removeCallbacks(releaseWebFocus) retryAt = Long.MAX_VALUE transitionQueued = false scrollNode = null @@ -118,11 +159,10 @@ class ScreenFocusRestorer internal constructor( /** Also called synchronously before the app's event for a successful focus action arrives. */ fun onFocusSet(node: AccessibilityNodeInfoCompat, info: FocusActionInfo, actionTime: Long = SystemClock.uptimeMillis()) { - if (stopped || restoring) return - val active = context() - if (active == null || node.packageName != active.root.packageName || - node.windowId != active.root.windowId) { + if (!enabled() || restoring) return + if (node.packageName?.toString() == "com.android.systemui") { onUserInteraction() + cachedFocus = null return } if (info.sourceAction == FocusActionInfo.SCREEN_STATE_CHANGE) { @@ -132,6 +172,10 @@ class ScreenFocusRestorer internal constructor( return } if (!isUserFocus(info)) return + val observed = observedContext + if (authorized() && observed != null && node.packageName == observed.root.packageName && + node.windowId == observed.root.windowId && node != observed.root && + !AccessibilityNodeInfoUtils.hasAncestor(node, observed.root)) return // Direct show/scroll actions can be followed by the manual-scroll interpreter's focus action. // Real input clears scrollNode first, so this exemption cannot override a user's gesture. if (info.sourceAction == FocusActionInfo.MANUAL_SCROLL && @@ -140,71 +184,109 @@ class ScreenFocusRestorer internal constructor( if (scrollNode == null) schedule() return } - // A focus action can precede attachment/reconstruction of its node. Only deduplicate its - // later event after recording actually succeeds, allowing that event to finish the first save. - if (recordPosition(node)) { + val finishActivation = awaitingWebFocus && actionTime <= activationTime && + (node.isHeading || WebInterfaceUtils.supportsWebActions(node)) + val expectedScreen = awaitingWebScreen + onUserInteraction() + if (!node.isPassword && !node.isEditable && Role.getRole(node) != Role.ROLE_WEB_VIEW) { + cachedFocus = CachedFocus(AccessibilityNodeInfoCompat.obtain(node), identity(node)) lastRecordedFocusTime = maxOf(lastRecordedFocusTime, actionTime) activationOrigin = null + // Firefox may resolve a pending HTML navigation only after native activation. This is + // still part of that activation, rather than work performed on every ordinary swipe. + if (finishActivation) { + activationOrigin = recordPosition(node, expectedScreen)?.screen + cachedFocus = null + } + } else { + cachedFocus = null } } /** Capture initial/app-assigned focus too when the user activates it without first swiping. */ fun rememberBeforeActivation(node: AccessibilityNodeInfoCompat) { + if (!enabled()) return if (isBackControl(node)) { requestBack() return } - val recorded = recordPosition(node) - if (!recorded && !isDocument(node)) return - if (recorded) focusHistory.lastFocusActionRecord?.let { record -> + val document = isDocument(node) && (Role.getRole(node) == Role.ROLE_WEB_VIEW || + (!node.isHeading && !node.isClickable && node.childCount > 0)) + val cached = cachedFocus + val recorded = if (document) { + if (cached != null && cached.identity.packageName == node.packageName?.toString() && + cached.identity.windowId == node.windowId) recordCachedPosition() else null + } else recordPosition(node) + if (recorded == null && !document) return + val origin = recorded ?: context(node) + onUserInteraction() + if (recorded != null) focusHistory.lastFocusActionRecord?.let { record -> if (record.focusedNode == node) { lastRecordedFocusTime = maxOf(lastRecordedFocusTime, record.actionTime) } } - activationOrigin = context()?.screen + activationOrigin = origin?.screen + awaitingWebFocus = document && recorded == null && origin != null + awaitingWebScreen = if (awaitingWebFocus) origin?.screen else null + activationTime = SystemClock.uptimeMillis() + if (awaitingWebFocus) handler.postDelayed(releaseWebFocus, ScreenFocusBookmarks.RESTORE_TIMEOUT_MS) + cachedFocus = null } private fun isBackControl(node: AccessibilityNodeInfoCompat): Boolean { if (Role.getRole(node) != Role.ROLE_BUTTON && Role.getRole(node) != Role.ROLE_IMAGE_BUTTON) return false return node.viewIdResourceName == "android:id/home" || node.viewIdResourceName?.substringAfterLast('/') in setOf("back", "back_button", "navigate_up") || - label(node) in setOf("Back", "Navigate up") + label(node) == service.getString(androidx.appcompat.R.string.abc_action_bar_up_description) } - private fun recordPosition(node: AccessibilityNodeInfoCompat): Boolean { - if (stopped) return false - val context = context() ?: return false - if (node.windowId != context.root.windowId || node.packageName != context.root.packageName) return false + private fun recordCachedPosition(): Context? { + val cached = cachedFocus ?: return null + // A framework handle may now represent a different recycled row. Never describe or save it + // against the current screen until the cheap identity snapshot has survived a refresh. + if (!cached.node.refresh() || identity(cached.node) != cached.identity) return null + return recordPosition(cached.node) + } + + private fun recordPosition(node: AccessibilityNodeInfoCompat, + expectedScreen: RememberedScreen? = null): Context? { + if (!enabled()) return null + val context = context(node) ?: return null + if (expectedScreen != null && context.screen != expectedScreen) return null + if (node.windowId != context.root.windowId || node.packageName != context.root.packageName) return null if (!node.refresh() || (node != context.root && !AccessibilityNodeInfoUtils.hasAncestor(node, context.root))) { - return false + return null } if (node.isPassword || node.isEditable || (isDocument(node) && (Role.getRole(node) == Role.ROLE_WEB_VIEW || (!node.isHeading && !node.isClickable && node.childCount > 0)))) { - return false + return null } val item = describe(node) - if (!item.hasContent && item.uniqueId.isEmpty()) return false - onUserInteraction() + if (!item.hasContent && item.uniqueId.isEmpty()) return null history.enter(context.screen, SystemClock.uptimeMillis(), allowRestore = false) history.remember(context.screen, context.anchors, item) lastWindowId = context.root.windowId - return true + return context } /** Only explicit Back (gesture, key, or app Back button) authorizes a restoration attempt. */ fun requestBack() { - val departing = context() - // User focus and activation already record positions. Reading native focus here can return - // a WebView container instead of the actual HTML heading and erase the browser bookmark. + if (!enabled()) return onUserInteraction() + val departing = recordCachedPosition() ?: context() + cachedFocus = null + awaitingWebFocus = false + if (departing == null && ordinaryWindow) return backFrom = if (departing?.screen == activationOrigin) null else departing?.screen backWindowId = if (backFrom != null) departing?.root?.windowId else null activationOrigin = null backRequestedAt = SystemClock.uptimeMillis() - cancelledBeforeEnter = false - cancelledScreen = null + observedContext = departing + // Failed searches/actions can cancel history before a retry is posted. Release every + // temporary node and context at the original deadline even in those failure paths. + handler.postDelayed(deadline, ScreenFocusBookmarks.RESTORE_TIMEOUT_MS) schedule() } @@ -215,7 +297,7 @@ class ScreenFocusRestorer internal constructor( } fun onAccessibilityEvent(event: AccessibilityEvent) { - if (stopped) return + if (!enabled()) return when (event.eventType) { AccessibilityEvent.TYPE_TOUCH_INTERACTION_START, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER -> onUserInteraction() @@ -231,14 +313,13 @@ class ScreenFocusRestorer internal constructor( AccessibilityEvent.TYPE_VIEW_CLICKED -> { val source = event.source?.let(AccessibilityNodeInfoCompat::wrap) // This is the event produced by the Back button whose activation armed the request. - if (backRequestedAt == null || source == null || !isBackControl(source)) onUserInteraction() + if (!authorized() || source == null || !isBackControl(source)) onUserInteraction() } AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED -> { val focused = event.source?.let(AccessibilityNodeInfoCompat::wrap) - val active = context() - if (focused != null && (active == null || focused.packageName != active.root.packageName || - focused.windowId != active.root.windowId)) { + if (focused?.packageName?.toString() == "com.android.systemui") { onUserInteraction() + cachedFocus = null return } // Firefox's HTML navigation is asynchronous and bypasses the synchronous focus-set hook. @@ -318,24 +399,29 @@ class ScreenFocusRestorer internal constructor( val now = SystemClock.uptimeMillis() if (history.pending(now) == null && backRequestedAt?.let { now - it < ScreenFocusBookmarks.RESTORE_TIMEOUT_MS } != true) return null - val active = context() ?: return null + val active = observedContext ?: return null if (active.screen.document.isEmpty() && (backFrom?.packageName != active.screen.packageName || backFrom?.document.isNullOrEmpty())) return null return active } private fun queueTransition() { + generation++ + if (!authorized()) { + cachedFocus = null + awaitingWebFocus = false + awaitingWebScreen = null + observedContext = null + return + } transitionQueued = true // Window/title events may repeat throughout reconstruction. Only a changed screen or // recreated window starts a new attempt; duplicate events cannot reset its deadline or input veto. - // Observe departures immediately, even if the other app/page is only open for less than - // the settling delay. Restoration itself still waits for normal initial focus to finish. - observeScreen(context(), SystemClock.uptimeMillis()) + // Only a Back attempt pays for observing a window; all other transitions are bookkeeping. schedule() } private fun observeScreen(context: Context?, now: Long) { - if (activationOrigin != null && context?.screen != activationOrigin) activationOrigin = null val requestedAt = backRequestedAt val validBack = requestedAt != null && now - requestedAt < ScreenFocusBookmarks.RESTORE_TIMEOUT_MS if (!validBack) backRequestedAt = null @@ -349,12 +435,6 @@ class ScreenFocusRestorer internal constructor( if (returning) backRequestedAt = null lastWindowId = context?.root?.windowId if (changed) { - // Input on the departing screen must not veto a later return. Input already performed - // on the destination still cancels restoration, including content-only transitions. - if (cancelledBeforeEnter && cancelledScreen != context?.screen) { - cancelledBeforeEnter = false - cancelledScreen = null - } triedPosition = false scrollNode = null waitingForScroll = false @@ -362,6 +442,7 @@ class ScreenFocusRestorer internal constructor( } private fun schedule(delay: Long = SETTLE_DELAY_MS) { + if (!enabled() || !authorized()) return // Do not debounce forever when pages continuously update. val at = SystemClock.uptimeMillis() + delay if (at >= retryAt) return @@ -376,19 +457,22 @@ class ScreenFocusRestorer internal constructor( return } val now = SystemClock.uptimeMillis() - val context = context() - observeScreen(context, now) - transitionQueued = false - if (cancelledBeforeEnter) { - history.cancel() - cancelledBeforeEnter = false - cancelledScreen = null + if (!authorized(now)) { + onUserInteraction() + return } + val attemptGeneration = generation + val context = context() if (context == null) { - scrollNode = null - waitingForScroll = false + // A browser may temporarily have no root while reconstructing a history entry. Keep + // the original deadline and wait for content events, but never cross into an ordinary + // dialog, keyboard, or system window with this supplementary restoration path. + if (ordinaryWindow) onUserInteraction() return } + observedContext = context + observeScreen(context, now) + transitionQueued = false val bookmark = history.pending(now) ?: run { scrollNode = null waitingForScroll = false @@ -423,7 +507,7 @@ class ScreenFocusRestorer internal constructor( node.windowId == context.root.windowId && AccessibilityNodeInfoUtils.hasAncestor(node, context.root)) { if (history.pending(SystemClock.uptimeMillis()) == null || - context()?.screen != context.screen) { + generation != attemptGeneration) { history.cancel() return } @@ -451,7 +535,7 @@ class ScreenFocusRestorer internal constructor( supports(node, AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN.id) && !waitingForScroll && history.beginScroll(now, viewport)) { if (requestScroll(node, AccessibilityActionCompat.ACTION_SHOW_ON_SCREEN.id, null, - context)) return + context, attemptGeneration)) return history.cancel() return } @@ -481,7 +565,7 @@ class ScreenFocusRestorer internal constructor( } } if (requestScroll(container, AccessibilityActionCompat.ACTION_SCROLL_TO_POSITION.id, - args, context)) return + args, context, attemptGeneration)) return // A rejected action terminates this attempt instead of issuing a second action against // the same viewport (which would also bypass the ten-action budget). history.cancel() @@ -491,7 +575,7 @@ class ScreenFocusRestorer internal constructor( bookmark.item.row < rows.minOf { it.row } val action = if (backwards) AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD else AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD - if (requestScroll(container, action, null, context)) return + if (requestScroll(container, action, null, context, attemptGeneration)) return history.cancel() } } @@ -518,9 +602,9 @@ class ScreenFocusRestorer internal constructor( node.actionList.any { it.id == action } private fun requestScroll(node: AccessibilityNodeInfoCompat, action: Int, args: Bundle?, - expected: Context): Boolean { + expected: Context, attemptGeneration: Long): Boolean { if (history.pending(SystemClock.uptimeMillis()) == null) return false - if (context()?.screen != expected.screen || !node.refresh() || + if (generation != attemptGeneration || !node.refresh() || (node != expected.root && !AccessibilityNodeInfoUtils.hasAncestor(node, expected.root))) { history.cancel() return false @@ -530,14 +614,9 @@ class ScreenFocusRestorer internal constructor( scrollNode = AccessibilityNodeInfoCompat.obtain(node) scrollTime = SystemClock.uptimeMillis() waitingForScroll = true - // A no-progress retry can cancel history before the deadline callback is scheduled. Release - // the temporary node independently, while recognizing late automatic scroll focus until then. - handler.removeCallbacks(releaseScroll) - handler.postDelayed(releaseScroll, history.remaining(scrollTime)) val success = pipeline.returnFeedback(Performance.EVENT_ID_UNTRACKED, Feedback.nodeAction(node, action, args)) if (!success) { - handler.removeCallbacks(releaseScroll) waitingForScroll = false scrollNode = null } else { @@ -548,14 +627,26 @@ class ScreenFocusRestorer internal constructor( return success } - private fun context(): Context? { + private fun context(node: AccessibilityNodeInfoCompat? = null): Context? { + ordinaryWindow = false + if (!enabled()) return null if (readContext != null) return readContext.invoke() - if (FormFactorUtils.isAndroidTv()) return null - val root = service.rootInActiveWindow?.let(AccessibilityNodeInfoCompat::wrap) ?: return null + val root = (if (node != null) windows.rootFor(node) ?: windows.activeRoot() + else windows.activeRoot()) ?: return null + // Some browser virtual nodes cannot resolve their window root. The active root is a safe + // fallback only when it belongs to that exact window and package, including split screen. + if (node != null && (root.windowId != node.windowId || root.packageName != node.packageName)) return null val window = root.window ?: return null - if (window.type != AccessibilityWindowInfo.TYPE_APPLICATION) return null + if (window.type != AccessibilityWindowInfo.TYPE_APPLICATION) { + ordinaryWindow = true + return null + } val packageName = root.packageName?.toString().orEmpty() - if (packageName.isEmpty() || packageName == service.packageName) return null + if (packageName.isEmpty() || packageName == service.packageName || + packageName == "com.android.systemui") { + ordinaryWindow = true + return null + } return captureContext(root, window.title?.toString().orEmpty()) } @@ -636,6 +727,7 @@ class ScreenFocusRestorer internal constructor( var child = node var previous = "" var next = "" + var siblingBudget = 100 repeat(8) { val parent = ancestor ?: return@repeat ancestors.add(containerId(parent)) @@ -650,10 +742,15 @@ class ScreenFocusRestorer internal constructor( } // Use siblings of the item wrapper as well as siblings of its focused child. if (it == 0 || nearestContainer) { - for (index in 0 until minOf(parent.childCount, 100)) { + for (index in 0 until minOf(parent.childCount, siblingBudget)) { + siblingBudget-- if (parent.getChild(index) == child) { - if (index > 0) previous = parent.getChild(index - 1)?.let(::itemText).orEmpty() - if (index + 1 < parent.childCount) { + if (index > 0 && siblingBudget > 0) { + siblingBudget-- + previous = parent.getChild(index - 1)?.let(::itemText).orEmpty() + } + if (index + 1 < parent.childCount && siblingBudget > 0) { + siblingBudget-- next = parent.getChild(index + 1)?.let(::itemText).orEmpty() } break @@ -722,6 +819,10 @@ class ScreenFocusRestorer internal constructor( stopped = true handler.removeCallbacksAndMessages(null) scrollNode = null + cachedFocus = null + observedContext = null + awaitingWebFocus = false + awaitingWebScreen = null lastWindowId = null history.clear() } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java b/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java index 8581fe169..d4f14e577 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/preference/TalkBackPreferenceFilter.java @@ -192,7 +192,8 @@ enum TalkBackPreference { // A TV has no touchscreen for direct touch, and no gesture for the status readout. DIRECT_TOUCH(R.string.pref_direct_touch_key, HIDDEN_ON_WATCH | HIDDEN_ON_TV), STATUS_READOUT(R.string.pref_status_readout_key, HIDDEN_ON_TV), - RESTORE_FOCUS_SCROLL(R.string.pref_scroll_to_restore_previous_focus_key, HIDDEN_ON_TV), + RESTORE_FOCUS_SCROLL( + R.string.pref_scroll_to_restore_previous_focus_key, HIDDEN_ON_TV | HIDDEN_ON_WATCH), SPEECH_VOLUME(R.string.pref_speech_volume_key, HIDE_HAS_VOLUME_KEY), BRAILLE_KEYBOARD( R.string.pref_brailleime_key, diff --git a/talkback/src/main/res/values-cs/strings_backtalk.xml b/talkback/src/main/res/values-cs/strings_backtalk.xml index 029c75775..cd4b2a1d1 100644 --- a/talkback/src/main/res/values-cs/strings_backtalk.xml +++ b/talkback/src/main/res/values-cs/strings_backtalk.xml @@ -129,4 +129,8 @@ Nepovolit mluvit prostřednictvím Backtalku Umožňuje aplikaci požádat o říkání textu vaším hlasem Backtalku. Backtalk se vás zeptá, než aplikace může mluvit. - \ No newline at end of file + + Posunout k obnovení předchozího zaměření + + Při návratu na obrazovku posunout na poslední zaměřenou položku. + diff --git a/talkback/src/main/res/values/strings.xml b/talkback/src/main/res/values/strings.xml index 56d26b413..78b561df6 100644 --- a/talkback/src/main/res/values/strings.xml +++ b/talkback/src/main/res/values/strings.xml @@ -4327,5 +4327,5 @@ Do not use any markdown in your response." Scroll to restore previous focus - When returning to a screen, allow Backtalk to scroll to the last focused item. Off by default. + Scroll to the last focused item when returning to a screen. diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorerTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorerTest.kt index a9160e2c9..87e957ec5 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorerTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusRestorerTest.kt @@ -293,6 +293,7 @@ class ScreenFocusRestorerTest { val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) shadowOf(focused).setSourceNode(post.unwrap()) restorer.onAccessibilityEvent(focused) + restorer.rememberBeforeActivation(post) assertEquals(1, bookmarks.size) val front = currentContext!! currentContext = front.copy(screen = front.screen.copy(pane = "Comments")) @@ -937,6 +938,7 @@ class ScreenFocusRestorerTest { restorer.onUserInteraction() restorer.onFocusSet(automaticTop, FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + restorer.rememberBeforeActivation(automaticTop) bookmarks.enter(currentContext!!.screen, now, true) assertEquals("Automatic top", bookmarks.pending(now)!!.item.text) } @@ -952,6 +954,7 @@ class ScreenFocusRestorerTest { val newFocus = node("Later focus", parent = currentContext!!.root) restorer.onFocusSet(newFocus, FocusActionInfo.builder().setSourceAction(FocusActionInfo.MANUAL_SCROLL).build()) + restorer.rememberBeforeActivation(newFocus) val now = android.os.SystemClock.uptimeMillis() bookmarks.enter(currentContext!!.screen, now, true) assertEquals("Later focus", bookmarks.pending(now)!!.item.text) @@ -1057,6 +1060,7 @@ class ScreenFocusRestorerTest { currentContext = restorer.captureContext(root, "Browser") restorer.onFocusSet(heading, FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + restorer.rememberBeforeActivation(document) document.text = "Page B" currentContext = restorer.captureContext(root, "Browser") event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) @@ -1223,6 +1227,7 @@ class ScreenFocusRestorerTest { val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) shadowOf(focused).setSourceNode(heading.unwrap()) restorer.onAccessibilityEvent(focused) + restorer.rememberBeforeActivation(container) assertEquals(1, bookmarks.size) currentContext = page.copy(screen = page.screen.copy(document = "Another document")) event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) @@ -1242,6 +1247,7 @@ class ScreenFocusRestorerTest { currentContext = restorer.captureContext(root, "Firefox") restorer.onFocusSet(link, FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + restorer.rememberBeforeActivation(link) val now = android.os.SystemClock.uptimeMillis() bookmarks.enter(currentContext!!.screen, now, true) assertEquals("A web link", bookmarks.pending(now)!!.item.text) @@ -1275,7 +1281,7 @@ class ScreenFocusRestorerTest { val origin = currentContext!! currentContext = origin.copy(screen = origin.screen.copy(pane = "Comments")) event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) - val back = node("Back", "android.widget.Button", parent = currentContext!!.root) + val back = node("Back", "android.widget.Button", "android:id/home", parent = currentContext!!.root) restorer.rememberBeforeActivation(back) val clicked = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_CLICKED) shadowOf(clicked).setSourceNode(back.unwrap()) @@ -1285,4 +1291,33 @@ class ScreenFocusRestorerTest { settle() assertEquals(row.text, feedback.single().failovers().single().focus()!!.target()!!.text) } + + @Test fun delayedHtmlFocusFromActivationCannotBookmarkAnotherDocument() { + val root = node(className = "android.widget.FrameLayout") + val document = node("Page A", "android.webkit.WebView", parent = root) + val heading = node("Pending heading", "android.view.View", parent = document) + heading.isHeading = true + heading.addAction(AccessibilityNodeInfoCompat.ACTION_NEXT_HTML_ELEMENT) + currentContext = restorer.captureContext(root, "Browser") + val time = android.os.SystemClock.uptimeMillis() + restorer.rememberBeforeActivation(document) + document.text = "Page B" + currentContext = restorer.captureContext(root, "Browser") + restorer.onFocusSet(heading, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build(), time) + assertEquals(0, bookmarks.size) + } + + @Test fun unresolvedHtmlActivationExpiresWithoutMakingLaterSwipesReadContext() { + val root = node(className = "android.widget.FrameLayout") + val document = node("Page A", "android.webkit.WebView", parent = root) + val heading = node("Later heading", "android.view.View", parent = document) + heading.isHeading = true + currentContext = restorer.captureContext(root, "Browser") + restorer.rememberBeforeActivation(document) + settle(3200) + restorer.onFocusSet(heading, + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build()) + assertEquals(0, bookmarks.size) + } } diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusWindowAccessTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusWindowAccessTest.kt new file mode 100644 index 000000000..34e789e3a --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/focusmanagement/ScreenFocusWindowAccessTest.kt @@ -0,0 +1,366 @@ +/* Copyright 2026 Backtalk contributors. Licensed under the Apache License, Version 2.0. */ +package com.google.android.accessibility.talkback.focusmanagement + +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.Rect +import android.os.Looper +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.talkback.TalkBackService +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory +import com.google.android.accessibility.talkback.focusmanagement.record.FocusActionInfo +import com.google.android.accessibility.utils.FormFactorUtils +import java.time.Duration +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.util.ReflectionHelpers + +/** Uses the production context capture and framework window/node objects, not canned contexts. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [35]) +class ScreenFocusWindowAccessTest { + private lateinit var service: TalkBackService + private lateinit var restorer: ScreenFocusRestorer + private val history = ScreenFocusBookmarks() + private val feedback = ArrayList() + private var activeRoot: AccessibilityNodeInfoCompat? = null + private var activeReads = 0 + private var focusedReads = 0 + private var missingFocusedRoot = false + private var nextWindowId = 1 + private val navigation = FocusActionInfo.builder() + .setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + + @Before fun setUp() { + service = Robolectric.buildService(TalkBackService::class.java).get() + FormFactorUtils.initialize(service) + restorer = ScreenFocusRestorer(service, + service.getSharedPreferences("window-access", Context.MODE_PRIVATE), + Pipeline.FeedbackReturner { feedback.add(it); true }, + AccessibilityFocusActionHistory(service).reader, history, { true }, null, + object : ScreenFocusRestorer.WindowAccess { + override fun activeRoot(): AccessibilityNodeInfoCompat? { activeReads++; return activeRoot } + override fun rootFor(node: AccessibilityNodeInfoCompat): AccessibilityNodeInfoCompat? { + focusedReads++ + return if (missingFocusedRoot) null else node.window?.root + } + }) + } + + @After fun tearDown() { + restorer.shutdown() + shadowOf(service.packageManager).setSystemFeature(PackageManager.FEATURE_WATCH, false) + FormFactorUtils.initialize(service) + } + + private fun screen(title: String, packageName: String = "example.app"): + AccessibilityNodeInfoCompat { + val root = node("", "android.widget.FrameLayout", packageName) + val window = AccessibilityWindowInfo.obtain() + shadowOf(window).setType(AccessibilityWindowInfo.TYPE_APPLICATION) + shadowOf(window).setId(nextWindowId++) + shadowOf(window).setTitle(title) + shadowOf(window).setBoundsInScreen(Rect(0, 0, 100, 100)) + shadowOf(window).setRoot(root.unwrap()) + shadowOf(root.unwrap()).setAccessibilityWindowInfo(window) + return root + } + + private fun node(text: String, className: String = "android.widget.TextView", + packageName: String = "example.app", parent: AccessibilityNodeInfoCompat? = null): + AccessibilityNodeInfoCompat { + val raw = AccessibilityNodeInfo.obtain() + raw.text = text + raw.className = className + raw.packageName = packageName + raw.isVisibleToUser = true + raw.isImportantForAccessibility = true + raw.setBoundsInScreen(Rect(0, 0, 100, 100)) + if (parent != null) { + shadowOf(parent.unwrap()).addChild(raw) + shadowOf(raw).setAccessibilityWindowInfo(parent.window!!.unwrap()) + } + return AccessibilityNodeInfoCompat.wrap(raw) + } + + private fun event(type: Int) = restorer.onAccessibilityEvent(AccessibilityEvent.obtain(type)) + private fun settle(ms: Long = 160) = shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(ms)) + + @Test fun idleContentTransitionsHoverAndSwipesNeverReadRoots() { + activeRoot = screen("List") + val row = node("Post", parent = activeRoot) + repeat(40) { + restorer.onFocusSet(row, navigation) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_VIEW_HOVER_ENTER) + val focused = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) + shadowOf(focused).setSourceNode(row.unwrap()) + restorer.onAccessibilityEvent(focused) + settle() + } + assertEquals(0, activeReads) + assertEquals(0, focusedReads) + assertEquals(0, history.size) + assertTrue(feedback.isEmpty()) + } + + @Test fun activationDescribesFocusedSplitScreenWindowInsteadOfActiveWindow() { + val source = screen("Origin") + val row = node("Post", parent = source) + activeRoot = screen("Other split-screen app", "example.other") + restorer.onFocusSet(row, navigation) + restorer.rememberBeforeActivation(row) + assertEquals(0, activeReads) + assertEquals(1, focusedReads) + assertEquals("Origin", history.activeScreen!!.title) + assertEquals(1, history.size) + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + activeRoot = source + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals("Post", feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun backCapturesLatestFocusAndReadsReturningRootOncePerAttempt() { + val list = screen("List") + val row = node("Post", parent = list) + activeRoot = list + restorer.onFocusSet(row, navigation) + restorer.requestBack() + assertEquals(1, focusedReads) + assertEquals(0, activeReads) + // Navigate away, then explicitly Back to the saved list. + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + activeRoot = list + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val before = activeReads + settle() + assertEquals(before + 1, activeReads) + assertEquals(1, feedback.size) + } + + @Test fun refreshedRecycledFocusCannotBecomeANewBookmark() { + activeRoot = screen("List") + val row = node("Original", parent = activeRoot) + restorer.onFocusSet(row, navigation) + row.text = "Replacement" + // Robolectric refresh does not reload obtained copies from an app's virtual tree. Update + // that copy to model the contents a real refresh would return for the recycled source ID. + val cached = ReflectionHelpers.getField(restorer, "cachedFocus") + ReflectionHelpers.getField(cached, "node").text = "Replacement" + restorer.requestBack() + assertEquals(0, focusedReads) + assertEquals(0, history.size) + } + + @Test fun hoverCancellationAndDeadlineDoNotReadRoots() { + activeRoot = screen("Details") + restorer.requestBack() + val before = activeReads + event(AccessibilityEvent.TYPE_VIEW_HOVER_ENTER) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(4000) + assertEquals(before, activeReads) + restorer.requestBack() + settle(4000) + assertNull(ReflectionHelpers.getField(restorer, "observedContext")) + val expired = activeReads + repeat(10) { event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); settle() } + assertEquals(expired, activeReads) + } + + @Test fun systemWindowDuringBackCancelsWithoutSearchingApplication() { + val origin = screen("List") + val row = node("Post", parent = origin) + activeRoot = origin + restorer.rememberBeforeActivation(row) + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + val system = screen("Status bar", "com.android.systemui") + shadowOf(system.window!!.unwrap()).setType(AccessibilityWindowInfo.TYPE_SYSTEM) + activeRoot = system + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + activeRoot = origin + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + } + + @Test @Config(qualifiers = "cs") fun localizedNavigateUpAuthorizesBack() { + val origin = screen("List") + val row = node("Post", parent = origin) + activeRoot = origin + restorer.rememberBeforeActivation(row) + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + val up = node(service.getString(androidx.appcompat.R.string.abc_action_bar_up_description), + "android.widget.Button", parent = activeRoot) + assertNotEquals("Navigate up", up.text.toString()) + restorer.rememberBeforeActivation(up) + activeRoot = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + } + + @Test fun wearNeverReadsWindowsEvenWithScrollingEnabled() { + shadowOf(service.packageManager).setSystemFeature(PackageManager.FEATURE_WATCH, true) + FormFactorUtils.initialize(service) + service.getSharedPreferences("window-access", Context.MODE_PRIVATE).edit() + .putBoolean(ScreenFocusRestorer.SCROLL_PREFERENCE, true).commit() + activeRoot = screen("List") + val row = node("Post", parent = activeRoot) + restorer.onFocusSet(row, navigation) + restorer.rememberBeforeActivation(row) + restorer.requestBack() + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle(4000) + assertEquals(0, activeReads) + assertEquals(0, focusedReads) + assertEquals(0, history.size) + assertTrue(feedback.isEmpty()) + } + + @Test fun recreatedWindowUsesContentBookmark() { + activeRoot = screen("List") + restorer.rememberBeforeActivation(node("Post", parent = activeRoot)) + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + activeRoot = screen("List") + node("Post", parent = activeRoot) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(1, feedback.size) + } + + @Test fun sameWindowBrowserReturnWaitsForDelayedDocumentWithoutExtraRootReads() { + val browser = screen("Browser", "example.browser") + activeRoot = browser + val document = node("Page A", "android.webkit.WebView", "example.browser", browser) + val heading = node("Heading A", "android.view.View", "example.browser", document) + heading.isHeading = true + heading.isFocusable = true + restorer.onFocusSet(heading, navigation) + restorer.rememberBeforeActivation(document) + document.text = "Page B" + heading.text = "Heading B" + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + document.text = "Page A" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + val before = activeReads + heading.text = "Heading A" + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(before + 1, activeReads) + assertEquals("Heading A", feedback.single().failovers().single().focus()!!.target()!!.text) + } + + @Test fun automaticInitialFocusDoesNotReplaceCachedUserPosition() { + activeRoot = screen("List") + val row = node("Post", parent = activeRoot) + restorer.onFocusSet(row, navigation) + restorer.onFocusSet(node("Toolbar", parent = activeRoot), + FocusActionInfo.builder().setSourceAction(FocusActionInfo.SCREEN_STATE_CHANGE).build()) + restorer.requestBack() + val now = android.os.SystemClock.uptimeMillis() + history.enter(history.activeScreen, now, true) + assertEquals("Post", history.pending(now)!!.item.text) + assertEquals(0, activeReads) + assertEquals(1, focusedReads) + } + + @Test fun transientMissingRootWaitsForContentButDoesNotExtendDeadline() { + val origin = screen("List") + activeRoot = origin + restorer.rememberBeforeActivation(node("Post", parent = origin)) + activeRoot = screen("Details") + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + activeRoot = null + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertTrue(feedback.isEmpty()) + activeRoot = origin + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(1, feedback.size) + settle(3200) + val before = activeReads + activeRoot = null + event(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) + settle() + assertEquals(before, activeReads) + } + + @Test fun backFromKeyboardDoesNotArmSupplementaryRestoration() { + activeRoot = screen("List") + restorer.rememberBeforeActivation(node("Post", parent = activeRoot)) + val origin = activeRoot + activeRoot = screen("Keyboard", "example.keyboard") + shadowOf(activeRoot!!.window!!.unwrap()).setType(AccessibilityWindowInfo.TYPE_INPUT_METHOD) + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + restorer.requestBack() + val before = activeReads + activeRoot = origin + event(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + settle() + assertEquals(before, activeReads) + assertTrue(feedback.isEmpty()) + } + + @Test fun missingVirtualWindowRootFallsBackOnlyToMatchingActiveWindow() { + activeRoot = screen("List") + val row = node("Post", parent = activeRoot) + missingFocusedRoot = true + restorer.rememberBeforeActivation(row) + assertEquals(1, history.size) + assertEquals(1, activeReads) + assertEquals(1, focusedReads) + } + + @Test fun missingSplitScreenRootCannotSaveItemAgainstOtherActiveWindow() { + val origin = screen("List") + val row = node("Post", parent = origin) + activeRoot = screen("Other app", "example.other") + missingFocusedRoot = true + restorer.rememberBeforeActivation(row) + assertEquals(0, history.size) + assertEquals(1, activeReads) + assertEquals(1, focusedReads) + } + + @Test fun nativeDocumentActivationCannotCaptureFocusFromAnotherWindow() { + val origin = screen("List") + restorer.onFocusSet(node("Post", parent = origin), navigation) + activeRoot = screen("Browser", "example.browser") + val document = node("Page", "android.webkit.WebView", "example.browser", activeRoot) + restorer.rememberBeforeActivation(document) + assertEquals(0, history.size) + assertEquals(0, activeReads) + assertEquals(1, focusedReads) + } +}