diff --git a/differences.md b/differences.md index 2bd759db..db29f181 100644 --- a/differences.md +++ b/differences.md @@ -509,6 +509,10 @@ Each script has a page with an on and off switch, a switch for each permission a Scripts never change Backtalk's own screens, so the Scripts screen always works. **Turn off all scripts** on that screen stops every script at once, and so does the **Turn all scripts on or off** action, which you can assign to a gesture that scripts can't take over. Scripts don't run while the screen is off or Backtalk is paused, and Backtalk only starts the script engine once a script is turned on. +Speech requested with `backtalk.speak` can be heard while media is playing, including queued and interrupting requests. The script still needs the Speech permission, and muting speech, recording, voice recognition and phone calls retain their existing speech policy. + +Hooks that change or suppress speech take priority over queued script work and share a 30-millisecond deadline. If the deadline expires or a long task is already running, Backtalk uses its normal speech, so scripts should keep these hooks fast. + Scripts run on phones and tablets only. [Writing Backtalk scripts](docs/scripting.md) describes the whole API, and [docs/scripting/examples](docs/scripting/examples) has a script to start from. ### Inspect focused item and Copy screen tree diff --git a/docs/scripting.md b/docs/scripting.md index 4c4080f5..bf5f1f49 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -215,7 +215,7 @@ The first four can change what's said. Return: * `false` or `{ silent: true }` to say nothing. * Nothing, to leave it alone. -Changing speech needs the `speech` permission. Backtalk waits 30 milliseconds for these hooks, then speaks without them, so keep them fast and don't `await` in them. If the script is in the middle of something that takes longer, such as a command, Backtalk doesn't wait at all. +Changing speech needs the `speech` permission. Backtalk waits 30 milliseconds for these hooks, then speaks without them, so keep them fast and don't `await` in them. Speech hooks take priority over queued background work, and all listeners share that deadline, including queue time. A hook cannot interrupt JavaScript already running. If the script is in the middle of something that takes longer, such as a command, Backtalk doesn't wait at all. ### Commands diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptFeedback.kt b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptFeedback.kt index d05842d3..1edbdc13 100644 --- a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptFeedback.kt +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptFeedback.kt @@ -41,6 +41,7 @@ import com.google.android.accessibility.utils.output.SpeechController.QUEUE_MODE import com.google.android.accessibility.utils.output.SpeechController.QUEUE_MODE_QUEUE import com.google.android.accessibility.utils.output.SpeechController.SpeakOptions import com.google.android.accessibility.utils.output.FailoverTextToSpeech.SpeechParam +import com.google.android.accessibility.utils.output.FeedbackItem import com.google.android.accessibility.utils.output.SpeechControllerImpl import com.google.android.libraries.accessibility.utils.log.LogUtils import java.io.File @@ -73,6 +74,7 @@ class ScriptFeedback( val options = SpeakOptions.create() .setQueueMode(if (interrupt) QUEUE_MODE_INTERRUPT else QUEUE_MODE_QUEUE) + .setFlags(FeedbackItem.FLAG_FORCE_FEEDBACK_EVEN_IF_AUDIO_PLAYBACK_ACTIVE) .setSpeechParams(params) give(Feedback.speech(inLanguage(text, voice.language), options)) } diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptManager.kt b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptManager.kt index 4db3f6c4..ea13d973 100644 --- a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptManager.kt +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptManager.kt @@ -42,7 +42,9 @@ class ScriptManager( private val mainHandler = Handler(Looper.getMainLooper()) private val scriptHandler = startScriptThread() private val loaded = LinkedHashMap() - private val speechHooks = ScriptSpeechHooks(scriptHandler) + private val speechHooks = ScriptSpeechHooks(scriptHandler) { + !closed && speechStateVersion == appliedSpeechStateVersion + } private val events = ScriptEventDelivery(scriptHandler) { loaded.values } private val actions = ScriptItemActions(scriptHandler, feedback) private val navigator = ScriptNavigator(this) @@ -58,6 +60,9 @@ class ScriptManager( private var rulesSource: List>> = emptyList() @Volatile private var closed = false + // A priority hook must not jump ahead of an app change or pending script/permission update. + @Volatile private var speechStateVersion = 0L + @Volatile private var appliedSpeechStateVersion = 0L @Volatile private var rules = ScriptRules.EMPTY @Volatile private var bindings = ScriptBindings.EMPTY @@ -312,7 +317,11 @@ class ScriptManager( rules.clearCache() val info = activation.appInfo val wanted = store.all().filter(activation::wants) - onScriptThread { apply(wanted, info) } + val version = ++speechStateVersion + onScriptThread { + apply(wanted, info) + appliedSpeechStateVersion = version + } } private fun apply(wanted: List, info: JSONObject) { diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChain.kt b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChain.kt new file mode 100644 index 00000000..ebbdb58d --- /dev/null +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChain.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.accessibility.talkback.scripting + +import android.os.SystemClock + +/** All listeners share the caller's deadline, including time already spent in the queue. */ +internal fun rewriteSpeechChain( + input: String, + listeners: List, + deadline: Long, + now: () -> Long = SystemClock::uptimeMillis, + dispatch: (T, String, Long) -> String?, +): String? { + var text = input + for (listener in listeners) { + val remaining = deadline - now() + if (remaining <= 0) return null + val result = dispatch(listener, text, remaining) + // The caller has already fallen back. Do not start more hooks or return a late rewrite. + if (now() >= deadline) return null + if (result != null) text = result + } + return text.takeIf { it != input } +} diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechHooks.kt b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechHooks.kt index f72f110c..d798d4a9 100644 --- a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechHooks.kt +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechHooks.kt @@ -20,7 +20,10 @@ import android.os.Handler import android.os.SystemClock import org.json.JSONObject -class ScriptSpeechHooks(private val handler: Handler) { +class ScriptSpeechHooks( + private val handler: Handler, + private val ready: () -> Boolean = { true }, +) { @Volatile private var chains: Map> = emptyMap() fun update(runtimes: List) { @@ -37,7 +40,7 @@ class ScriptSpeechHooks(private val handler: Handler) { chains = emptyMap() } - fun hasListeners(hook: String): Boolean = chains[hook].orEmpty().isNotEmpty() + fun hasListeners(hook: String): Boolean = ready() && chains[hook].orEmpty().isNotEmpty() fun rewrite( hook: String, @@ -45,42 +48,44 @@ class ScriptSpeechHooks(private val handler: Handler) { accepts: (ScriptRuntime) -> Boolean = { true }, arg: (ScriptRuntime) -> Any?, ): String? { + if (!ready()) return null val chain = chains[hook].orEmpty().filter(accepts).ifEmpty { return null } if (ScriptThread.heldUpFor(WAIT_MS)) return null - return handler.await(WAIT_MS, hook) { runChain(chain, hook, input, arg) } + val deadline = SystemClock.uptimeMillis() + WAIT_MS + // Speech cannot wait behind a backlog of content events and timers. It still cannot + // preempt JavaScript already running, and the main-thread wait remains bounded. + return handler.await(WAIT_MS, hook, atFrontOfQueue = true) { + if (ready()) runChain(chain, hook, input, deadline, arg) else null + } } private fun runChain( chain: List, hook: String, input: String, + deadline: Long, arg: (ScriptRuntime) -> Any?, ): String? { val handlerName = ScriptRuntime.handlerName(hook) - val text = - chain.filter { it.isLoaded }.fold(input) { text, runtime -> - val start = SystemClock.uptimeMillis() - val data = jsonObject("arg" to arg(runtime), "text" to text) - val result = runtime.dispatch(hook, data, LIMIT_MS) - val elapsed = SystemClock.uptimeMillis() - start - if (elapsed > WAIT_MS) { - runtime.warnOnce( - "slow $hook", - "$handlerName took $elapsed ms, so Backtalk may have spoken without it", - ) - } - when { - result == null -> text - !runtime.allowed(ScriptPermission.SPEECH, "Changing speech from $handlerName") -> text - else -> JSONObject(result).optString("text", text) - } + return rewriteSpeechChain(input, chain.filter { it.isLoaded }, deadline) { runtime, text, budget -> + val data = jsonObject("arg" to arg(runtime), "text" to text) + // Building a node snapshot also spends the budget; do not start JS after it expires. + val remaining = minOf(budget, deadline - SystemClock.uptimeMillis()) + if (remaining <= 0) return@rewriteSpeechChain null + val result = runtime.dispatch(hook, data, remaining) + if (SystemClock.uptimeMillis() >= deadline) { + runtime.warnOnce("slow $hook", "$handlerName exceeded the speech-hook deadline") + } + when { + result == null -> null + !runtime.allowed(ScriptPermission.SPEECH, "Changing speech from $handlerName") -> null + else -> JSONObject(result).optString("text", text) } - return text.takeIf { it != input } + } } private companion object { const val WAIT_MS = 30L - const val LIMIT_MS = 250L val PERMISSIONS: Map = mapOf( // The focus hook is given the item and what Backtalk will say for it, in every app. diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptWait.kt b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptWait.kt index 199f1139..668a0860 100644 --- a/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptWait.kt +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptWait.kt @@ -25,14 +25,19 @@ import java.util.concurrent.TimeoutException private const val TAG = "ScriptWait" -fun Handler.await(waitMs: Long, what: String, task: () -> T): T? { +fun Handler.await( + waitMs: Long, + what: String, + atFrontOfQueue: Boolean = false, + task: () -> T, +): T? { if (looper.isCurrentThread) return null val future = FutureTask { task() } - post(future) + val posted = if (atFrontOfQueue) postAtFrontOfQueue(future) else post(future) + if (!posted) return null return try { future.get(waitMs, TimeUnit.MILLISECONDS) } catch (e: TimeoutException) { - future.cancel(false) LogUtils.w(TAG, "Scripts took longer than %d ms for %s", waitMs, what) null } catch (e: ExecutionException) { @@ -41,5 +46,9 @@ fun Handler.await(waitMs: Long, what: String, task: () -> T): T? { } catch (e: InterruptedException) { Thread.currentThread().interrupt() null + } finally { + // A request whose caller stopped waiting must not remain ahead of later speech hooks. + future.cancel(false) + removeCallbacks(future) } } diff --git a/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChainTest.kt b/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChainTest.kt new file mode 100644 index 00000000..6350f09e --- /dev/null +++ b/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChainTest.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.accessibility.talkback.scripting + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ScriptSpeechChainTest { + @Test + fun queuedTimeAndEarlierListenersShareOneBudget() { + var clock = 10L // Ten milliseconds already spent waiting in the queue. + val budgets = mutableListOf() + val result = rewriteSpeechChain("original", listOf(1, 2), 30, { clock }) { _, text, budget -> + budgets.add(budget) + clock += 8 + "$text changed" + } + assertEquals(listOf(20L, 12L), budgets) + assertEquals("original changed changed", result) + } + + @Test + fun expiredQueuedRequestNeverCallsAnyListener() { + var calls = 0 + val result = rewriteSpeechChain("original", listOf(1), 30, { 30L }) { _, _, _ -> + calls++ + "changed" + } + assertNull(result) + assertEquals(0, calls) + } + + @Test + fun overrunDiscardsTheLateRewriteAndDoesNotRunLaterListeners() { + var clock = 0L + var calls = 0 + val result = rewriteSpeechChain("original", listOf(1, 2), 30, { clock }) { _, _, _ -> + calls++ + clock = 31 + "" + } + assertNull(result) + assertEquals(1, calls) + } + + @Test + fun timelySuppressionAndUnchangedResultsRemainDistinct() { + assertEquals("", rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, _, _ -> "" }) + assertNull(rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, _, _ -> null }) + assertNull(rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, text, _ -> text }) + } +} diff --git a/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptWaitTest.kt b/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptWaitTest.kt new file mode 100644 index 00000000..96d0eb85 --- /dev/null +++ b/talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptWaitTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.accessibility.talkback.scripting + +import android.app.Application +import android.os.Handler +import android.os.Looper +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.LooperMode + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, application = Application::class, sdk = [34]) +@LooperMode(LooperMode.Mode.PAUSED) +class ScriptWaitTest { + private val handler = Handler(Looper.getMainLooper()) + + @Test + fun speechRequestRunsBeforeQueuedBackgroundWork() { + val order = mutableListOf() + handler.post { order.add("background") } + var result: String? = null + val caller = Thread { + result = handler.await(2000, "speech", atFrontOfQueue = true) { + order.add("speech") + "rewritten" + } + } + caller.start() + awaitWaiting(caller) + shadowOf(handler.looper).idle() + caller.join(2000) + assertFalse(caller.isAlive) + assertEquals("rewritten", result) + assertEquals(listOf("speech", "background"), order) + } + + @Test + fun ordinaryRequestsKeepQueueOrder() { + val order = mutableListOf() + handler.post { order.add("background") } + val caller = Thread { handler.await(2000, "actions") { order.add("actions") } } + caller.start() + awaitWaiting(caller) + shadowOf(handler.looper).idle() + caller.join(2000) + assertFalse(caller.isAlive) + assertEquals(listOf("background", "actions"), order) + } + + @Test + fun timedOutRequestDoesNotRunWhenTheQueueResumes() { + var called = false + var result: String? = "pending" + val caller = Thread { + result = handler.await(10, "speech", atFrontOfQueue = true) { + called = true + "late" + } + } + caller.start() + caller.join(2000) + assertFalse(caller.isAlive) + shadowOf(handler.looper).idle() + assertNull(result) + assertFalse(called) + } + + @Test + fun sameThreadRequestDoesNotReenterTheRuntime() { + var called = false + assertNull(handler.await(30, "speech", atFrontOfQueue = true) { called = true }) + shadowOf(handler.looper).idle() + assertFalse(called) + } + + private fun awaitWaiting(thread: Thread) { + val deadline = System.nanoTime() + 2_000_000_000L + while (thread.isAlive && thread.state != Thread.State.TIMED_WAITING && + System.nanoTime() < deadline) { + Thread.yield() + } + assertTrue("Caller must have queued its task and entered the bounded wait", + thread.state == Thread.State.TIMED_WAITING) + } +}