Allow script speech during media playback and prioritize speech hooks

differences.md

@@ -509,6 +509,10 @@ Each script has a page with an on and off switch, a switch for each permission a
 
 Scripts never change Backtalk's own screens, so the Scripts screen always works. **Turn off all scripts** on that screen stops every script at once, and so does the **Turn all scripts on or off** action, which you can assign to a gesture that scripts can't take over. Scripts don't run while the screen is off or Backtalk is paused, and Backtalk only starts the script engine once a script is turned on.
 
+Speech requested with `backtalk.speak` can be heard while media is playing, including queued and interrupting requests. The script still needs the Speech permission, and muting speech, recording, voice recognition and phone calls retain their existing speech policy.
+
+Hooks that change or suppress speech take priority over queued script work and share a 30-millisecond deadline. If the deadline expires or a long task is already running, Backtalk uses its normal speech, so scripts should keep these hooks fast.
+
 Scripts run on phones and tablets only. [Writing Backtalk scripts](docs/scripting.md) describes the whole API, and [docs/scripting/examples](docs/scripting/examples) has a script to start from.
 
 ### Inspect focused item and Copy screen tree

docs/scripting.md

@@ -215,7 +215,7 @@ The first four can change what's said. Return:
 *   `false` or `{ silent: true }` to say nothing.
 *   Nothing, to leave it alone.
 
-Changing speech needs the `speech` permission. Backtalk waits 30 milliseconds for these hooks, then speaks without them, so keep them fast and don't `await` in them. If the script is in the middle of something that takes longer, such as a command, Backtalk doesn't wait at all.
+Changing speech needs the `speech` permission. Backtalk waits 30 milliseconds for these hooks, then speaks without them, so keep them fast and don't `await` in them. Speech hooks take priority over queued background work, and all listeners share that deadline, including queue time. A hook cannot interrupt JavaScript already running. If the script is in the middle of something that takes longer, such as a command, Backtalk doesn't wait at all.
 
 ### Commands
 

talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptFeedback.kt

@@ -41,6 +41,7 @@ import com.google.android.accessibility.utils.output.SpeechController.QUEUE_MODE
 import com.google.android.accessibility.utils.output.SpeechController.QUEUE_MODE_QUEUE
 import com.google.android.accessibility.utils.output.SpeechController.SpeakOptions
 import com.google.android.accessibility.utils.output.FailoverTextToSpeech.SpeechParam
+import com.google.android.accessibility.utils.output.FeedbackItem
 import com.google.android.accessibility.utils.output.SpeechControllerImpl
 import com.google.android.libraries.accessibility.utils.log.LogUtils
 import java.io.File
@@ -73,6 +74,7 @@ class ScriptFeedback(
     val options =
       SpeakOptions.create()
         .setQueueMode(if (interrupt) QUEUE_MODE_INTERRUPT else QUEUE_MODE_QUEUE)
+        .setFlags(FeedbackItem.FLAG_FORCE_FEEDBACK_EVEN_IF_AUDIO_PLAYBACK_ACTIVE)
         .setSpeechParams(params)
     give(Feedback.speech(inLanguage(text, voice.language), options))
   }

talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptManager.kt

@@ -42,7 +42,9 @@ class ScriptManager(
   private val mainHandler = Handler(Looper.getMainLooper())
   private val scriptHandler = startScriptThread()
   private val loaded = LinkedHashMap<String, ScriptRuntime>()
-  private val speechHooks = ScriptSpeechHooks(scriptHandler)
+  private val speechHooks = ScriptSpeechHooks(scriptHandler) {
+    !closed && speechStateVersion == appliedSpeechStateVersion
+  }
   private val events = ScriptEventDelivery(scriptHandler) { loaded.values }
   private val actions = ScriptItemActions(scriptHandler, feedback)
   private val navigator = ScriptNavigator(this)
@@ -58,6 +60,9 @@ class ScriptManager(
   private var rulesSource: List<Pair<InstalledScript, List<ScriptRule>>> = emptyList()
 
   @Volatile private var closed = false
+  // A priority hook must not jump ahead of an app change or pending script/permission update.
+  @Volatile private var speechStateVersion = 0L
+  @Volatile private var appliedSpeechStateVersion = 0L
   @Volatile private var rules = ScriptRules.EMPTY
   @Volatile private var bindings = ScriptBindings.EMPTY
 
@@ -312,7 +317,11 @@ class ScriptManager(
     rules.clearCache()
     val info = activation.appInfo
     val wanted = store.all().filter(activation::wants)
-    onScriptThread { apply(wanted, info) }
+    val version = ++speechStateVersion
+    onScriptThread {
+      apply(wanted, info)
+      appliedSpeechStateVersion = version
+    }
   }
 
   private fun apply(wanted: List<InstalledScript>, info: JSONObject) {

talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChain.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2026 Backtalk contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.accessibility.talkback.scripting
+
+import android.os.SystemClock
+
+/** All listeners share the caller's deadline, including time already spent in the queue. */
+internal fun <T> rewriteSpeechChain(
+  input: String,
+  listeners: List<T>,
+  deadline: Long,
+  now: () -> Long = SystemClock::uptimeMillis,
+  dispatch: (T, String, Long) -> String?,
+): String? {
+  var text = input
+  for (listener in listeners) {
+    val remaining = deadline - now()
+    if (remaining <= 0) return null
+    val result = dispatch(listener, text, remaining)
+    // The caller has already fallen back. Do not start more hooks or return a late rewrite.
+    if (now() >= deadline) return null
+    if (result != null) text = result
+  }
+  return text.takeIf { it != input }
+}

talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechHooks.kt

@@ -20,7 +20,10 @@ import android.os.Handler
 import android.os.SystemClock
 import org.json.JSONObject
 
-class ScriptSpeechHooks(private val handler: Handler) {
+class ScriptSpeechHooks(
+  private val handler: Handler,
+  private val ready: () -> Boolean = { true },
+) {
   @Volatile private var chains: Map<String, List<ScriptRuntime>> = emptyMap()
 
   fun update(runtimes: List<ScriptRuntime>) {
@@ -37,50 +40,52 @@ class ScriptSpeechHooks(private val handler: Handler) {
     chains = emptyMap()
   }
 
-  fun hasListeners(hook: String): Boolean = chains[hook].orEmpty().isNotEmpty()
+  fun hasListeners(hook: String): Boolean = ready() && chains[hook].orEmpty().isNotEmpty()
 
   fun rewrite(
     hook: String,
     input: String,
     accepts: (ScriptRuntime) -> Boolean = { true },
     arg: (ScriptRuntime) -> Any?,
   ): String? {
+    if (!ready()) return null
     val chain = chains[hook].orEmpty().filter(accepts).ifEmpty { return null }
     if (ScriptThread.heldUpFor(WAIT_MS)) return null
-    return handler.await(WAIT_MS, hook) { runChain(chain, hook, input, arg) }
+    val deadline = SystemClock.uptimeMillis() + WAIT_MS
+    // Speech cannot wait behind a backlog of content events and timers. It still cannot
+    // preempt JavaScript already running, and the main-thread wait remains bounded.
+    return handler.await(WAIT_MS, hook, atFrontOfQueue = true) {
+      if (ready()) runChain(chain, hook, input, deadline, arg) else null
+    }
   }
 
   private fun runChain(
     chain: List<ScriptRuntime>,
     hook: String,
     input: String,
+    deadline: Long,
     arg: (ScriptRuntime) -> Any?,
   ): String? {
     val handlerName = ScriptRuntime.handlerName(hook)
-    val text =
-      chain.filter { it.isLoaded }.fold(input) { text, runtime ->
-        val start = SystemClock.uptimeMillis()
-        val data = jsonObject("arg" to arg(runtime), "text" to text)
-        val result = runtime.dispatch(hook, data, LIMIT_MS)
-        val elapsed = SystemClock.uptimeMillis() - start
-        if (elapsed > WAIT_MS) {
-          runtime.warnOnce(
-            "slow $hook",
-            "$handlerName took $elapsed ms, so Backtalk may have spoken without it",
-          )
-        }
-        when {
-          result == null -> text
-          !runtime.allowed(ScriptPermission.SPEECH, "Changing speech from $handlerName") -> text
-          else -> JSONObject(result).optString("text", text)
-        }
+    return rewriteSpeechChain(input, chain.filter { it.isLoaded }, deadline) { runtime, text, budget ->
+      val data = jsonObject("arg" to arg(runtime), "text" to text)
+      // Building a node snapshot also spends the budget; do not start JS after it expires.
+      val remaining = minOf(budget, deadline - SystemClock.uptimeMillis())
+      if (remaining <= 0) return@rewriteSpeechChain null
+      val result = runtime.dispatch(hook, data, remaining)
+      if (SystemClock.uptimeMillis() >= deadline) {
+        runtime.warnOnce("slow $hook", "$handlerName exceeded the speech-hook deadline")
+      }
+      when {
+        result == null -> null
+        !runtime.allowed(ScriptPermission.SPEECH, "Changing speech from $handlerName") -> null
+        else -> JSONObject(result).optString("text", text)
       }
-    return text.takeIf { it != input }
+    }
   }
 
   private companion object {
     const val WAIT_MS = 30L
-    const val LIMIT_MS = 250L
     val PERMISSIONS: Map<String, ScriptPermission?> =
       mapOf(
         // The focus hook is given the item and what Backtalk will say for it, in every app.

talkback/src/phone/java/com/google/android/accessibility/talkback/scripting/ScriptWait.kt

@@ -25,14 +25,19 @@ import java.util.concurrent.TimeoutException
 
 private const val TAG = "ScriptWait"
 
-fun <T> Handler.await(waitMs: Long, what: String, task: () -> T): T? {
+fun <T> Handler.await(
+  waitMs: Long,
+  what: String,
+  atFrontOfQueue: Boolean = false,
+  task: () -> T,
+): T? {
   if (looper.isCurrentThread) return null
   val future = FutureTask<T> { task() }
-  post(future)
+  val posted = if (atFrontOfQueue) postAtFrontOfQueue(future) else post(future)
+  if (!posted) return null
   return try {
     future.get(waitMs, TimeUnit.MILLISECONDS)
   } catch (e: TimeoutException) {
-    future.cancel(false)
     LogUtils.w(TAG, "Scripts took longer than %d ms for %s", waitMs, what)
     null
   } catch (e: ExecutionException) {
@@ -41,5 +46,9 @@ fun <T> Handler.await(waitMs: Long, what: String, task: () -> T): T? {
   } catch (e: InterruptedException) {
     Thread.currentThread().interrupt()
     null
+  } finally {
+    // A request whose caller stopped waiting must not remain ahead of later speech hooks.
+    future.cancel(false)
+    removeCallbacks(future)
   }
 }

talkback/src/testPhone/java/com/google/android/accessibility/talkback/scripting/ScriptSpeechChainTest.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2026 Backtalk contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.accessibility.talkback.scripting
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNull
+import org.junit.Test
+
+class ScriptSpeechChainTest {
+  @Test
+  fun queuedTimeAndEarlierListenersShareOneBudget() {
+    var clock = 10L // Ten milliseconds already spent waiting in the queue.
+    val budgets = mutableListOf<Long>()
+    val result = rewriteSpeechChain("original", listOf(1, 2), 30, { clock }) { _, text, budget ->
+      budgets.add(budget)
+      clock += 8
+      "$text changed"
+    }
+    assertEquals(listOf(20L, 12L), budgets)
+    assertEquals("original changed changed", result)
+  }
+
+  @Test
+  fun expiredQueuedRequestNeverCallsAnyListener() {
+    var calls = 0
+    val result = rewriteSpeechChain("original", listOf(1), 30, { 30L }) { _, _, _ ->
+      calls++
+      "changed"
+    }
+    assertNull(result)
+    assertEquals(0, calls)
+  }
+
+  @Test
+  fun overrunDiscardsTheLateRewriteAndDoesNotRunLaterListeners() {
+    var clock = 0L
+    var calls = 0
+    val result = rewriteSpeechChain("original", listOf(1, 2), 30, { clock }) { _, _, _ ->
+      calls++
+      clock = 31
+      ""
+    }
+    assertNull(result)
+    assertEquals(1, calls)
+  }
+
+  @Test
+  fun timelySuppressionAndUnchangedResultsRemainDistinct() {
+    assertEquals("", rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, _, _ -> "" })
+    assertNull(rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, _, _ -> null })
+    assertNull(rewriteSpeechChain("original", listOf(1), 30, { 0L }) { _, text, _ -> text })
+  }
+}