diff --git a/packages/accessibility-android-sys/resources/AxDump.java b/packages/accessibility-android-sys/resources/AxDump.java new file mode 100644 index 0000000..02d288c --- /dev/null +++ b/packages/accessibility-android-sys/resources/AxDump.java @@ -0,0 +1,194 @@ +import android.accessibilityservice.AccessibilityServiceInfo; +import android.app.UiAutomation; +import android.os.HandlerThread; +import android.os.SystemClock; +import android.view.accessibility.AccessibilityNodeInfo; +import android.graphics.Rect; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +/** + * Persistent accessibility dump server for ADB. Runs under app_process with + * shell permissions, connects a UiAutomation once, then serves repeated + * hierarchy dumps over stdin/stdout: read the line "dump", write the tree + * as uiautomator-compatible XML terminated by "###END###". + * + * This avoids the ~2s per-call cost of `uiautomator dump`, which spawns a + * fresh JVM, connects UiAutomation, dumps once, and exits. + */ +public final class AxDump { + private AxDump() {} + + /** + * Hidden {@code AccessibilityNodeInfo.getSourceNodeId()}: a per-window + * stable identity for the underlying view (virtual descendant id packed + * with the accessibility view id). Survives text/label changes and moves, + * so hosts can tell "same control, new state" from "replaced control". + */ + private static Method sourceNodeId; + + static { + try { + sourceNodeId = AccessibilityNodeInfo.class.getDeclaredMethod("getSourceNodeId"); + sourceNodeId.setAccessible(true); + } catch (Exception ignored) { + sourceNodeId = null; + } + } + + public static void main(String[] args) throws Exception { + HandlerThread thread = new HandlerThread("axdump"); + thread.start(); + + UiAutomation automation = connect(thread); + AccessibilityServiceInfo info = automation.getServiceInfo(); + if (info != null) { + info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS + | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS; + automation.setServiceInfo(info); + } + + System.out.println("axdump ready"); + System.out.flush(); + + BufferedReader stdin = new BufferedReader(new InputStreamReader(System.in)); + String line; + while ((line = stdin.readLine()) != null) { + line = line.trim(); + if (line.equals("quit")) { + break; + } + if (!line.equals("dump")) { + continue; + } + long started = SystemClock.uptimeMillis(); + AccessibilityNodeInfo root = automation.getRootInActiveWindow(); + StringBuilder out = new StringBuilder(16 * 1024); + if (root == null) { + out.append("error: no active window root\n"); + } else { + out.append(""); + out.append(""); + emit(root, 0, out); + out.append("\n"); + } + out.append("###END### ").append(SystemClock.uptimeMillis() - started).append("ms\n"); + System.out.print(out); + System.out.flush(); + } + // Unregister the UiTestAutomationService before exiting: a bare + // System.exit can leave the registration alive in the accessibility + // manager long enough that the next AxDump start dies with + // "UiAutomationService ... already registered!". + try { + Method disconnect = UiAutomation.class.getDeclaredMethod("disconnect"); + disconnect.setAccessible(true); + disconnect.invoke(automation); + } catch (Exception ignored) { + } + System.exit(0); + } + + private static UiAutomation connect(HandlerThread thread) throws Exception { + // UiAutomationConnection and IUiAutomationConnection are hidden APIs: + // reach them purely reflectively so android.jar can compile this. + Class connectionClass = Class.forName("android.app.UiAutomationConnection"); + Object connection = connectionClass.getDeclaredConstructor().newInstance(); + Class interfaceClass = Class.forName("android.app.IUiAutomationConnection"); + Constructor constructor = UiAutomation.class.getDeclaredConstructor( + android.os.Looper.class, interfaceClass); + constructor.setAccessible(true); + UiAutomation automation = constructor.newInstance(thread.getLooper(), connection); + Method connect; + try { + connect = UiAutomation.class.getDeclaredMethod("connect", int.class); + connect.setAccessible(true); + connect.invoke(automation, 0); + } catch (NoSuchMethodException e) { + connect = UiAutomation.class.getDeclaredMethod("connect"); + connect.setAccessible(true); + connect.invoke(automation); + } + return automation; + } + + private static void emit(AccessibilityNodeInfo node, int index, StringBuilder out) { + if (node == null || !node.isVisibleToUser()) { + // uiautomator dump prunes invisible subtrees (unrealized/off-screen + // list rows report clamped, inverted bounds); match that. + return; + } + Rect bounds = new Rect(); + node.getBoundsInScreen(bounds); + out.append(""); + return; + } + out.append('>'); + for (int i = 0; i < count; i++) { + emit(node.getChild(i), i, out); + } + out.append(""); + } + + private static void attr(StringBuilder out, String name, CharSequence value) { + out.append(' ').append(name).append("=\""); + if (value != null) { + String text = value.toString(); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + switch (c) { + case '&': out.append("&"); break; + case '<': out.append("<"); break; + case '>': out.append(">"); break; + case '"': out.append("""); break; + case '\'': out.append("'"); break; + default: + if (c < 0x20 && c != '\t') { + out.append(' '); + } else { + out.append(c); + } + } + } + } + out.append('"'); + } + + private static void flag(StringBuilder out, String name, boolean value) { + out.append(' ').append(name).append("=\"").append(value).append('"'); + } +} diff --git a/packages/accessibility-android-sys/resources/axdump.dex b/packages/accessibility-android-sys/resources/axdump.dex new file mode 100644 index 0000000..aeaab6f Binary files /dev/null and b/packages/accessibility-android-sys/resources/axdump.dex differ diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 45fa586..88299c8 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -8,6 +8,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use keyboard_types::Code; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; const UI_DUMP_ATTEMPTS: usize = 3; const UI_DUMP_RETRY_DELAY: Duration = Duration::from_millis(500); @@ -754,7 +755,15 @@ impl AdbClient { } async fn dump_ui_once(&self) -> Result { - let result = self.shell(&["uiautomator", "dump", "/dev/tty"]).await; + // uiautomator's stdout is not forwarded over the shell-v2 service (only + // the trailer arrives), so shell()-based dumps always fell through to + // the tmp-file fallback — a second full uiautomator run, ~2s extra on + // an emulator. The exec: service does forward it, and /dev/tty resolves + // to that stream, so a single run returns the XML directly. + let result = self + .exec_out(&["uiautomator", "dump", "/dev/tty"]) + .await + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()); match result { Ok(output) => match extract_ui_xml(&output) { @@ -798,6 +807,83 @@ impl AdbClient { } } + /// Push `data` to `device_path` (equivalent to `adb push`). + pub async fn push(&self, device_path: &str, data: &[u8]) -> Result<()> { + let transport = self.transport(); + self.run( + "push", + transport.push(self.serial.as_deref(), device_path, data, 0o644), + ) + .await + } + + /// Push the bundled AxDump dex to the device, ready for + /// [`Self::start_dump_server`]. + pub async fn ensure_axdump(&self) -> Result<()> { + self.push(AXDUMP_DEX_DEVICE_PATH, AXDUMP_DEX).await + } + + /// Start the persistent accessibility dump server (experimental). + /// + /// Requires the AxDump dex at `dex_device_path` (push it first with + /// `adb push classes.dex /data/local/tmp/axdump.dex`). The returned + /// server holds one `UiAutomation` connection per display, so only one + /// instance may run at a time; steady-state dumps are ~2ms versus ~2s + /// for `uiautomator dump`. Callers must fall back to [`Self::dump_ui`] + /// on any server error. + /// + /// `UiAutomation` registration is first-come: while any client (this + /// server or a running `uiautomator dump`) owns the display, a new + /// server is killed at startup ("already registered"). After a clean + /// [`UiDumpServer::shutdown`] the registration takes a couple of seconds + /// to release, so startup retries with backoff before giving up. + pub async fn start_dump_server(&self, dex_device_path: &str) -> Result { + const BACKOFF: [Duration; 3] = [ + Duration::from_millis(500), + Duration::from_millis(1_500), + Duration::from_millis(3_000), + ]; + let mut last_error = None; + for (attempt, delay) in BACKOFF.iter().enumerate() { + match self.start_dump_server_once(dex_device_path).await { + Ok(server) => return Ok(server), + Err(error) => { + last_error = Some(error); + if attempt + 1 < BACKOFF.len() { + tokio::time::sleep(*delay).await; + } + } + } + } + Err(last_error.expect("at least one startup attempt")) + } + + async fn start_dump_server_once(&self, dex_device_path: &str) -> Result { + let command = format!("CLASSPATH={dex_device_path} app_process /system/bin AxDump"); + let transport = self.transport(); + let stream = self + .run( + "exec", + transport.exec_stream(self.serial.as_deref(), &[&command]), + ) + .await?; + let mut server = UiDumpServer { + stream, + buffer: Vec::new(), + timeout: self.timeout, + }; + let ready = tokio::time::timeout(self.timeout, server.read_line()) + .await + .map_err(|_| anyhow::anyhow!("AxDump server did not report ready in time"))??; + if !ready.contains("axdump ready") { + bail!( + "AxDump server startup failed: {}", + truncate_for_error(&ready) + ); + } + Ok(server) + } + /// Launch an app by package name and optional activity. pub async fn launch_app(&self, package: &str, activity: Option<&str>) -> Result<()> { match activity { @@ -849,6 +935,79 @@ impl AdbClient { } } +/// The bundled AxDump persistent dump server dex (source: +/// `resources/AxDump.java`, compiled with `javac` + `d8`). +pub const AXDUMP_DEX: &[u8] = include_bytes!("../resources/axdump.dex"); + +/// Where [`AdbClient::ensure_axdump`] installs the dex on the device. +pub const AXDUMP_DEX_DEVICE_PATH: &str = "/data/local/tmp/axdump.dex"; + +/// A live connection to the persistent AxDump accessibility server +/// (see [`AdbClient::start_dump_server`]). Experimental: any protocol error +/// poisons the connection; drop it and fall back to `uiautomator dump`. +pub struct UiDumpServer { + stream: tokio::net::TcpStream, + buffer: Vec, + timeout: Duration, +} + +impl UiDumpServer { + /// Request one hierarchy dump; returns uiautomator-compatible XML. + /// + /// A `no active window root` response (a transient race right after + /// startup or during window transitions) is surfaced as an error the + /// caller may retry. + pub async fn dump(&mut self) -> Result { + tokio::time::timeout(self.timeout, self.dump_inner()) + .await + .map_err(|_| anyhow::anyhow!("AxDump dump request timed out"))? + } + + async fn dump_inner(&mut self) -> Result { + self.stream.write_all(b"dump\n").await?; + let mut body = String::new(); + loop { + let line = self.read_line().await?; + if line.starts_with("###END###") { + break; + } + body.push_str(&line); + body.push('\n'); + } + if body.starts_with("error:") { + bail!("AxDump server error: {}", body.trim()); + } + extract_ui_xml(&body) + .with_context(|| format!("AxDump output was not XML: {}", truncate_for_error(&body))) + } + + /// Ask the server to exit and release its `UiAutomation` connection. + /// + /// While the server is running, other `UiAutomation` clients — including + /// `uiautomator dump` — are killed by the system (only one client may + /// own a display), so releasing before handing control elsewhere matters. + pub async fn shutdown(mut self) -> Result<()> { + self.stream.write_all(b"quit\n").await?; + let _ = self.stream.shutdown().await; + Ok(()) + } + + async fn read_line(&mut self) -> Result { + loop { + if let Some(position) = self.buffer.iter().position(|byte| *byte == b'\n') { + let line: Vec = self.buffer.drain(..=position).collect(); + return Ok(String::from_utf8_lossy(&line[..line.len() - 1]).into_owned()); + } + let mut chunk = [0; 8192]; + let read = self.stream.read(&mut chunk).await?; + if read == 0 { + bail!("AxDump server stream closed"); + } + self.buffer.extend_from_slice(&chunk[..read]); + } + } +} + fn adb_binary_name() -> &'static str { if cfg!(target_os = "windows") { "adb.exe" @@ -969,6 +1128,104 @@ mod tests { ); } + /// Serve one scripted response to a `dump` request on a loopback socket + /// and return a `UiDumpServer` connected to it, exercising the real + /// framing path without a device. + async fn scripted_server(response: &'static [u8], timeout: Duration) -> UiDumpServer { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept"); + let mut request = [0; 64]; + let _ = socket.read(&mut request).await; + let _ = socket.write_all(response).await; + // Keep the socket open so short reads are the timeout's problem, + // not an EOF: a real server blocks between requests too. + tokio::time::sleep(Duration::from_secs(5)).await; + }); + UiDumpServer { + stream: tokio::net::TcpStream::connect(addr).await.expect("connect"), + buffer: Vec::new(), + timeout, + } + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn dump_server_parses_framed_xml() { + let mut server = scripted_server( + b"\n###END###\n", + Duration::from_secs(2), + ) + .await; + let xml = server.dump().await.expect("dump"); + assert!(xml.contains("\n", + Duration::from_millis(200), + ) + .await; + let error = server.dump().await.unwrap_err(); + assert!(error.to_string().contains("timed out")); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn dump_server_reports_closed_stream() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept"); + drop(socket); + }); + let mut server = UiDumpServer { + stream: tokio::net::TcpStream::connect(addr).await.expect("connect"), + buffer: Vec::new(), + timeout: Duration::from_secs(2), + }; + // A dropped peer surfaces as either a clean EOF ("stream closed") + // or an ECONNRESET depending on whether the RST beats the read. + let error = server.dump().await.unwrap_err(); + let text = error.to_string(); + assert!( + text.contains("stream closed") || text.contains("reset"), + "{text}" + ); + } + #[cfg(unix)] #[tokio::test] #[cfg_attr(miri, ignore)] diff --git a/packages/accessibility-android-sys/src/transport.rs b/packages/accessibility-android-sys/src/transport.rs index 855b568..ad23ea5 100644 --- a/packages/accessibility-android-sys/src/transport.rs +++ b/packages/accessibility-android-sys/src/transport.rs @@ -60,6 +60,65 @@ impl AdbTransport { read_to_end_limited(&mut stream, MAX_OUTPUT_LENGTH, "exec output").await } + /// Open a bidirectional `exec:` stream to a long-lived device command. + /// + /// Unlike [`Self::exec`], the stream is returned after the OKAY status so + /// the caller can keep writing to the command's stdin and reading its + /// stdout across many requests (e.g. a persistent dump server). + pub(crate) async fn exec_stream( + &self, + serial: Option<&str>, + args: &[&str], + ) -> Result { + let service = format!("exec:{}", args.join(" ")); + let mut stream = self.switch_to_device(serial).await?; + write_service(&mut stream, &service).await?; + read_status(&mut stream).await?; + Ok(stream) + } + + /// Write `data` to `device_path` via the `sync:` service. + pub(crate) async fn push( + &self, + serial: Option<&str>, + device_path: &str, + data: &[u8], + mode: u32, + ) -> Result<()> { + let mut stream = self.switch_to_device(serial).await?; + write_service(&mut stream, "sync:").await?; + read_status(&mut stream).await?; + + let target = format!("{device_path},{mode}"); + write_sync_request(&mut stream, b"SEND", target.as_bytes()).await?; + for chunk in data.chunks(SYNC_DATA_CHUNK_LENGTH) { + write_sync_request(&mut stream, b"DATA", chunk).await?; + } + let mtime = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() as u32) + .unwrap_or(0); + stream.write_all(b"DONE").await?; + stream.write_all(&mtime.to_le_bytes()).await?; + + let mut reply = [0; 8]; + stream + .read_exact(&mut reply) + .await + .context("truncated sync push reply")?; + match &reply[..4] { + b"OKAY" => Ok(()), + b"FAIL" => { + let length = + (u32::from_le_bytes(reply[4..].try_into().unwrap()) as usize).min(4096); + let mut message = vec![0; length]; + stream.read_exact(&mut message).await.ok(); + bail!("sync push failed: {}", String::from_utf8_lossy(&message)); + } + _ => bail!("malformed sync push reply"), + } + } + async fn connect(&self) -> Result { match TcpStream::connect(self.server_addr).await { Ok(stream) => Ok(stream), @@ -164,6 +223,15 @@ async fn write_service(stream: &mut TcpStream, service: &str) -> Result<()> { Ok(()) } +async fn write_sync_request(stream: &mut TcpStream, id: &[u8; 4], payload: &[u8]) -> Result<()> { + stream.write_all(id).await?; + stream + .write_all(&(payload.len() as u32).to_le_bytes()) + .await?; + stream.write_all(payload).await?; + Ok(()) +} + async fn read_status(stream: &mut TcpStream) -> Result<()> { let mut status = [0; 4]; stream @@ -299,6 +367,9 @@ const MAX_OUTPUT_LENGTH: usize = 64 * 1024 * 1024; const IO_BUFFER_LENGTH: usize = 8192; +/// Maximum payload per sync-protocol DATA packet (adb's SYNC_DATA_MAX). +const SYNC_DATA_CHUNK_LENGTH: usize = 64 * 1024; + #[cfg(test)] mod tests { use super::*; diff --git a/packages/accessibility-core/Cargo.toml b/packages/accessibility-core/Cargo.toml index 5773af1..2f6a012 100644 --- a/packages/accessibility-core/Cargo.toml +++ b/packages/accessibility-core/Cargo.toml @@ -28,12 +28,12 @@ serde.workspace = true serde_json.workspace = true slotmap.workspace = true tokio.workspace = true +tracing.workspace = true viuer.workspace = true [target.'cfg(target_os = "macos")'.dependencies] accessibility-ios-sys.workspace = true accessibility-macos-sys.workspace = true -tracing.workspace = true [target.'cfg(target_os = "windows")'.dependencies] accessibility-windows-sys.workspace = true diff --git a/packages/accessibility-core/src/accessibility/query.rs b/packages/accessibility-core/src/accessibility/query.rs index 3bb5398..4494a5a 100644 --- a/packages/accessibility-core/src/accessibility/query.rs +++ b/packages/accessibility-core/src/accessibility/query.rs @@ -744,9 +744,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(0.0, 0.0), Size::new(800.0, 600.0))), enabled: true, focused: false, + checked: None, actions: vec![], children: vec![ Element { @@ -759,9 +761,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(10.0, 10.0), Size::new(80.0, 30.0))), enabled: true, focused: true, + checked: None, actions: vec!["Click".to_string()], children: vec![], }, @@ -775,9 +779,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(10.0, 50.0), Size::new(200.0, 25.0))), enabled: false, focused: false, + checked: None, actions: vec![], children: vec![], }, @@ -791,9 +797,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(100.0, 10.0), Size::new(80.0, 30.0))), enabled: true, focused: false, + checked: None, actions: vec!["Click".to_string()], children: vec![], }, @@ -807,9 +815,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(10.0, 90.0), Size::new(200.0, 20.0))), enabled: true, focused: false, + checked: None, actions: vec![], children: vec![], }, @@ -823,9 +833,11 @@ mod tests { help: None, role_description: None, identifier: None, + native_id: None, bounds: Some(Rect::new(Point::new(10.0, 120.0), Size::new(200.0, 20.0))), enabled: true, focused: false, + checked: None, actions: vec![], children: vec![], }, diff --git a/packages/accessibility-core/src/accessibility/types.rs b/packages/accessibility-core/src/accessibility/types.rs index 7a4122f..708e5b6 100644 --- a/packages/accessibility-core/src/accessibility/types.rs +++ b/packages/accessibility-core/src/accessibility/types.rs @@ -158,6 +158,14 @@ pub struct Element { #[serde(skip_serializing_if = "Option::is_none")] pub identifier: Option, + /// Platform-native stable node identity, when the platform exposes one + /// (Android `AccessibilityNodeInfo.getSourceNodeId()`; Linux a hash of + /// the AT-SPI D-Bus bus name and object path): stable for the + /// same underlying view across dumps within a window, so consumers can + /// distinguish "same control, new state" from "replaced control". + #[serde(skip_serializing_if = "Option::is_none")] + pub native_id: Option, + /// Screen bounds of the element. #[serde(skip_serializing_if = "Option::is_none")] pub bounds: Option, @@ -168,6 +176,11 @@ pub struct Element { /// Whether the element currently has keyboard focus. pub focused: bool, + /// Checked/toggled state for checkable elements (checkboxes, toggles, + /// radio buttons). `None` when the element is not checkable. + #[serde(skip_serializing_if = "Option::is_none")] + pub checked: Option, + /// Available actions on this element. #[serde(skip_serializing_if = "Vec::is_empty")] pub actions: Vec, @@ -190,9 +203,11 @@ impl Element { help: None, role_description: None, identifier: None, + native_id: None, bounds: None, enabled: true, focused: false, + checked: None, actions: Vec::new(), children: Vec::new(), } diff --git a/packages/accessibility-core/src/platform/android.rs b/packages/accessibility-core/src/platform/android.rs index 1474af9..293a396 100644 --- a/packages/accessibility-core/src/platform/android.rs +++ b/packages/accessibility-core/src/platform/android.rs @@ -62,10 +62,110 @@ pub mod ax; pub mod input; pub mod session; pub mod video; +use accessibility_android_sys::UiDumpServer; pub use accessibility_android_sys::{AdbClient, AndroidKeyCode}; pub use input::{HardwareButton, InputCommand, Orientation, TouchPhase, spawn_input_worker}; pub use video::AndroidVideoCapture; +/// How long to keep using classic `uiautomator dump` after the persistent +/// dump server fails before trying to start it again. +const DUMP_SERVER_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(30); + +/// A process-wide, per-device hybrid UI dump source. +/// +/// The persistent AxDump server holds the device's only `UiAutomation` +/// registration, and while it runs the system kills competing clients — +/// including a plain `uiautomator dump` issued by another +/// [`AndroidAccessibility`] in the same process. Sharing one source per +/// serial keeps every reader on the fast path and avoids that conflict. +/// +/// Steady-state server dumps take a few milliseconds versus ~2s for +/// `uiautomator dump`; any server failure falls back to the classic dump +/// for [`DUMP_SERVER_RETRY_DELAY`] before the server is retried. +#[derive(Clone)] +pub struct SharedDumpSource { + state: std::sync::Arc>, +} + +#[derive(Default)] +struct DumpSourceState { + server: Option, + retry_at: Option, +} + +impl SharedDumpSource { + /// The shared source for `serial` (or the default device when `None`). + pub fn for_device(serial: Option<&str>) -> Self { + static SOURCES: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + let key = serial.unwrap_or("").to_string(); + SOURCES + .get_or_init(Default::default) + .lock() + .expect("dump source registry poisoned") + .entry(key) + .or_insert_with(|| SharedDumpSource { + state: Default::default(), + }) + .clone() + } + + /// Dump the UI hierarchy, preferring the persistent server. + pub async fn dump_ui(&self, adb: &AdbClient) -> Result { + if dump_server_disabled() { + return adb.dump_ui().await; + } + let mut state = self.state.lock().await; + if state.server.is_none() + && state + .retry_at + .is_none_or(|at| std::time::Instant::now() >= at) + { + match Self::start_server(adb).await { + Ok(server) => state.server = Some(server), + Err(error) => { + tracing::debug!(?error, "AxDump server unavailable; using classic dumps"); + state.retry_at = Some(std::time::Instant::now() + DUMP_SERVER_RETRY_DELAY); + } + } + } + if let Some(server) = state.server.as_mut() { + // One retry covers the transient `no active window root` race + // during window transitions. + for attempt in 0..2 { + match server.dump().await { + Ok(xml) => return Ok(xml), + Err(error) + if attempt == 0 && error.to_string().contains("no active window") => + { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + Err(error) => { + tracing::debug!(?error, "AxDump server dump failed; falling back"); + break; + } + } + } + state.server = None; + state.retry_at = Some(std::time::Instant::now() + DUMP_SERVER_RETRY_DELAY); + } + drop(state); + adb.dump_ui().await + } + + async fn start_server(adb: &AdbClient) -> Result { + adb.ensure_axdump().await?; + adb.start_dump_server(accessibility_android_sys::AXDUMP_DEX_DEVICE_PATH) + .await + } +} + +fn dump_server_disabled() -> bool { + std::env::var("AX_ANDROID_DUMP_SERVER") + .is_ok_and(|value| value == "0" || value.eq_ignore_ascii_case("off")) +} + /// Parse Android bounds string like "[0,0][1080,1920]" into a Rect. fn parse_bounds(bounds_str: &str) -> Option { // Format: "[left,top][right,bottom]" @@ -251,6 +351,10 @@ struct UiNode { long_clickable: bool, /// Package name. package: Option, + /// Stable native node identity (AxDump's `ax-node-id`, from the hidden + /// `AccessibilityNodeInfo.getSourceNodeId()`); absent in classic + /// `uiautomator dump` output. + node_id: Option, /// Child nodes. children: Vec, } @@ -272,6 +376,7 @@ impl UiNode { scrollable: false, long_clickable: false, package: None, + node_id: None, children: Vec::new(), } } @@ -294,6 +399,7 @@ impl UiNode { .or_else(|| self.content_desc.clone().filter(|s| !s.is_empty())); elem.description = self.content_desc.clone().filter(|s| !s.is_empty()); elem.identifier = self.resource_id.clone(); + elem.native_id = self.node_id; elem.bounds = bounds; elem.enabled = self.enabled; elem.focused = self.focused; @@ -379,6 +485,7 @@ fn parse_ui_xml(xml: &str) -> Result { "scrollable" => node.scrollable = value == "true", "long-clickable" => node.long_clickable = value == "true", "package" => node.package = Some(value), + "ax-node-id" => node.node_id = value.parse().ok(), _ => {} } } @@ -419,6 +526,7 @@ fn parse_ui_xml(xml: &str) -> Result { "scrollable" => node.scrollable = value == "true", "long-clickable" => node.long_clickable = value == "true", "package" => node.package = Some(value), + "ax-node-id" => node.node_id = value.parse().ok(), _ => {} } } @@ -484,6 +592,8 @@ pub struct AndroidAccessibility { element_bounds: SecondaryMap, /// Cached screen size. screen_size: Option<(u32, u32)>, + /// Hybrid dump source shared across readers of the same device. + dump_source: SharedDumpSource, /// Last known app package (for PID-like targeting). last_package: Option, } @@ -514,6 +624,7 @@ impl AndroidAccessibility { let screen_size = adb.get_screen_size().await.ok(); Ok(Self { + dump_source: SharedDumpSource::for_device(serial), adb, cache: ElementCache::new(), element_bounds: SecondaryMap::new(), @@ -530,6 +641,7 @@ impl AndroidAccessibility { let screen_size = adb.get_screen_size().await.ok(); Ok(Self { + dump_source: SharedDumpSource::for_device(serial), adb, cache: ElementCache::new(), element_bounds: SecondaryMap::new(), @@ -575,7 +687,7 @@ impl AccessibilityReader for AndroidAccessibility { self.element_bounds.clear(); // Dump UI hierarchy - let xml = self.adb.dump_ui().await?; + let xml = self.dump_source.dump_ui(&self.adb).await?; // Parse XML into node tree let root_node = parse_ui_xml(&xml)?; diff --git a/packages/accessibility-core/src/platform/android/ax.rs b/packages/accessibility-core/src/platform/android/ax.rs index 347d1ec..073d7ef 100644 --- a/packages/accessibility-core/src/platform/android/ax.rs +++ b/packages/accessibility-core/src/platform/android/ax.rs @@ -3,7 +3,7 @@ use serde::Serialize; use tokio::sync::{mpsc, oneshot}; use crate::accessibility::{ - AccessibilityReader, AndroidTarget, Element, ElementTree, Point, Rect, Size, Target, TreeFilter, + AccessibilityReader, AndroidTarget, Element, Point, Rect, Size, Target, TreeFilter, }; use super::AndroidAccessibility; @@ -52,6 +52,9 @@ pub struct ElementDetail { pub label: Option, pub value: Option, pub identifier: Option, + /// Stable per-window native node identity (AxDump's `ax-node-id`); + /// absent on the classic `uiautomator dump` path. + pub native_id: Option, pub enabled: bool, pub focused: bool, pub actions: Vec, @@ -116,8 +119,18 @@ async fn snapshot( _scan: bool, ) -> Result<(AxSnapshot, Rect)> { let tree = reader.get_tree(target, &TreeFilter::default()).await?; - let screen = - screen_bounds(&tree).ok_or_else(|| anyhow!("Android tree has no screen bounds"))?; + // Normalise against the physical display, not the tree's max extent: when a + // dialog is up the active window's tree is the dialog, so tree-derived + // normalisation scales normalized (x, y) by the dialog size and taps land + // outside it. + let (width, height) = match reader.screen_size() { + Some(size) => size, + None => reader.refresh_screen_size().await?, + }; + let screen = Rect::new( + Point::new(0.0, 0.0), + Size::new(f64::from(width), f64::from(height)), + ); let mut elements = Vec::with_capacity(tree.element_count); flatten(&tree.root, &screen, 0, &mut elements); elements.retain(|element| { @@ -158,20 +171,6 @@ async fn hit_test( .map(|element| to_detail(element, screen, 0))) } -fn screen_bounds(tree: &ElementTree) -> Option { - let mut max_x = 0.0f64; - let mut max_y = 0.0f64; - let mut stack = vec![&tree.root]; - while let Some(element) = stack.pop() { - if let Some(bounds) = &element.bounds { - max_x = max_x.max(bounds.origin.x + bounds.size.width); - max_y = max_y.max(bounds.origin.y + bounds.size.height); - } - stack.extend(element.children.iter()); - } - (max_x > 0.0 && max_y > 0.0).then(|| Rect::new(Point::new(0.0, 0.0), Size::new(max_x, max_y))) -} - fn flatten(element: &Element, screen: &Rect, depth: u32, out: &mut Vec) { out.push(to_detail(element, screen, depth)); for child in &element.children { @@ -187,6 +186,7 @@ fn to_detail(element: &Element, screen: &Rect, depth: u32) -> ElementDetail { label: element.title.clone().filter(|value| !value.is_empty()), value: element.value.clone().filter(|value| !value.is_empty()), identifier: element.identifier.clone().filter(|value| !value.is_empty()), + native_id: element.native_id, enabled: element.enabled, focused: element.focused, actions: element.actions.clone(), diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index eabd732..25f5b6e 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -249,6 +249,8 @@ impl IOSSimulatorAccessibility { enabled: sys_element.enabled, focused: sys_element.focused, actions: sys_element.actions.clone(), + checked: None, + native_id: None, children, }); diff --git a/packages/accessibility-core/src/platform/msft.rs b/packages/accessibility-core/src/platform/msft.rs index 342de23..37607c4 100644 --- a/packages/accessibility-core/src/platform/msft.rs +++ b/packages/accessibility-core/src/platform/msft.rs @@ -134,6 +134,8 @@ impl WindowsAccessibility { enabled: sys_element.enabled, focused: sys_element.focused, actions: sys_element.actions.clone(), + checked: None, + native_id: None, children, }); @@ -394,6 +396,8 @@ fn from_sys_element_standalone(element: sys::Element) -> Element { enabled: element.enabled, focused: element.focused, actions: element.actions, + checked: None, + native_id: None, children: element .children .into_iter() diff --git a/packages/accessibility-core/src/platform/x11.rs b/packages/accessibility-core/src/platform/x11.rs index 8a38812..9fe85d5 100644 --- a/packages/accessibility-core/src/platform/x11.rs +++ b/packages/accessibility-core/src/platform/x11.rs @@ -10,6 +10,7 @@ use crate::accessibility::{ }; use accessibility_linux_sys::atspi::proxy::accessible::AccessibleProxy; use accessibility_linux_sys::atspi::proxy::action::ActionProxy; +use accessibility_linux_sys::atspi::proxy::cache::CacheProxy; use accessibility_linux_sys::atspi::proxy::component::ComponentProxy; use accessibility_linux_sys::atspi::proxy::editable_text::EditableTextProxy; use accessibility_linux_sys::atspi::proxy::text::TextProxy; @@ -25,8 +26,56 @@ use accesskit::{Action, Role}; use anyhow::{Result, anyhow, bail}; use slotmap::SecondaryMap; use std::collections::HashMap; +use std::collections::HashSet; use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock, RwLock}; + +/// WebKit web-process connections (plug graft targets), which number their +/// AT-SPI roles differently from GTK apps (see [`LinuxAccessibility::fetch_role`]), +/// keyed by web-process bus name with the UI-process bus its plug grafts into +/// as the value (so a stalled web process can be attributed to its embedder). +fn webkit_buses() -> &'static RwLock> { + static BUSES: OnceLock>> = OnceLock::new(); + BUSES.get_or_init(|| RwLock::new(HashMap::new())) +} + +/// Which applications' web content is unreachable because their WebKit web +/// process stopped answering (typically a script dialog blocking it). +#[derive(Default)] +struct BlockedGrafts { + /// UI-process bus names whose grafted web subtree is stalled; a walk that + /// never touches these buses is unaffected. + ui_buses: HashSet, + /// A stalled web process that could not be attributed to any UI-process + /// bus — every walk must be treated as potentially incomplete. + unattributed: bool, +} + +/// The process name (`/proc//comm`, kernel-truncated to 15 bytes). +fn process_comm(pid: u32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/comm")) + .ok() + .map(|comm| comm.trim().to_string()) +} + +/// The parent PID from `/proc//stat` (field 4, after the parenthesised +/// comm, which may itself contain spaces or parentheses). +fn process_parent_pid(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let after_comm = &stat[stat.rfind(')')? + 1..]; + after_comm.split_whitespace().nth(1)?.parse().ok() +} + +/// Whether a D-Bus error means the peer never answered (reply timeout), as +/// opposed to answering with an error. Applications blocked mid-walk (e.g. a +/// WebKit web process stalled on a script dialog) fail this way. +fn is_no_reply_error(error: &impl std::fmt::Display) -> bool { + let text = error.to_string(); + text.contains("Did not receive a reply") + || text.contains("NoReply") + || text.contains("timed out") + || text.contains("Timeout") +} /// Macro to generate D-Bus proxy factory functions with consistent error handling. macro_rules! create_proxy_fn { @@ -66,6 +115,12 @@ pub struct LinuxAccessibility { /// AT-SPI connection to the accessibility bus. connection: AccessibilityConnection, + + /// Whether the most recent tree walk hit a D-Bus no-reply timeout on a + /// child fetch — the application (typically a WebKit web process blocked + /// by a script dialog) stopped answering mid-walk, so the returned tree + /// is silently missing subtrees. + walk_blocked: bool, } impl LinuxAccessibility { @@ -85,9 +140,37 @@ impl LinuxAccessibility { cache: ElementCache::new(), handles: SecondaryMap::new(), connection, + walk_blocked: false, }) } + /// Create a reader connected to an explicit accessibility bus address + /// (e.g. the address published in the X root window's `AT_SPI_BUS` + /// property), for environments where the session bus's `org.a11y.Bus` + /// answer points at a different bus than the one applications register on. + pub async fn with_bus_address(address: &str) -> Result { + let address = address + .parse() + .map_err(|e| anyhow!("Invalid AT-SPI bus address {address:?}: {e}"))?; + let connection = AccessibilityConnection::from_address(address) + .await + .map_err(|e| anyhow!("Failed to connect to AT-SPI bus at explicit address: {e}"))?; + + Ok(Self { + cache: ElementCache::new(), + handles: SecondaryMap::new(), + connection, + walk_blocked: false, + }) + } + + /// Whether the most recent tree walk was silently cut short by an + /// application that stopped answering (D-Bus no-reply timeout on a child + /// fetch). Callers should treat such a tree as incomplete. + pub fn last_walk_blocked(&self) -> bool { + self.walk_blocked + } + /// Get the PID of a D-Bus bus name owner. async fn get_pid_for_bus_name(conn: &zbus::Connection, bus_name: &str) -> Option { use accessibility_linux_sys::zbus::names::BusName; @@ -130,11 +213,177 @@ impl LinuxAccessibility { .map_err(|e| anyhow!("Failed to create TextProxy: {}", e)) } + /// Discover plug roots that are grafted into other applications' trees. + /// + /// Out-of-process web content (WebKitGTK web processes, used by Epiphany, + /// Tauri on Linux, and other WebKitGTK embedders) registers on the + /// accessibility bus as separate connections. The UI-process socket node + /// reports zero children over `GetChildren`/`GetChildAtIndex`, so a plain + /// walk never crosses into web content. Each plug connection exposes an + /// `org.a11y.atspi.Cache` whose root item's parent points at the + /// UI-process socket node; this returns that socket -> plug-root mapping + /// so the tree walk can graft the web content subtree in place. + /// + /// Also reports whether a bus previously seen serving WebKit plug objects + /// failed to answer: its web process has stopped responding (typically a + /// script dialog blocking it), so the walk would silently drop the web + /// content subtree and the returned tree must be treated as blocked. + async fn discover_plug_grafts( + conn: &zbus::Connection, + ) -> (HashMap<(String, String), NativeHandle>, BlockedGrafts) { + let mut grafts = HashMap::new(); + let mut blocked = BlockedGrafts::default(); + let mut stalled_unknown: Vec = Vec::new(); + let Ok(dbus_proxy) = DBusProxy::new(conn).await else { + return (grafts, blocked); + }; + let Ok(names) = dbus_proxy.list_names().await else { + return (grafts, blocked); + }; + for name in &names { + let name = name.as_str(); + if !name.starts_with(':') { + continue; + } + let Ok(cache_proxy) = CacheProxy::builder(conn) + .destination(name.to_string()) + .and_then(|b| b.path("/org/a11y/atspi/cache")) + .map(|b| b.cache_properties(CacheProperties::No)) + .map(|b| b.build()) + else { + continue; + }; + let Ok(cache_proxy) = cache_proxy.await else { + continue; + }; + // Bound each query: connections that do not serve an AT-SPI cache + // (ATs, monitors) may never reply. + let items = match tokio::time::timeout( + std::time::Duration::from_millis(500), + cache_proxy.get_items(), + ) + .await + { + Ok(Ok(items)) => items, + _ => { + // Connections that never serve a cache (ATs, monitors) + // time out here routinely; that is the intended bound. + // But a bus already known to host WebKit plug objects + // going quiet means its web process is stalled and its + // graft (the whole web subtree) is about to be missing + // from that embedder's tree specifically. + let embedder = webkit_buses() + .read() + .ok() + .and_then(|buses| buses.get(name).cloned()); + if let Some(embedder) = embedder { + blocked.ui_buses.insert(embedder); + } else { + stalled_unknown.push(name.to_string()); + } + continue; + } + }; + for item in items { + let Some(parent_name) = item.parent.name_as_str() else { + continue; + }; + // A cross-connection parent marks a plug root embedded in + // another process's tree (e.g. a WebKit web process plugged + // into its UI process). + if parent_name.is_empty() || parent_name == name { + continue; + } + // Every application root's cache parent is the registry's + // root object; those are ordinary registrations, not plugs. + if item.parent.path_as_str() == "/org/a11y/atspi/accessible/root" { + continue; + } + let Some(object_name) = item.object.name_as_str() else { + continue; + }; + // Only WebKit web processes number roles differently; their + // plug objects live under /org/a11y/webkit/. + if item.object.path_as_str().starts_with("/org/a11y/webkit/") + && let Ok(mut buses) = webkit_buses().write() + { + buses.insert(object_name.to_string(), parent_name.to_string()); + } + grafts.insert( + ( + parent_name.to_string(), + item.parent.path_as_str().to_string(), + ), + NativeHandle { + bus_name: object_name.to_string(), + object_path: item.object.path_as_str().to_string(), + }, + ); + } + } + // Cold start: a WebKit web process already stalled before this + // process ever saw its plug objects is not in `webkit_buses()`, so + // the timeout above cannot classify it. The bus daemon still answers + // for the stalled connection, so identify it by its process name and + // attribute it to its embedder (its parent process) by matching the + // parent PID against the other connections' PIDs. + for name in stalled_unknown { + let Some(pid) = Self::get_pid_for_bus_name(conn, &name).await else { + continue; + }; + if !process_comm(pid).is_some_and(|comm| comm.starts_with("WebKitWebProc")) { + continue; + } + let embedder_pid = process_parent_pid(pid); + let mut attributed = false; + if let Some(embedder_pid) = embedder_pid { + for candidate in &names { + let candidate = candidate.as_str(); + if candidate == name || !candidate.starts_with(':') { + continue; + } + if Self::get_pid_for_bus_name(conn, candidate).await == Some(embedder_pid) { + blocked.ui_buses.insert(candidate.to_string()); + attributed = true; + } + } + } + if !attributed { + blocked.unattributed = true; + } + } + (grafts, blocked) + } + create_proxy_fn!(create_component_proxy, ComponentProxy); create_proxy_fn!(create_action_proxy, ActionProxy); create_proxy_fn!(create_editable_text_proxy, EditableTextProxy); create_proxy_fn!(create_value_proxy, ValueProxy); + /// Fetch an accessible's role, correcting for WebKitGTK's role numbering. + /// + /// WebKitGTK >= 2.50 numbers its AT-SPI roles per current at-spi2-core, + /// where the deprecated `Footer` slot (72) is gone and `SectionFooter`/ + /// `SectionHeader` sit at the end, so every wire value >= 72 is one less + /// than the numbering the `atspi` crate decodes. WebKit objects live on + /// their web process's own bus connection, recorded at plug discovery. + async fn fetch_role(proxy: &AccessibleProxy<'_>) -> Option { + let raw: u32 = proxy.inner().call("GetRole", &()).await.ok()?; + let destination = proxy.inner().destination().as_str().to_string(); + let webkit = webkit_buses() + .read() + .map(|buses| buses.contains_key(&destination)) + .unwrap_or(false); + let adjusted = match raw { + _ if !webkit => raw, + 0..=71 => raw, + 115 => 72, // SectionFooter -> Footer + 116 => 71, // SectionHeader -> Header + n => n + 1, + }; + AtspiRole::try_from(adjusted).ok() + } + /// Map AT-SPI Role to accesskit Role. fn map_role(atspi_role: AtspiRole) -> Role { match atspi_role { @@ -145,7 +394,8 @@ impl LinuxAccessibility { AtspiRole::Text | AtspiRole::Terminal => Role::MultilineTextInput, AtspiRole::Label | AtspiRole::Static => Role::Label, AtspiRole::ComboBox => Role::ComboBox, - AtspiRole::Slider | AtspiRole::SpinButton => Role::Slider, + AtspiRole::Slider => Role::Slider, + AtspiRole::SpinButton => Role::SpinButton, AtspiRole::Menu | AtspiRole::PopupMenu => Role::Menu, AtspiRole::MenuItem => Role::MenuItem, AtspiRole::CheckMenuItem => Role::MenuItemCheckBox, @@ -161,7 +411,7 @@ impl LinuxAccessibility { AtspiRole::MenuBar => Role::MenuBar, AtspiRole::ScrollBar => Role::ScrollBar, AtspiRole::ScrollPane => Role::ScrollView, - AtspiRole::StatusBar => Role::Tooltip, + AtspiRole::StatusBar => Role::Status, AtspiRole::Panel | AtspiRole::Filler => Role::Group, AtspiRole::Application => Role::Application, AtspiRole::DocumentFrame | AtspiRole::DocumentWeb => Role::WebView, @@ -180,6 +430,36 @@ impl LinuxAccessibility { AtspiRole::ColumnHeader | AtspiRole::TableColumnHeader => Role::ColumnHeader, AtspiRole::RowHeader | AtspiRole::TableRowHeader => Role::RowHeader, AtspiRole::DesktopFrame | AtspiRole::DesktopIcon => Role::Unknown, + // High-numbered roles emitted by WebKitGTK for ordinary web content + AtspiRole::Section | AtspiRole::Grouping => Role::Group, + AtspiRole::Header => Role::Header, + AtspiRole::Footer => Role::Footer, + AtspiRole::Caption => Role::Caption, + AtspiRole::BlockQuote => Role::Blockquote, + AtspiRole::Article => Role::Article, + AtspiRole::Landmark => Role::Region, + AtspiRole::Comment => Role::Comment, + AtspiRole::Notification | AtspiRole::InfoBar => Role::Status, + AtspiRole::LevelBar => Role::Meter, + AtspiRole::TitleBar => Role::TitleBar, + AtspiRole::Audio => Role::Audio, + AtspiRole::Video => Role::Video, + AtspiRole::Definition => Role::Definition, + AtspiRole::Log => Role::Log, + AtspiRole::Marquee => Role::Marquee, + AtspiRole::Math => Role::Math, + AtspiRole::Timer => Role::Timer, + AtspiRole::DescriptionList => Role::DescriptionList, + AtspiRole::DescriptionTerm => Role::Term, + AtspiRole::DescriptionValue => Role::Definition, + AtspiRole::Footnote => Role::DocFootnote, + AtspiRole::ContentDeletion => Role::ContentDeletion, + AtspiRole::ContentInsertion => Role::ContentInsertion, + AtspiRole::Mark => Role::Mark, + AtspiRole::Suggestion => Role::Suggestion, + AtspiRole::Autocomplete => Role::ComboBox, + AtspiRole::Editbar => Role::TextInput, + AtspiRole::Embedded => Role::EmbeddedObject, _ => Role::Unknown, } } @@ -193,11 +473,12 @@ impl LinuxAccessibility { id: ElementKey, ) -> Option { // Get role - let atspi_role = proxy.get_role().await.ok()?; + let atspi_role = Self::fetch_role(proxy).await?; let role = Self::map_role(atspi_role); // Build element let mut element = Element::new(id, role); + element.native_id = Some(native_identity(&handle.bus_name, &handle.object_path)); // Get basic properties element.title = proxy.name().await.ok().filter(|s| !s.is_empty()); @@ -211,6 +492,9 @@ impl LinuxAccessibility { element.enabled = !states.contains(atspi::State::Sensitive) || states.contains(atspi::State::Enabled); element.focused = states.contains(atspi::State::Focused); + if is_checkable(role, &states) { + element.checked = Some(states.contains(atspi::State::Checked)); + } } // Get bounds from Component interface if available @@ -232,15 +516,8 @@ impl LinuxAccessibility { if interfaces.contains(atspi::Interface::Action) && let Ok(action_proxy) = Self::create_action_proxy(conn, &handle.bus_name, &handle.object_path).await - && let Ok(n_actions) = action_proxy.nactions().await + && let Ok(actions) = Self::list_action_names(&action_proxy).await { - // Use nactions + get_name instead of get_actions for compatibility - let mut actions = Vec::new(); - for i in 0..n_actions { - if let Ok(name) = action_proxy.get_name(i).await { - actions.push(name); - } - } element.actions = actions; } @@ -270,6 +547,35 @@ impl LinuxAccessibility { Some(element) } + /// List an element's action names via NActions + GetName. + /// + /// GetActions crashes on some older GTK applications and never answers on + /// WebKitGTK, so it is avoided entirely. The spec property is `NActions`, + /// but atspi-proxies' generated getter asks for `Nactions`; some + /// implementations (WebKitGTK) only answer the correctly-cased name, so + /// try both. + async fn list_action_names(action_proxy: &ActionProxy<'_>) -> Result> { + let n_actions = match action_proxy.inner().get_property::("NActions").await { + Ok(n) => n, + Err(_) => action_proxy + .nactions() + .await + .map_err(|e| anyhow!("Failed to get action count: {}", e))?, + }; + let mut actions = Vec::new(); + for i in 0..n_actions { + // WebKitGTK advertises a nameless action on plain text nodes + // (paragraphs, headings); an empty name carries no information + // and reads as actionable, so it is dropped. + if let Ok(name) = action_proxy.get_name(i).await + && !name.is_empty() + { + actions.push(name); + } + } + Ok(actions) + } + /// Build the accessibility tree iteratively using a stack. async fn build_tree_async( &mut self, @@ -291,6 +597,10 @@ impl LinuxAccessibility { // Clone the connection for use in async block let conn = self.connection.connection().clone(); + // Socket nodes report zero children; plug roots discovered here are + // grafted in as their children so web content is reachable. + let (plug_grafts, grafts_blocked) = Self::discover_plug_grafts(&conn).await; + // Collect results and handles in async block using temporary IDs let async_result: Option<_> = async { // Use temporary indices that will be remapped to real slotmap IDs later @@ -299,6 +609,7 @@ impl LinuxAccessibility { let mut element_count = 0usize; let mut root_temp_id: Option = None; let mut next_temp_id: TempId = 1; + let mut blocked = false; // Initialize stack with root let root_proxy = Self::create_accessible_proxy( @@ -364,7 +675,38 @@ impl LinuxAccessibility { // Get children if we should recurse let should_recurse = filter.max_depth.is_none_or(|max| entry.depth < max); - if should_recurse && let Ok(children) = proxy.get_children().await { + if should_recurse + && let Some(plug_root) = plug_grafts.get(&( + entry.handle.bus_name.clone(), + entry.handle.object_path.clone(), + )) + && let Ok(plug_proxy) = Self::create_accessible_proxy( + &conn, + &plug_root.bus_name, + &plug_root.object_path, + ) + .await + && let Ok(plug_interfaces) = plug_proxy.get_interfaces().await + { + stack.push(StackEntry { + handle: plug_root.clone(), + interfaces: plug_interfaces, + parent_temp_id: Some(temp_id), + depth: entry.depth + 1, + }); + } + let children = if should_recurse { + match proxy.get_children().await { + Ok(children) => Some(children), + Err(error) => { + blocked |= is_no_reply_error(&error); + None + } + } + } else { + None + }; + if let Some(children) = children { // Push children to stack in reverse order so first child is processed first for child_ref in children.into_iter().rev() { let child_handle = NativeHandle { @@ -395,11 +737,18 @@ impl LinuxAccessibility { element_count += 1; } - Some((results, handles_to_insert, root_temp_id)) + Some((results, handles_to_insert, root_temp_id, blocked)) } .await; - let (mut results, handles_to_insert, root_temp_id) = async_result?; + let (mut results, handles_to_insert, root_temp_id, blocked) = async_result?; + // A stalled web process only blocks walks that actually cross the + // embedder it plugs into; other applications' trees stay trustworthy. + let grafts_blocked = grafts_blocked.unattributed + || handles_to_insert + .iter() + .any(|(_, handle)| grafts_blocked.ui_buses.contains(&handle.bus_name)); + self.walk_blocked = blocked || grafts_blocked; let root_temp_id = root_temp_id?; // Build temp_id -> handle mapping for later use @@ -412,6 +761,12 @@ impl LinuxAccessibility { children_map.entry(*pid).or_default().push(temp_id); } } + // Temp ids are assigned in depth-first document order, but the map + // above is iterated in hash order — sort so siblings keep the order + // the application reports them in. + for siblings in children_map.values_mut() { + siblings.sort_unstable(); + } // Build tree recursively, storing elements and handles as we go fn build_tree_from_results( @@ -460,9 +815,11 @@ impl LinuxAccessibility { help: element.help, role_description: element.role_description, identifier: element.identifier, + native_id: element.native_id, bounds: element.bounds, enabled: element.enabled, focused: element.focused, + checked: element.checked, actions: element.actions, children: children_elements, }); @@ -514,26 +871,41 @@ impl LinuxAccessibility { None } - /// Find the focused application. + /// Find the focused application. The ACTIVE state lives on an + /// application's window children, not on the application node itself, so + /// check each application's top-level windows; only fall back to + /// application-level Active/Focused states. There is deliberately no + /// "first registered application" fallback: background daemons (e.g. + /// desktop portals) register first and would win. async fn find_focused_app( conn: &zbus::Connection, root: &AccessibleProxy<'_>, ) -> Option<(NativeHandle, u32)> { let children = root.get_children().await.ok()?; - // First pass: look for focused/active application for child_ref in &children { let handle = NativeHandle { bus_name: child_ref.name_as_str().unwrap_or_default().to_string(), object_path: child_ref.path_as_str().to_string(), }; - - if let Ok(proxy) = + let Ok(proxy) = Self::create_accessible_proxy(conn, &handle.bus_name, &handle.object_path).await - && let Ok(states) = proxy.get_state().await - { - // Check for Active or Focused state - if states.contains(atspi::State::Active) || states.contains(atspi::State::Focused) { + else { + continue; + }; + let Ok(windows) = proxy.get_children().await else { + continue; + }; + for window_ref in &windows { + if let Ok(window) = Self::create_accessible_proxy( + conn, + window_ref.name_as_str().unwrap_or_default(), + window_ref.path_as_str(), + ) + .await + && let Ok(states) = window.get_state().await + && states.contains(atspi::State::Active) + { let pid = Self::get_pid_for_bus_name(conn, &handle.bus_name) .await .unwrap_or(0); @@ -542,19 +914,20 @@ impl LinuxAccessibility { } } - // Fallback: return first application with a valid PID for child_ref in &children { - let bus_name = child_ref.name_as_str().unwrap_or_default().to_string(); - if let Some(pid) = Self::get_pid_for_bus_name(conn, &bus_name).await - && pid > 0 + let handle = NativeHandle { + bus_name: child_ref.name_as_str().unwrap_or_default().to_string(), + object_path: child_ref.path_as_str().to_string(), + }; + if let Ok(proxy) = + Self::create_accessible_proxy(conn, &handle.bus_name, &handle.object_path).await + && let Ok(states) = proxy.get_state().await + && (states.contains(atspi::State::Active) || states.contains(atspi::State::Focused)) { - return Some(( - NativeHandle { - bus_name, - object_path: child_ref.path_as_str().to_string(), - }, - pid, - )); + let pid = Self::get_pid_for_bus_name(conn, &handle.bus_name) + .await + .unwrap_or(0); + return Some((handle, pid)); } } @@ -733,6 +1106,83 @@ impl LinuxAccessibility { Self::find_window_by_pid_recursive(&conn, root, pid_atom, pid) } + /// Raise and activate the toplevel window belonging to a PID. + /// + /// Sends an EWMH `_NET_ACTIVE_WINDOW` request for the first managed + /// toplevel whose `_NET_WM_PID` matches, so synthesized pointer events + /// aimed at that application are not swallowed by a covering window. + pub fn activate_window_for_pid(pid: u32) -> Result<()> { + use accessibility_linux_sys::x11rb::connection::Connection; + use accessibility_linux_sys::x11rb::protocol::xproto::{ + ClientMessageEvent, ConnectionExt as _, EventMask, + }; + + let (conn, screen_num) = + x11rb::connect(None).map_err(|e| anyhow!("Failed to connect to X11: {}", e))?; + let root = conn.setup().roots[screen_num].root; + let atom = |name: &[u8]| -> Result { + Ok(conn + .intern_atom(false, name) + .map_err(|e| anyhow!("intern_atom: {}", e))? + .reply() + .map_err(|e| anyhow!("intern_atom reply: {}", e))? + .atom) + }; + let client_list = atom(b"_NET_CLIENT_LIST")?; + let pid_atom = atom(b"_NET_WM_PID")?; + let active_window = atom(b"_NET_ACTIVE_WINDOW")?; + + let list = conn + .get_property( + false, + root, + client_list, + x11rb::protocol::xproto::AtomEnum::WINDOW, + 0, + u32::MAX, + ) + .map_err(|e| anyhow!("get _NET_CLIENT_LIST: {}", e))? + .reply() + .map_err(|e| anyhow!("_NET_CLIENT_LIST reply: {}", e))?; + let windows: Vec = list.value32().map(|v| v.collect()).unwrap_or_default(); + for window in windows { + let Ok(cookie) = conn.get_property( + false, + window, + pid_atom, + x11rb::protocol::xproto::AtomEnum::CARDINAL, + 0, + 1, + ) else { + continue; + }; + let Ok(reply) = cookie.reply() else { continue }; + let window_pid = reply.value32().and_then(|mut v| v.next()); + if window_pid != Some(pid) { + continue; + } + let event = ClientMessageEvent::new( + 32, + window, + active_window, + // source indication 2 = pager/direct user action, so the + // window manager honors the activation instead of setting + // the demands-attention hint. + [2, x11rb::CURRENT_TIME, 0, 0, 0], + ); + conn.send_event( + false, + root, + EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY, + event, + ) + .map_err(|e| anyhow!("send _NET_ACTIVE_WINDOW: {}", e))?; + conn.flush().map_err(|e| anyhow!("flush: {}", e))?; + return Ok(()); + } + bail!("no toplevel window found for pid {}", pid) + } + /// Recursively search for a window with matching PID. fn find_window_by_pid_recursive( conn: &impl x11rb::connection::Connection, @@ -869,23 +1319,123 @@ impl AccessibilityReader for LinuxAccessibility { return Ok(()); } - // Use Action interface for other actions + // Most elements have no context-menu action; the reliable route is a + // synthesized right-click at the element's center through the AT-SPI + // DeviceEventController (no direct X dependency), matching what a + // user does. An explicit context-named action is preferred when the + // element exposes one. + if action == Action::ShowContextMenu { + // Querying the Action interface on an element that does not + // implement it crashes GTK's atk-bridge (an ATK_IS_ACTION + // assertion followed by a D-Bus marshalling abort takes the whole + // app down), so gate on the advertised interface set. + if let Ok(accessible) = + Self::create_accessible_proxy(&conn, &handle.bus_name, &handle.object_path).await + && let Ok(interfaces) = accessible.get_interfaces().await + && interfaces.contains(atspi::Interface::Action) + && let Ok(action_proxy) = + Self::create_action_proxy(&conn, &handle.bus_name, &handle.object_path).await + && let Ok(names) = Self::list_action_names(&action_proxy).await + && let Some(index) = names + .iter() + .position(|a| a.to_lowercase().contains("context")) + && matches!(action_proxy.do_action(index as i32).await, Ok(true)) + { + return Ok(()); + } + let component = + Self::create_component_proxy(&conn, &handle.bus_name, &handle.object_path).await?; + let (x, y, width, height) = component + .get_extents(CoordType::Screen) + .await + .map_err(|e| anyhow!("Failed to get extents: {}", e))?; + // The synthesized pointer event lands on whatever window is on + // top at those coordinates, so activate the target app's window + // first in case another application is covering it. + if let Ok(pid_reply) = conn + .call_method( + Some("org.freedesktop.DBus"), + "/org/freedesktop/DBus", + Some("org.freedesktop.DBus"), + "GetConnectionUnixProcessID", + &(handle.bus_name.as_str(),), + ) + .await + && let Ok(pid) = pid_reply.body().deserialize::() + { + let _ = Self::activate_window_for_pid(pid); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + } + conn.call_method( + Some("org.a11y.atspi.Registry"), + "/org/a11y/atspi/registry/deviceeventcontroller", + Some("org.a11y.atspi.DeviceEventController"), + "GenerateMouseEvent", + &(x + width / 2, y + height / 2, "b3c"), + ) + .await + .map_err(|e| anyhow!("Failed to synthesize right-click: {}", e))?; + return Ok(()); + } + + // A user click on a table cell both selects the row and activates it; + // the AT-SPI "activate" action alone does not move the selection, so + // grab focus first (which selects the row in GTK tree views). + if action == Action::Click + && let Ok(accessible) = + Self::create_accessible_proxy(&conn, &handle.bus_name, &handle.object_path).await + && Self::fetch_role(&accessible).await == Some(AtspiRole::TableCell) + && let Ok(component) = + Self::create_component_proxy(&conn, &handle.bus_name, &handle.object_path).await + { + let _ = component.grab_focus().await; + } + + // Use Action interface for other actions. Querying the Action + // interface on an element that does not implement it crashes GTK's + // atk-bridge (an ATK_IS_ACTION assertion followed by a D-Bus + // marshalling abort takes the whole app down), so gate on the + // advertised interface set first. + let accessible = + Self::create_accessible_proxy(&conn, &handle.bus_name, &handle.object_path).await?; + let interfaces = accessible + .get_interfaces() + .await + .map_err(|e| anyhow!("Failed to get interfaces: {}", e))?; + if !interfaces.contains(atspi::Interface::Action) { + bail!("Element does not support the Action interface"); + } let action_proxy = Self::create_action_proxy(&conn, &handle.bus_name, &handle.object_path).await?; + let action_names = Self::list_action_names(&action_proxy) + .await + .map_err(|e| anyhow!("Failed to get actions: {}", e))?; + // Map accesskit Action to AT-SPI action index let action_index = match action { - Action::Click => 0, // Primary action is usually index 0 + Action::Click => { + // Prefer an explicitly click-like action by name: GTK tree + // cells list "expand or contract" before "activate", so + // index 0 is not always the primary action. + action_names + .iter() + .position(|a| { + let name = a.to_lowercase(); + name.contains("activate") + || name.contains("click") + || name.contains("press") + || name.contains("toggle") + }) + .map(|i| i as i32) + .unwrap_or(0) + } Action::ShowContextMenu => { // Find "showContextMenu" or similar action by name - let actions = action_proxy - .get_actions() - .await - .map_err(|e| anyhow!("Failed to get actions: {}", e))?; - actions + action_names .iter() .position(|a| { - let name = a.name.to_lowercase(); + let name = a.to_lowercase(); name.contains("context") || name.contains("menu") }) .map(|i| i as i32) @@ -893,25 +1443,17 @@ impl AccessibilityReader for LinuxAccessibility { } Action::Increment => { // Find "increment" action - let actions = action_proxy - .get_actions() - .await - .map_err(|e| anyhow!("Failed to get actions: {}", e))?; - actions + action_names .iter() - .position(|a| a.name.to_lowercase().contains("increment")) + .position(|a| a.to_lowercase().contains("increment")) .map(|i| i as i32) .unwrap_or(-1) } Action::Decrement => { // Find "decrement" action - let actions = action_proxy - .get_actions() - .await - .map_err(|e| anyhow!("Failed to get actions: {}", e))?; - actions + action_names .iter() - .position(|a| a.name.to_lowercase().contains("decrement")) + .position(|a| a.to_lowercase().contains("decrement")) .map(|i| i as i32) .unwrap_or(-1) } @@ -944,23 +1486,24 @@ impl AccessibilityReader for LinuxAccessibility { let conn = self.connection.connection().clone(); let value = value.to_string(); - // Try EditableText interface first (for text fields) - if let Ok(editable) = - Self::create_editable_text_proxy(&conn, &handle.bus_name, &handle.object_path).await - && editable.set_text_contents(&value).await.is_ok() + // Prefer the Value interface for numeric values: GTK spin buttons + // expose both EditableText and Value, but text written through + // EditableText is not committed to the numeric value until the + // widget is activated, so setting the value directly is the only + // route that takes effect immediately. + if let Ok(numeric_value) = value.parse::() + && let Ok(value_proxy) = + Self::create_value_proxy(&conn, &handle.bus_name, &handle.object_path).await + && value_proxy.set_current_value(numeric_value).await.is_ok() { return Ok(()); } - // Fallback to Value interface (for sliders, spin buttons) - if let Ok(value_proxy) = - Self::create_value_proxy(&conn, &handle.bus_name, &handle.object_path).await - && let Ok(numeric_value) = value.parse::() + // EditableText interface (for text fields) + if let Ok(editable) = + Self::create_editable_text_proxy(&conn, &handle.bus_name, &handle.object_path).await + && editable.set_text_contents(&value).await.is_ok() { - value_proxy - .set_current_value(numeric_value) - .await - .map_err(|e| anyhow!("Failed to set value: {}", e))?; return Ok(()); } @@ -1033,9 +1576,11 @@ impl AccessibilityReader for LinuxAccessibility { help: element.help.clone(), role_description: element.role_description.clone(), identifier: element.identifier.clone(), + native_id: element.native_id, bounds: element.bounds, enabled: element.enabled, focused: element.focused, + checked: element.checked, actions: element.actions.clone(), children: vec![], // hit_test returns a single element without children }); @@ -1174,6 +1719,31 @@ fn current_timestamp() -> u64 { .unwrap_or(0) } +/// GTK (unlike WebKitGTK) does not set the Checkable AT-SPI state on its +/// toggle widgets, so treat inherently-checkable roles as checkable too. +fn is_checkable(role: Role, states: &atspi::StateSet) -> bool { + states.contains(atspi::State::Checkable) + || matches!( + role, + Role::CheckBox + | Role::RadioButton + | Role::Switch + | Role::MenuItemCheckBox + | Role::MenuItemRadio + ) +} + +/// Stable per-node identity for AT-SPI accessibles: a hash of the D-Bus +/// (bus name, object path) pair, which addresses the same underlying +/// accessible object across walks for as long as the app keeps it alive. +fn native_identity(bus_name: &str, object_path: &str) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + bus_name.hash(&mut hasher); + object_path.hash(&mut hasher); + hasher.finish() +} + /// Build a minimal Element from AT-SPI event data. async fn build_element_from_event( conn: &zbus::Connection, @@ -1184,13 +1754,14 @@ async fn build_element_from_event( .await .ok()?; - let atspi_role = proxy.get_role().await.ok()?; + let atspi_role = LinuxAccessibility::fetch_role(&proxy).await?; let role = LinuxAccessibility::map_role(atspi_role); // Use a placeholder key since we're not caching this element let placeholder_key = ElementKey::from_ffi(1); let mut element = Element::new(placeholder_key, role); + element.native_id = Some(native_identity(bus_name, object_path)); element.title = proxy.name().await.ok().filter(|s| !s.is_empty()); element.description = proxy.description().await.ok().filter(|s| !s.is_empty()); element.identifier = proxy.accessible_id().await.ok().filter(|s| !s.is_empty()); @@ -1200,6 +1771,9 @@ async fn build_element_from_event( element.enabled = !states.contains(atspi::State::Sensitive) || states.contains(atspi::State::Enabled); element.focused = states.contains(atspi::State::Focused); + if is_checkable(role, &states) { + element.checked = Some(states.contains(atspi::State::Checked)); + } } // Try to get bounds from Component interface