From df1c15a325af6508dca9d7f84fcec48228bac431 Mon Sep 17 00:00:00 2001 From: Kevin Edey Date: Fri, 29 May 2026 14:06:20 -0600 Subject: [PATCH 1/2] MobBridge template: screenshot + id-addressable scroll for the test harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the Kotlin side of mob's in-process test-harness driving (paired with mob's screenshot/3, scroll_info/1, scroll_to/3 NIFs): - screenshot(format, quality, scale): PixelCopy of the activity window → PNG/JPEG bytes (decor-view draw fallback pre-API-26). - An id-keyed ScrollHandle registry (ScrollState + LazyListState + measured viewport); the :scroll and lazy-list composables register themselves by their :id prop and report viewport size via onGloballyPositioned. - scrollInfo(id) → flat JSON {offset,content,viewport,max,kind} (kind "pixel" for verticalScroll, "index" for LazyColumn); scrollTo(id, x, y) drives the registered scroll state on the main thread. Lets a remotely-connected agent capture pixels and scroll deterministically over Erlang dist with no adb. Verified on a moto g power (Android 11). Co-Authored-By: Claude Opus 4.8 --- .../app/src/main/java/MobBridge.kt.eex | 198 +++++++++++++++++- 1 file changed, 195 insertions(+), 3 deletions(-) diff --git a/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex b/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex index e4340ad..b83d3fd 100644 --- a/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex +++ b/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex @@ -90,9 +90,15 @@ import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.LazyListState import androidx.compose.foundation.lazy.items import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.ScrollState import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll +import androidx.compose.ui.layout.onGloballyPositioned +import android.view.PixelCopy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import androidx.compose.foundation.text.KeyboardActions import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.ui.draw.clip @@ -279,6 +285,171 @@ object MobBridge { fun getOrCreateLazyListState(handle: Int): LazyListState = lazyListStates.getOrPut(handle) { LazyListState() } + // ── Test harness: id-addressable scroll registry + in-process capture ────── + // + // Gives a remotely-connected agent (Mob.Test.screenshot/scroll_info/scroll_to) + // pixels and deterministic scroll over Erlang dist, with no adb/xcrun. A + // ScrollHandle carries whichever Compose scroll state backs the node plus its + // measured viewport; the :scroll / lazy-list composables register themselves + // here by their :id prop. + class ScrollHandle { + var scrollState: ScrollState? = null // pixel-precise vertical/horizontal scroll + var lazyState: LazyListState? = null // item-indexed list + var viewportPx: Int = 0 // measured viewport extent (for ScrollState kind) + var horizontal: Boolean = false + } + + private val scrollHandlesById = mutableMapOf() + private val mainScope = CoroutineScope(Dispatchers.Main) + + fun scrollHandle(id: String): ScrollHandle = + scrollHandlesById.getOrPut(id) { ScrollHandle() } + + /** + * Capture the activity window in-process and return PNG/JPEG bytes. + * Called from nif_screenshot/3 via JNI. `scale` is a multiplier of native + * resolution (1.0 = full, 0.5 = half). Returns null when there is no live + * window (e.g. backgrounded) so the NIF can report {:error, :no_window}. + */ + @JvmStatic + fun screenshot(format: String, quality: Int, scale: Double): ByteArray? { + val activity = activityRef?.get() ?: return null + val window = activity.window ?: return null + val decor = window.decorView + val w = decor.width + val h = decor.height + if (w <= 0 || h <= 0) return null + + val src = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888) + val latch = java.util.concurrent.CountDownLatch(1) + var ok = false + val handler = android.os.Handler(android.os.Looper.getMainLooper()) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + PixelCopy.request(window, src, { result -> + ok = result == PixelCopy.SUCCESS + latch.countDown() + }, handler) + } else { + // Pre-O fallback: draw the decor view (misses SurfaceView/GL layers). + handler.post { + ok = + try { + decor.draw(android.graphics.Canvas(src)); true + } catch (e: Throwable) { + false + } + latch.countDown() + } + } + + if (!latch.await(2, java.util.concurrent.TimeUnit.SECONDS) || !ok) return null + + val bmp = + if (scale > 0.0 && scale != 1.0) { + Bitmap.createScaledBitmap( + src, + (w * scale).toInt().coerceAtLeast(1), + (h * scale).toInt().coerceAtLeast(1), + true + ) + } else { + src + } + + val out = java.io.ByteArrayOutputStream() + if (format == "jpeg") { + bmp.compress(Bitmap.CompressFormat.JPEG, quality.coerceIn(0, 100), out) + } else { + bmp.compress(Bitmap.CompressFormat.PNG, 100, out) + } + return out.toByteArray() + } + + /** + * Read a scroll view's offset/extent as a flat JSON object (the shape + * Mob.Test.scroll_info/2 decodes). Returns null when no scroll view is + * registered under `id`. `kind` is "pixel" for ScrollState (px units) or + * "index" for LazyListState (item-index units; viewport = visible items). + */ + @JvmStatic + fun scrollInfo(id: String): String? { + val h = scrollHandlesById[id] ?: return null + + h.scrollState?.let { s -> + val vp = h.viewportPx.toDouble() + val maxV = s.maxValue.toDouble() + val content = maxV + vp + val off = s.value.toDouble() + val o = JSONObject() + if (h.horizontal) { + o.put("offset_x", off); o.put("offset_y", 0.0) + o.put("content_w", content); o.put("content_h", vp) + o.put("viewport_w", vp); o.put("viewport_h", vp) + o.put("max_x", maxV); o.put("max_y", 0.0) + } else { + o.put("offset_x", 0.0); o.put("offset_y", off) + o.put("content_w", vp); o.put("content_h", content) + o.put("viewport_w", vp); o.put("viewport_h", vp) + o.put("max_x", 0.0); o.put("max_y", maxV) + } + o.put("kind", "pixel") + return o.toString() + } + + h.lazyState?.let { ls -> + val li = ls.layoutInfo + val total = li.totalItemsCount.toDouble() + val visible = li.visibleItemsInfo.size.toDouble() + val first = ls.firstVisibleItemIndex.toDouble() + val maxIdx = (total - visible).coerceAtLeast(0.0) + val o = JSONObject() + o.put("offset_x", 0.0); o.put("offset_y", first) + o.put("content_w", 0.0); o.put("content_h", total) + o.put("viewport_w", 0.0); o.put("viewport_h", visible) + o.put("max_x", 0.0); o.put("max_y", maxIdx) + o.put("kind", "index") + return o.toString() + } + + return null + } + + /** + * Scroll the view registered under `id` to absolute (x, y). Pixel views use + * the relevant axis; index lists use y as an item index. Runs the suspend + * scroll on the main thread and blocks the NIF thread until it completes. + */ + @JvmStatic + fun scrollTo(id: String, x: Double, y: Double): Boolean { + val h = scrollHandlesById[id] ?: return false + val latch = java.util.concurrent.CountDownLatch(1) + var ok = false + mainScope.launch { + try { + val s = h.scrollState + val ls = h.lazyState + when { + s != null -> { + val target = (if (h.horizontal) x else y).toInt() + s.scrollTo(target.coerceIn(0, s.maxValue)) + ok = true + } + ls != null -> { + ls.scrollToItem(y.toInt().coerceAtLeast(0)) + ok = true + } + } + } catch (e: Throwable) { + ok = false + } finally { + latch.countDown() + } + } + latch.await(2, java.util.concurrent.TimeUnit.SECONDS) + return ok + } + private var activityRef: WeakReference? = null /** Called from mob_nif.c via JNI — initialise anything activity-scoped. */ @@ -399,6 +570,7 @@ object MobBridge { // is no longer relevant and would scroll the wrong list to a stale position. val newKey = if (transition != "none") { lazyListStates.clear() + scrollHandlesById.clear() _rootState.value.navKey + 1 } else { _rootState.value.navKey @@ -2681,12 +2853,27 @@ private fun RenderNodeInner(node: MobNode, modifier: Modifier) { } "scroll" -> { val scrollState = rememberScrollState() - if (node.props["axis"] == "horizontal") { - Row(modifier = m.horizontalScroll(scrollState)) { + val horizontal = node.props["axis"] == "horizontal" + // Register by :id so Mob.Test.scroll_info/scroll_to can address it, + // and record the measured viewport (ScrollState doesn't expose it). + val id = node.props["id"] as? String + val regMod: Modifier = + if (id != null) { + val handle = MobBridge.scrollHandle(id) + handle.scrollState = scrollState + handle.horizontal = horizontal + Modifier.onGloballyPositioned { + handle.viewportPx = if (horizontal) it.size.width else it.size.height + } + } else { + Modifier + } + if (horizontal) { + Row(modifier = m.then(regMod).horizontalScroll(scrollState)) { node.children.forEach { RenderNode(it) } } } else { - Column(modifier = m.verticalScroll(scrollState).imePadding()) { + Column(modifier = m.then(regMod).verticalScroll(scrollState).imePadding()) { node.children.forEach { RenderNode(it) } } } @@ -3782,6 +3969,11 @@ private fun MobLazyList(node: MobNode, modifier: Modifier) { else LazyListState() } + // Register by :id so Mob.Test.scroll_info/scroll_to can address this list. + (node.props["id"] as? String)?.let { id -> + MobBridge.scrollHandle(id).lazyState = listState + } + val reachedEnd by remember { derivedStateOf { val lastVisible = listState.layoutInfo.visibleItemsInfo.lastOrNull()?.index ?: -1 From fa2cf9c90f110c20cba5b5247bbaeb404ad0c612 Mon Sep 17 00:00:00 2001 From: Kevin Edey Date: Fri, 29 May 2026 14:27:23 -0600 Subject: [PATCH 2/2] MobBridge template: element-frame registry for the test harness Kotlin side of mob's element_frames NIF: RenderNodeInner attaches a frameTrackingModifier (Modifier.testTag(id) + onGloballyPositioned) to any node carrying an :id, recording its window bounds into elementFramesById. elementFrames() returns them as JSON in dp (matching screenInfo / tap_xy units); cleared on navigation. Lets a remotely-connected agent read element positions by id over dist with no screenshot. Verified on a moto g power. Co-Authored-By: Claude Opus 4.8 --- .../app/src/main/java/MobBridge.kt.eex | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex b/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex index b83d3fd..4d83ba9 100644 --- a/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex +++ b/priv/templates/mob.new/android/app/src/main/java/MobBridge.kt.eex @@ -95,6 +95,8 @@ import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.ui.layout.onGloballyPositioned +import androidx.compose.ui.layout.boundsInWindow +import androidx.compose.ui.platform.testTag import android.view.PixelCopy import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -305,6 +307,42 @@ object MobBridge { fun scrollHandle(id: String): ScrollHandle = scrollHandlesById.getOrPut(id) { ScrollHandle() } + // ── Element frame registry (positions without a screenshot) ──────────────── + // + // Any rendered node with an :id gets a frameTrackingModifier that records its + // window bounds (px) here and tags it with a Compose testTag. elementFrames() + // returns them in dp so an agent can locate/drive elements by id over dist + // with no image bytes. Populated by RenderNodeInner. + private val elementFramesById = mutableMapOf() + + fun recordElementFrame(id: String, x: Float, y: Float, w: Float, h: Float) { + elementFramesById[id] = floatArrayOf(x, y, w, h) + } + + fun frameTrackingModifier(id: String): Modifier = + Modifier + .testTag(id) + .onGloballyPositioned { c -> + val b = c.boundsInWindow() + recordElementFrame(id, b.left, b.top, b.width, b.height) + } + + /** JSON {id:[x,y,w,h], ...} in dp (matches screenInfo / tap_xy units). */ + @JvmStatic + fun elementFrames(): String { + val density = activityRef?.get()?.resources?.displayMetrics?.density ?: 1f + val o = JSONObject() + for ((id, f) in elementFramesById) { + val arr = org.json.JSONArray() + arr.put((f[0] / density).toDouble()) + arr.put((f[1] / density).toDouble()) + arr.put((f[2] / density).toDouble()) + arr.put((f[3] / density).toDouble()) + o.put(id, arr) + } + return o.toString() + } + /** * Capture the activity window in-process and return PNG/JPEG bytes. * Called from nif_screenshot/3 via JNI. `scale` is a multiplier of native @@ -571,6 +609,7 @@ object MobBridge { val newKey = if (transition != "none") { lazyListStates.clear() scrollHandlesById.clear() + elementFramesById.clear() _rootState.value.navKey + 1 } else { _rootState.value.navKey @@ -2825,7 +2864,11 @@ private fun RenderNodeInner(node: MobNode, modifier: Modifier) { val tapModifier = if (tapHandle != null && node.type != "button") { modifier.clickable { MobBridge.nativeSendTap(tapHandle) } } else modifier - val m = tapModifier.then(nodeModifier(node.props)) + val base = tapModifier.then(nodeModifier(node.props)) + // Track on-screen frame + set a testTag for any node carrying an :id, so the + // agent can read positions (Mob.Test.element_frames) without a screenshot. + val trackId = node.props["id"] as? String + val m = if (trackId != null) base.then(MobBridge.frameTrackingModifier(trackId)) else base when (node.type) { "column" -> Column(modifier = m) { node.children.forEach { child ->