diff --git a/README.md b/README.md index 8683bcce3..2d6d47241 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,21 @@ cd artemis > **Tip**: Opens `http://localhost:8000` in your default browser with a device connection wizard, live screen mirroring, prompt sandbox, and execution replays. You can also run directly from CLI: `uv run artemis run "Open Settings, find Battery and tell me current level" --profile flash`. +### iOS Devices (macOS, Xcode 27+) + +CLI, embedded Python SDK, web console, and daemon tasks can target iOS +simulators through Xcode's native MCP device-interaction tools, and paired +physical iPhones/iPads through `devicectl` plus a WebDriverAgent runner +installed on the device. See the +[iOS setup and usage guide](./docs/ios.md) for prerequisites, Xcode access +approval, WDA setup, and current limits: + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "Open Settings and inspect the General screen" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -300,7 +315,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators. +- [x] **iOS Platform Expansion**: [iOS support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming, covering simulators and paired physical devices. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index 67f461c58..2594d4440 100644 --- a/README_CN.md +++ b/README_CN.md @@ -106,6 +106,17 @@ cd artemis > **提示**:启动后将自动在默认浏览器中打开 Web 控制台(`http://localhost:8000`),提供设备连接向导、实时投屏、任务演练与状态回放面板。你也可以通过命令行直接运行:`uv run artemis run "打开系统设置,找到电池选项并告诉我当前电量" --profile flash`。 +### iOS 设备(macOS,Xcode 27+) + +CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器;真机(iPhone/iPad)则通过 `devicectl` 管理生命周期、并在设备上安装 WebDriverAgent 负责界面操作。 +安装要求、Xcode 权限审批、WDA 配置和当前限制见 [iOS 使用指南(英文)](./docs/ios.md): + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "打开系统设置并查看通用页面" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -298,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求: ## 路线图 - [ ] **Android Studio 深度集成**:推出官方 IDE 插件与协同工作流,支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**:将视觉感知与自动化执行引擎拓展至 iOS 真机与模拟器。 +- [x] **iOS 跨平台支持**:[iOS 支持](./docs/ios.md)已支持 Xcode 27+,覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏,含模拟器与已配对真机。 - [ ] **端侧轻量化模型**:支持离线运行的轻量级 Edge VLM,实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**:支持自然语音下发任务与实时打断(Barge-in)控制。 diff --git a/apps/admin_console/index.html b/apps/admin_console/index.html index 0d60c4c06..fc0fc2dfa 100644 --- a/apps/admin_console/index.html +++ b/apps/admin_console/index.html @@ -1442,29 +1442,6 @@

No devices detected'; - return; - } - - devices.forEach(d => { - const opt = document.createElement('option'); - opt.value = d.serial; - opt.innerText = `${d.serial} (${d.status})`; - select.appendChild(opt); - }); - } catch (err) { - console.error("Failed to fetch devices:", err); - } - } - let replayTools = []; async function fetchReplayTools() { diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index c708629dd..50af42506 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -113,21 +113,15 @@ class ReplayManager: def __init__( self, workspace_root: Path = WORKSPACE_ROOT, - device_id: str = None, original_db_path: Path = None, - init_device: bool = False, ): - """Initializes the ReplayManager and autodetects the connected Android device. + """Initializes the ReplayManager. Args: workspace_root (Path, optional): The absolute path to the workspace root. Defaults to WORKSPACE_ROOT. - device_id (str, optional): The serial ID of the target Android - device. If not specified, autodetects the first connected device. original_db_path (Path, optional): Path to the master SQLite database. Defaults to DB_PATH. - init_device (bool, optional): Whether to initialize the ADB device - connection on startup. """ self.workspace_root = workspace_root self.traces_path = TRACES_PATH @@ -137,62 +131,6 @@ def __init__( self.db_path = Path(original_db_path) if original_db_path else DB_PATH self.images_dir = IMAGES_DIR - # Device connection and autodetection - self.device_id = None - self.w = None - self.h = None - self.adb = None - self.ui_client = None - - if init_device: - try: - self._init_device(device_id) - except Exception as e: - print( - "Warning: Connected device could not be initialized during" - f" ReplayManager startup: {e}" - ) - - def _init_device(self, device_id: str = None): - """Initializes connection to the target device and queries screen metrics.""" - from adbutils import AdbClient - - try: - from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient - except ImportError: - raise ImportError( - "Failed to import UIAutomatorClient. Ensure artemis package is installed in path." - ) - - self.adb = AdbClient(host="localhost", port=5037) - try: - devices = self.adb.device_list() - except Exception as adb_err: - raise ConnectionError( - f"Failed to query device list from ADB server: {adb_err}" - ) from adb_err - - if not devices: - raise ConnectionError( - "No active ADB devices connected. Please connect an Android device via ADB." - ) - - if device_id: - matched_device = next((d.serial for d in devices if d.serial == device_id), None) - if not matched_device: - raise ConnectionError( - f"Requested device '{device_id}' is not connected." - f" Connected devices: {[d.serial for d in devices]}" - ) - self.device_id = matched_device - else: - self.device_id = devices[0].serial - - self.ui_client = UIAutomatorClient(device_id=self.device_id) - ui_data = self.ui_client.get_screen_data() - self.w, self.h = ui_data.width, ui_data.height - print(f"Connected to device: {self.device_id} ({self.w}x{self.h})") - def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path: """Chunks traces, screenshots, and metadata for a specific execution session. @@ -396,18 +334,6 @@ def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path print("Chunking trace completion status: success") return output_dir - def list_devices(self) -> list[dict]: - """Dynamically queries the ADB server for connected Android devices.""" - try: - from adbutils import AdbClient - - adb = AdbClient(host="localhost", port=5037) - devices = adb.device_list() - return [{"serial": d.serial, "status": "online"} for d in devices] - except Exception as e: - print(f"Warning: Failed to query device list from ADB: {e}") - return [] - def load_session_goal( self, session_id: str, step_dir: Path, original_db_path: str = None ) -> str: @@ -1604,12 +1530,69 @@ def _preemptive_clip_video( except Exception as e: print(f"Warning: Failed to update sandbox SQLite database: {e}") + def _session_device_info(self, session_id: str) -> dict: + """The recorded session's ``device_info`` blob (``{}`` when absent).""" + import sqlite3 + + try: + conn = sqlite3.connect(self.db_path) + try: + row = conn.execute( + "SELECT device_info FROM sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + finally: + conn.close() + except sqlite3.Error: + return {} + if not row or not row[0]: + return {} + try: + info = json.loads(row[0]) + except (ValueError, TypeError): + # Malformed device_info: simulate with the defaults. + return {} + return info if isinstance(info, dict) else {} + + @staticmethod + def _replay_device_context(device_info: dict, override_device_id: str | None): + """Build the simulated ``DeviceContext`` for a replayed session. + + The recorded session decides the platform: an iOS session replays + through the native iOS driver, never through ADB. The frontend's + device pick only retargets iOS replays (the picker was always + decorative for Android and stays that way); the driver still + validates the UDID — simctl for simulators, devicectl for physical + devices — at connect time. + """ + from artemis.context import DeviceContext, DevicePlatform + + sim_platform = str( + device_info.get("mobile_platform") or DevicePlatform.ANDROID.value + ).lower() + mobile_platform = ( + DevicePlatform.IOS + if sim_platform == DevicePlatform.IOS.value + else DevicePlatform.ANDROID + ) + sim_device_id = device_info.get("device_id", "replay-device") + if mobile_platform == DevicePlatform.IOS and override_device_id: + sim_device_id = override_device_id + return DeviceContext( + host_platform=("DARWIN" if mobile_platform == DevicePlatform.IOS else "LINUX"), + mobile_platform=mobile_platform, + device_id=sim_device_id, + device_width=device_info.get("device_width", 1080), + device_height=device_info.get("device_height", 2400), + ) + def create_ctx( self, session_id: str, step_number: int, agent_name: str = "explorer", replay_id: str = None, + override_device_id: str = None, ) -> ArtemisContext: """Creates a sandboxed ArtemisContext and DataEngine for the given session and step.""" self._ensure_session_chunked(session_id) @@ -1682,37 +1665,13 @@ def create_ctx( except ImportError as import_err: raise ImportError(f"Failed to import Artemis core modules: {import_err}") - import sqlite3 - - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - cursor.execute( - "SELECT device_info FROM sessions WHERE session_id = ?", - (session_id,), - ) - row = cursor.fetchone() - conn.close() - - device_info = {} - if row and row[0]: - try: - device_info = json.loads(row[0]) - except (ValueError, TypeError): - # Malformed device_info: simulate with the defaults below. - pass - - sim_device_id = device_info.get("device_id", "replay-device") - sim_w = device_info.get("device_width", 1080) - sim_h = device_info.get("device_height", 2400) - - device_context = DeviceContext( - host_platform="LINUX", - mobile_platform=DevicePlatform.ANDROID, - device_id=sim_device_id, - device_width=sim_w, - device_height=sim_h, + device_info = self._session_device_info(session_id) + device_context = self._replay_device_context(device_info, override_device_id) + sim_device_id = device_context.device_id + print( + f"Replay simulated device: {sim_device_id} " + f"({device_context.device_width}x{device_context.device_height})" ) - print(f"Replay simulated device: {sim_device_id} ({sim_w}x{sim_h})") from artemis.config import get_default_llm_config @@ -1765,6 +1724,7 @@ def instantiate_state( pre_image_meta = json.load(f) initial_goal = self.load_session_goal(session_id, step_dir, str(self.db_path)) + device_info = self._session_device_info(session_id) # State has no fields for the app info or device date (extra="forbid"). ui_hier, decisions, _app_info, _dev_date = self.extract_state_prepopulation_data( @@ -1781,7 +1741,10 @@ def instantiate_state( latest_screenshot=str(step_dir / "pre.jpg") if (step_dir / "pre.jpg").exists() else str(step_dir / "post.jpg"), - operator_raw_data={"width": self.w, "height": self.h}, + operator_raw_data={ + "width": device_info.get("device_width"), + "height": device_info.get("device_height"), + }, current_step_id=step_data["step_id"], subagent_calls=step_data.get("subagent_calls") or [], latest_ui_hierarchy=ui_hier, @@ -1872,6 +1835,7 @@ async def replay_step_tool( step_number=step_number, agent_name=agent_name, replay_id=replay_id, + override_device_id=override_device_id, ) state = self.instantiate_state( diff --git a/apps/admin_console/routers/replay.py b/apps/admin_console/routers/replay.py index 2a83b6e24..d6de82c9e 100644 --- a/apps/admin_console/routers/replay.py +++ b/apps/admin_console/routers/replay.py @@ -30,12 +30,6 @@ router = APIRouter(tags=["replay"]) -@router.get("/api/devices") -async def list_devices(): - """Dynamically queries the ADB server for connected Android devices.""" - return replay_manager.list_devices() - - @router.get("/api/replay/tools") async def get_replay_tools(): """Returns the list of registered tools available for replay.""" diff --git a/apps/admin_console/routers/stream.py b/apps/admin_console/routers/stream.py index 5bc3c6889..6cdd2f580 100644 --- a/apps/admin_console/routers/stream.py +++ b/apps/admin_console/routers/stream.py @@ -45,12 +45,13 @@ async def stream_device_live(): @router.get("/api/stream/device-state") async def get_device_stream_state(): - """Returns whether an ADB device is connected and live streaming is available.""" - serial = await device_stream_service.get_device_serial() + """Returns whether a device is connected and live streaming is available.""" + target = await device_stream_service.get_stream_target() return JSONResponse( { - "connected": serial is not None, - "serial": serial, - "live_stream_url": "/api/stream/device-live" if serial else None, + "connected": target is not None, + "serial": target["serial"] if target else None, + "platform": target["platform"] if target else None, + "live_stream_url": "/api/stream/device-live" if target else None, } ) diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index c8ce18343..ac1e0e835 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -28,6 +28,7 @@ adb_server_connection, ) from artemis.core.diagnostics.schema import SystemReadinessReport +from artemis.runtime.device_target import normalize_device_platform router = APIRouter(prefix="/api/system", tags=["system"]) @@ -93,9 +94,13 @@ def _require_local_lifecycle_request(request: Request) -> None: class SelectDeviceRequest(BaseModel): - """Payload to select an active target Android device.""" + """Payload to select an active target device.""" - serial: str = Field(description="Serial number or identifier of the Android device to select") + serial: str = Field(description="Serial number or identifier of the device to select") + platform: str = Field( + default="android", + description="Device platform: 'android' (default) or 'ios' (device UDID)", + ) @router.get("/readiness", response_model=SystemReadinessReport) @@ -106,11 +111,28 @@ async def get_system_readiness(force: bool = False) -> SystemReadinessReport: @router.post("/devices/select") async def select_active_device(request: SelectDeviceRequest): - """Select the active Android device or emulator for subsequent automated tasks.""" + """Select the active Android or iOS device for subsequent automated tasks.""" serial = request.serial.strip() if not serial: raise HTTPException(status_code=400, detail="Device serial cannot be empty.") + try: + platform = normalize_device_platform(request.platform) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + if platform == "ios": + # iOS has no ADB probe to retarget; validate the UDID and let the + # task payload's device_serial + platform carry the binding. + from artemis.runtime import ios_device_pool + + rejection = await ios_device_pool.validate_explicit_serial_async(serial) + if rejection: + raise HTTPException(status_code=400, detail=rejection) + return {"status": "success", "selected_serial": serial, "platform": "ios"} + readiness_engine.set_probe_target_serial(serial) # Return updated readiness report = await readiness_engine.run_all(force_refresh=True) diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 26fca4611..8e80d3198 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -19,7 +19,12 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse from artemis.core.diagnostics import readiness_engine -from artemis.runtime import DeviceExecutionLock, device_pool +from artemis.runtime import DeviceExecutionLock, device_pool, ios_device_pool +from artemis.runtime.device_target import ( + IOS_LOCK_SCOPE, + device_pool_for, + normalize_device_platform, +) try: from admin_console.core.state import state @@ -66,6 +71,26 @@ async def get_task_catalog(): } +@router.get("/api/v1/capabilities") +async def get_capabilities(): + """Static wire-feature advertisement — no device or service probing. + + ``platform.ios`` means the API accepts iOS submissions; it does not + imply Xcode approval or attached hardware on this host. + """ + return { + "api_version": "1", + "features": [ + "tasks.submit", + "tasks.get", + "tasks.stop", + "devices.list", + "system.readiness", + "platform.ios", + ], + } + + @router.post("/api/run") async def run_task(request: RunRequest): incoming_goals = [] @@ -113,6 +138,19 @@ async def run_task(request: RunRequest): "total_queued": len(state.queue_tasks), } + try: + platform = normalize_device_platform(request.platform) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + if platform == "ios" and request.locked_app_package: + raise HTTPException( + status_code=400, + detail="locked_app_package is unavailable for iOS tasks.", + ) + # Reject an explicit unknown/offline target before running the more # expensive readiness probe. Besides producing a stable SDK response, # this avoids probing the currently active device for a serial that can @@ -121,7 +159,9 @@ async def run_task(request: RunRequest): # fail downstream with a clear error instead. if request.device_serial: try: - rejection = await device_pool.validate_explicit_serial_async(request.device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + request.device_serial + ) except Exception: rejection = None if rejection: @@ -140,30 +180,35 @@ async def run_task(request: RunRequest): # With no explicit serial the probe itself resolves a live target (it # prefers the diagnostics target preference, then any unlocked ready # device); the verified serial is bound below. + # iOS skips the Android screen-lock probe entirely: the Xcode driver + # validates and boots the pinned simulator at execution time. target_serial = request.device_serial - device_probe = await readiness_engine.run_device_submission_probe(target_serial=target_serial) - if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: - locked_serial = ( - device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" - ) - detail = ( - f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( - " ", " " - ).strip() - if device_probe.summary == "Device Locked" - else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( - " ", " " - ).strip() + if platform != "ios": + device_probe = await readiness_engine.run_device_submission_probe( + target_serial=target_serial ) - raise HTTPException(status_code=409, detail=detail) + if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: + locked_serial = ( + device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" + ) + detail = ( + f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( + " ", " " + ).strip() + if device_probe.summary == "Device Locked" + else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( + " ", " " + ).strip() + ) + raise HTTPException(status_code=409, detail=detail) - if device_probe and device_probe.metadata.get("active_device"): - verified_serial = device_probe.metadata["active_device"].get("serial") - # Only auto-selected targets may be re-bound to the probed device. An - # explicitly requested serial is never silently replaced -- if it is - # invalid, enqueue_tasks rejects the submission with a clear error. - if verified_serial and not request.device_serial: - target_serial = verified_serial + if device_probe and device_probe.metadata.get("active_device"): + verified_serial = device_probe.metadata["active_device"].get("serial") + # Only auto-selected targets may be re-bound to the probed device. An + # explicitly requested serial is never silently replaced -- if it is + # invalid, enqueue_tasks rejects the submission with a clear error. + if verified_serial and not request.device_serial: + target_serial = verified_serial return await task_queue_service.enqueue_tasks( incoming_goals, @@ -178,6 +223,8 @@ async def run_task(request: RunRequest): ingress=request.ingress or "frontend", session_id=request.session_id, conversation_id=request.conversation_id, + platform=platform, + ios_workspace=request.ios_workspace, ) @@ -199,9 +246,11 @@ async def get_run_defaults(): @router.get("/api/devices") async def list_devices(): - """List all connected Android devices with their busy / idle status.""" - devices = await device_pool.list_devices_async() - return {"devices": [d.to_dict() for d in devices]} + """List all connected devices (Android + iOS) with lock status.""" + android_devices, ios_devices = await asyncio.gather( + device_pool.list_devices_async(), ios_device_pool.list_devices_async() + ) + return {"devices": [d.to_dict() for d in (*android_devices, *ios_devices)]} @router.post("/api/stop") @@ -276,6 +325,9 @@ async def get_status(): "pid": owner.pid, "ingress": owner.ingress, "acquired_at": owner.acquired_at, + "platform": "ios" + if getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + else "android", } for owner in active_owners.values() ] diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 144a97203..e0ee9cbe1 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -29,6 +29,10 @@ class RunRequest(BaseModel): locked_app_package: str | None = None app_path: str | None = None device_serial: str | None = None + # Target platform: "android" (default) or "ios". For iOS, device_serial is a + # simulator or physical UDID and ios_workspace optionally scopes the Xcode approval grant. + platform: str | None = None + ios_workspace: str | None = None ingress: str | None = "frontend" session_id: str | None = None conversation_id: str | None = None diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 6455a3e77..df9352751 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -15,7 +15,9 @@ """Device Live Screen Streaming Service. Provides real-time, low-latency device screen frames over HTTP MJPEG and WebSocket. -Operates concurrently with ADB agent actions with zero interference. +Android frames come from ``adb exec-out screencap``; iOS frames come +from the native ``xcrun simctl io screenshot`` capture. The target is +picked per frame so the stream follows whichever platform is under automation. """ import asyncio @@ -23,7 +25,11 @@ import subprocess import time from collections.abc import AsyncGenerator +from pathlib import Path +from artemis.config.paths import get_temp_dir +from artemis.runtime import DeviceExecutionLock, ios_device_pool +from artemis.runtime.device_target import IOS_LOCK_SCOPE from artemis.toolchain import find_adb logger = logging.getLogger("artemis.stream_service") @@ -39,8 +45,41 @@ def __init__(self): self._last_frame_time: float = 0.0 self._is_capturing = False self._capture_task: asyncio.Task | None = None + self._last_target: dict[str, str] | None = None + + async def get_stream_target(self) -> dict[str, str] | None: + """Pick the device the stream should follow right now. + + An actively locked iOS device wins (it is the device under + automation), then a connected Android device, then the single + unambiguous booted simulator. Returns ``{"platform", "serial"}``. + """ + try: + for owner in DeviceExecutionLock.get_active_owners().values(): + if ( + owner + and getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + and owner.device_id + ): + return {"platform": "ios", "serial": str(owner.device_id)} + except Exception as exc: + logger.debug(f"[StreamService] iOS lock-owner scan failed: {exc}") + + serial = await self._android_serial() + if serial: + return {"platform": "android", "serial": serial} + + udid = await ios_device_pool.select_device_async() + if udid: + return {"platform": "ios", "serial": udid} + return None async def get_device_serial(self) -> str | None: + """Serial of the currently streamable device, regardless of platform.""" + target = await self.get_stream_target() + return target["serial"] if target else None + + async def _android_serial(self) -> str | None: """Find the currently connected active ADB device serial.""" try: adb_bin = find_adb() @@ -60,27 +99,95 @@ async def get_device_serial(self) -> str | None: logger.warning(f"Error checking adb devices: {e}") return None + async def _capture_android(self, serial: str | None) -> bytes | None: + adb_bin = find_adb() + cmd = ( + [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] + if serial + else [adb_bin, "exec-out", "screencap", "-p"] + ) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout if proc.returncode == 0 and len(stdout) > 1000 else None + + async def _capture_ios(self, udid: str) -> bytes | None: + """One PNG frame via simctl (simulator) or devicectl (physical). + + Both tools only write files, so frames stream through one reused temp + path per UDID. + """ + from artemis.drivers.ios.discovery import find_physical_ios_device_sync + + frame_path = Path(get_temp_dir("streams")) / f"ios_stream_{udid}.png" + physical = await asyncio.to_thread(find_physical_ios_device_sync, udid) + if physical is not None: + cmd = [ + "xcrun", + "devicectl", + "device", + "capture", + "screenshot", + "--device", + udid, + "--destination", + str(frame_path), + ] + else: + cmd = [ + "xcrun", + "simctl", + "io", + udid, + "screenshot", + "--type=png", + str(frame_path), + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + await proc.communicate() + except asyncio.CancelledError: + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + await proc.communicate() + raise + if proc.returncode == 0 and frame_path.exists(): + data = frame_path.read_bytes() + return data if len(data) > 1000 else None + finally: + frame_path.unlink(missing_ok=True) + return None + async def _capture_loop(self): """Background frame capture loop that runs while listeners > 0.""" logger.info("[StreamService] Starting live screen capture loop...") while self._active_listeners > 0: try: start_t = time.time() - serial = await self.get_device_serial() - adb_bin = find_adb() - cmd = ( - [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] - if serial - else [adb_bin, "exec-out", "screencap", "-p"] - ) - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - if proc.returncode == 0 and len(stdout) > 1000: - self._latest_frame = stdout + target = await self.get_stream_target() + if target != self._last_target: + # Never carry frames across a device/platform switch. + self._latest_frame = None + self._last_target = target + frame = None + if target is not None: + if target["platform"] == "ios": + frame = await self._capture_ios(target["serial"]) + else: + frame = await self._capture_android(target["serial"]) + if frame is not None: + self._latest_frame = frame self._last_frame_time = time.time() elapsed = time.time() - start_t diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b99b4c919..f99d54e6b 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -52,6 +52,13 @@ request_cancel, trace_store, ) +from artemis.runtime.device_target import ( + IOS_LOCK_SCOPE, + IosTarget, + device_pool_for, + normalize_device_platform, + target_for_platform, +) logger = logging.getLogger(__name__) @@ -216,15 +223,16 @@ def _enforce_sync() -> None: ).start() @staticmethod - def _task_target(task_item: dict[str, Any]) -> AdbTarget: + def _task_target(task_item: dict[str, Any]) -> AdbTarget | IosTarget: + serial = str(task_item.get("device_serial") or "") or None + if normalize_device_platform(task_item.get("platform"), strict=False) == "ios": + # iOS targets never read a stale/malformed Android endpoint blob. + return IosTarget(serial=serial) endpoint_data = task_item.get("adb_endpoint") endpoint = ( - AdbEndpoint.from_mapping(endpoint_data) - if isinstance(endpoint_data, dict) - else current_adb_endpoint() + AdbEndpoint.from_mapping(endpoint_data) if isinstance(endpoint_data, dict) else None ) - serial = task_item.get("device_serial") - return AdbTarget(endpoint=endpoint, serial=str(serial) if serial else None) + return target_for_platform(task_item.get("platform"), serial=serial, endpoint=endpoint) @classmethod def _broadcast_event(cls, event_type: str, data: Any): @@ -498,7 +506,7 @@ def _build_worker_invocation( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, ) -> tuple[list[str], dict[str, str]]: """Assemble the worker subprocess command line and environment.""" expected_output = task_item.get("expected_output") @@ -525,7 +533,7 @@ def _build_worker_invocation( env["ARTEMIS_SESSION_ID"] = str(sess_id) env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend")) env["ARTEMIS_TASK_WORKER"] = "1" - target.endpoint.apply_to_environment(env) + target.apply_to_environment(env) env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope queue_ticket = task_item.get("queue_ticket") if queue_ticket: @@ -556,7 +564,14 @@ def _build_worker_invocation( if app_path: cmd.extend(["--app-path", str(app_path)]) device_serial = task_item.get("device_serial") - if device_serial: + if isinstance(target, IosTarget): + cmd.extend(["--platform", "ios"]) + ios_workspace = task_item.get("ios_workspace") + if ios_workspace: + cmd.extend(["--ios-workspace", str(ios_workspace)]) + if device_serial: + cmd.extend(["--device-serial", str(device_serial)]) + elif device_serial: cmd.extend(["--device-serial", str(device_serial)]) env["ADB_DEVICE_SERIAL"] = str(device_serial) return cmd, env @@ -569,7 +584,7 @@ def _register_worker_run( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, proc: asyncio.subprocess.Process, ) -> None: """Record the spawned worker in shared state and hand it the device reservation.""" @@ -580,7 +595,9 @@ def _register_worker_run( "process": proc, "device_id": str(device_serial) if device_serial else None, "lock_key": target.lock_key if device_serial else None, - "adb_endpoint": target.endpoint.to_dict(), + "platform": target.platform, + "target": target.to_dict(), + "adb_endpoint": (target.endpoint.to_dict() if isinstance(target, AdbTarget) else None), "goal": goal, "profile": profile, } @@ -909,6 +926,8 @@ def _find_duplicate_submission( device_serial: str | None, endpoint: AdbEndpoint, now: float, + *, + platform: str = "android", ) -> dict[str, Any] | None: """Return the short-circuit response for a duplicate submission, if any.""" # 1. Deduplication by session_id: if session_id is already running or queued, do not re-enqueue @@ -947,8 +966,14 @@ def _find_duplicate_submission( if isinstance(item, dict) and item.get("status") == "pending" and item.get("goal") == first_goal + and normalize_device_platform(item.get("platform"), strict=False) == platform and (not device_serial or item.get("device_serial") == device_serial) - and item.get("adb_endpoint", {}).get("identity") == endpoint.identity + # iOS items carry no ADB endpoint snapshot; matching the + # platform + serial is the whole identity there. + and ( + platform == "ios" + or (item.get("adb_endpoint") or {}).get("identity") == endpoint.identity + ) and (now - float(item.get("created_at", 0))) < 1.0 ), None, @@ -963,7 +988,9 @@ def _find_duplicate_submission( return None @classmethod - async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str, Any] | None: + async def _reject_unavailable_device( + cls, device_serial: str | None, platform: str = "android" + ) -> dict[str, Any] | None: """Return the rejection response for an unattached explicit serial, if any.""" # Strict device binding: reject an explicitly requested serial that is not # attached and authorized, instead of silently running on another device. @@ -971,9 +998,9 @@ async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str # the task can proceed and fail downstream with a clear no-device error. if device_serial: try: - from artemis.runtime import device_pool - - rejection = await device_pool.validate_explicit_serial_async(device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + device_serial + ) except Exception: rejection = None if rejection: @@ -1004,6 +1031,8 @@ def _create_queue_item( conversation_id: str | None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ) -> dict[str, Any]: """Reserve a device slot and build one pending queue item for a goal.""" sess_id = single_session_id if single_session_id else str(uuid.uuid4()) @@ -1015,7 +1044,7 @@ def _create_queue_item( device_id=assigned_serial or "pending", session_id=sess_id, ingress=ingress, - lock_scope=endpoint.identity, + lock_scope=IOS_LOCK_SCOPE if platform == "ios" else endpoint.identity, ) return { "session_id": sess_id, @@ -1028,7 +1057,11 @@ def _create_queue_item( "locked_app_package": locked_app_package, "app_path": app_path, "device_serial": assigned_serial, - "adb_endpoint": endpoint.to_dict(), + # iOS targets have no endpoint; serialize None rather than leaking + # the caller's ADB preference into the task snapshot. + "adb_endpoint": endpoint.to_dict() if platform != "ios" else None, + "platform": platform, + "ios_workspace": ios_workspace, "ingress": ingress, "conversation_id": conversation_id, "status": "pending", @@ -1052,13 +1085,20 @@ async def enqueue_tasks( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, ) -> dict[str, Any]: """Enqueues one or more goals and wakes up the background worker. ``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. + + ``platform="ios"`` enqueues a native iOS task: the serial is a simulator + or paired physical UDID, the device lock is scoped under ``ios``, and the worker + runs with ``--platform ios`` instead of ADB bindings. """ + platform = normalize_device_platform(platform, strict=False) verification_level = ( str(verification_level).strip().lower() or None if verification_level else None ) @@ -1070,22 +1110,19 @@ async def enqueue_tasks( endpoint = current_adb_endpoint() duplicate_response = cls._find_duplicate_submission( - goals, session_id, device_serial, endpoint, now + goals, session_id, device_serial, endpoint, now, platform=platform ) if duplicate_response is not None: return duplicate_response - rejection_response = await cls._reject_unavailable_device(device_serial) + rejection_response = await cls._reject_unavailable_device(device_serial, platform) if rejection_response is not None: return rejection_response single_session_id = session_id if (session_id and len(goals) == 1) else None if not device_serial: - # Device enumeration may block on ADB. - from artemis.runtime import device_pool - try: - device_serial = await device_pool.select_device_async() + device_serial = await device_pool_for(platform).select_device_async() except Exception: device_serial = None for i, goal in enumerate(goals): @@ -1105,6 +1142,8 @@ async def enqueue_tasks( conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, ) state.queue_items.append(task_item) enqueued_tasks.append(task_item) diff --git a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html index 44ec41208..c7bd726bb 100644 --- a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html @@ -35,7 +35,7 @@

👣 Real-time Agent Stream (Legacy Card View)

} diff --git a/apps/showcase_ui/src/app/core/models/session.model.ts b/apps/showcase_ui/src/app/core/models/session.model.ts index ef322d1cf..8733ac3d5 100644 --- a/apps/showcase_ui/src/app/core/models/session.model.ts +++ b/apps/showcase_ui/src/app/core/models/session.model.ts @@ -30,6 +30,7 @@ export interface TaskQueueItem { start_time?: number; device_serial?: string | null; device_id?: string | null; + platform?: 'android' | 'ios' | string | null; } export interface Session { @@ -44,6 +45,7 @@ export interface Session { device_serial?: string | null; device_id?: string | null; device_info?: any; + platform?: 'android' | 'ios' | string | null; } export interface AgentStatusResponse { diff --git a/apps/showcase_ui/src/app/core/models/system.model.ts b/apps/showcase_ui/src/app/core/models/system.model.ts index cf15406f0..cfa529573 100644 --- a/apps/showcase_ui/src/app/core/models/system.model.ts +++ b/apps/showcase_ui/src/app/core/models/system.model.ts @@ -34,6 +34,8 @@ export interface DeviceInfo { is_locked: boolean | null; is_emulator: boolean; installed_packages?: string[]; + /** 'android' (default) or 'ios' (iOS Simulator; serial is the UDID). */ + platform?: 'android' | 'ios'; } export interface AdbServerEndpoint { diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 060a01d25..571e0c82d 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -672,11 +672,12 @@

AI Model Setup

-

Android Device & Emulator Connection

+

Device & Emulator Connection

-

Local emulator, attached phone, wireless device, or another ADB server

+

Android device or emulator via ADB, or an iOS Simulator / paired iPhone or iPad (Xcode 27+)

+ @if (deviceGuidePlatform() === 'android') {

{{ isRemoteAdbServer() ? 'Refresh Devices' : 'Restart ADB' }} } + }
+ +
+ + +
+ + @if (deviceGuidePlatform() === 'android') { + - @if (isDeviceReady()) { + @if (androidDeviceReady()) {
@@ -750,22 +782,22 @@

Android Device & Emulator Connection

- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) { @@ -777,7 +809,7 @@

Android Device & Emulator Connection

} - @if (isEmulatorLaunching() || (!isDeviceReady() && adbProbe()?.summary === 'Device Booting')) { + @if (isEmulatorLaunching() || (!androidDeviceReady() && adbProbe()?.summary === 'Device Booting')) {
@@ -1561,22 +1593,22 @@

No Android Virtual Devices (AVD) Detected

- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) { @@ -1674,6 +1706,90 @@

No Android Virtual Devices (AVD) Detected

} + } @else { + + +
+ + + @if (selectedIosDevice(); as sim) { +
+
+
+ + {{ sim.model?.includes('iPad') ? 'tablet_mac' : 'phone_iphone' }} + +
+
+
+ {{ sim.model || sim.serial }} +
+
+ {{ sim.is_emulator ? 'iOS Simulator' : 'Physical iOS Device' }} + · + {{ sim.serial }} + @if (sim.product) { + · + {{ sim.product }} + } +
+
+
+ + {{ sim.state === 'device' ? (sim.is_emulator ? 'Booted' : 'Connected') : (sim.is_emulator ? 'Shutdown — boots on run' : sim.state) }} +
+
+
+ } + + + @if (iosDevices().length > 0) { +
+
+ phone_iphone +
+ iOS Devices + + {{ iosDevices().length }} available — select a simulator or paired iPhone/iPad to run iOS tasks + +
+
+
+ @for (dev of iosDevices(); track dev.serial) { + + } +
+
+ } @else { +
+ phone_iphone +
+
No iOS Devices Available
+
+ {{ iosProbe()?.description || 'iOS automation requires macOS with Xcode 27+ and an available simulator or a paired iPhone/iPad. Physical devices also need WebDriverAgent on the device.' }} +
+
+
+ } + +
+ + } +
@@ -1686,7 +1802,7 @@

No Android Virtual Devices (AVD) Detected

System Ready for Autonomous Tasks

-

All dependencies, AI model credentials, and Android device are verified and ready to run.

+

All dependencies, AI model credentials, and a target device are verified and ready to run.

@@ -1808,6 +1924,23 @@

System Ready for Autonomous Tasks

Pro
+ + + @if (selectedTarget(); as tgt) { + + }
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index c1a5e68d6..14c3c4dbd 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -1761,6 +1761,10 @@ $text-light: #94a3b8; margin-left: auto; flex-shrink: 0; + &.status-idle { + color: #b45309; + } + .status-dot { width: 7px; height: 7px; @@ -1768,6 +1772,12 @@ $text-light: #94a3b8; background: #10b981; box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); animation: statusDotPulse 2s infinite cubic-bezier(0.4, 0, 0.6, 1); + + &.dot-warn { + background: #f59e0b; + box-shadow: none; + animation: none; + } } } @@ -1811,6 +1821,26 @@ $text-light: #94a3b8; font-size: 14px; } + .chip-name { + color: $text-main; + } + + .sim-state-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: #94a3b8; + flex-shrink: 0; + + &.booted { + background: #10b981; + } + + &.booting { + background: #f59e0b; + } + } + .active-check { color: $emerald; font-size: 10.5px; @@ -2394,6 +2424,120 @@ $text-light: #94a3b8; display: flex; flex-direction: column; gap: 12px; + } + + .platform-tabs { + display: inline-flex; + align-items: center; + gap: 4px; + align-self: flex-start; + background: rgba(241, 245, 249, 0.85); + padding: 3px; + border-radius: 10px; + margin-bottom: 12px; + + .platform-tab { + display: inline-flex; + align-items: center; + gap: 6px; + height: 28px; + padding: 0 12px; + border: none; + border-radius: 8px; + background: transparent; + font-family: inherit; + font-size: 12px; + font-weight: 600; + color: $text-muted; + cursor: pointer; + transition: all 0.15s ease; + + .plat-icon { + font-size: 15px; + } + + .plat-count { + font-size: 10px; + font-weight: 700; + padding: 1px 6px; + border-radius: 8px; + background: rgba(100, 116, 139, 0.12); + color: inherit; + } + + &.active { + background: #ffffff; + color: $text-main; + box-shadow: 0 1px 3px rgba(15, 23, 42, 0.1); + } + } + } + + .ios-empty-state { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 16px; + border: 1px dashed $card-border-hover; + border-radius: 8px; + background: #fafbff; + + .ios-empty-icon { + font-size: 22px; + color: #818cf8; + } + + .ios-empty-title { + font-size: 12.5px; + font-weight: 700; + color: $text-main; + margin-bottom: 3px; + } + + .ios-empty-desc { + font-size: 11.5px; + color: $text-secondary; + line-height: 1.5; + } + } + + .ios-sim-panel { + display: flex; + flex-direction: column; + gap: 10px; + padding: 12px; + border: 1px solid #e0e7ff; + border-radius: 8px; + background: #f8faff; + + .ios-sim-header { + display: flex; + align-items: center; + gap: 10px; + + .ios-sim-icon { + font-size: 20px; + color: #4f46e5; + } + + .ios-sim-header-text { + display: flex; + flex-direction: column; + } + + .ios-sim-title { + font-size: 12.5px; + font-weight: 700; + color: $text-main; + } + + .ios-sim-desc { + font-size: 11px; + color: $text-secondary; + } + } + + } .connection-methods-nav { display: grid; @@ -4056,6 +4200,55 @@ $text-light: #94a3b8; .bar-left { display: flex; align-items: center; + gap: 8px; + + .target-device-chip { + display: inline-flex; + align-items: center; + gap: 6px; + height: 30px; + padding: 0 10px; + background: #ffffff; + border: 1px solid $card-border; + border-radius: 15px; + font-family: inherit; + font-size: 11.5px; + font-weight: 600; + color: $text-secondary; + cursor: pointer; + transition: all 0.15s ease; + + .target-icon { + font-size: 15px; + } + + .target-name { + max-width: 160px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .target-plat { + font-size: 9px; + font-weight: 700; + padding: 1px 5px; + border-radius: 4px; + background: #f1f5f9; + color: $text-muted; + letter-spacing: 0.3px; + } + + &.is-ios { + border-color: #c7d2fe; + color: #4338ca; + + .target-plat { + background: #eef2ff; + color: #4f46e5; + } + } + } .model-toggle-capsule { display: inline-flex; @@ -4993,7 +5186,6 @@ $text-light: #94a3b8; } } -} // ============================================================================= // KEYFRAME ANIMATIONS diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index e189b9ce5..b4d98f3d7 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -371,6 +371,21 @@ export class HomeComponent implements OnInit, OnDestroy { // Device information public activeDevice = computed(() => this.systemService.activeDevice()); public connectedDevices = computed(() => this.systemService.connectedDevices()); + public selectedDeviceSerial = computed(() => this.systemService.selectedDeviceSerial()); + public iosDevices = computed(() => this.systemService.iosDevices()); + public selectedIosDevice = computed(() => this.systemService.selectedIosDevice()); + public iosProbe = computed(() => this.systemService.iosProbe()); + public androidDevices = computed( + () => this.connectedDevices().filter(d => d.platform !== 'ios') + ); + // Raw ADB state for the Android tab — isDeviceReady also counts an iOS pick. + public androidDeviceReady = computed(() => this.adbProbe()?.status === 'pass'); + public selectedTarget = computed( + () => this.systemService.selectedIosDevice() ?? this.activeDevice() + ); + // Device guide platform split: Android (ADB) vs iOS Simulator + public deviceGuidePlatform = signal<'android' | 'ios'>('android'); + private deviceGuidePlatformAutoSet = false; public installedAvds = computed(() => this.systemService.installedAvds()); public emulatorPath = computed(() => this.systemService.emulatorPath()); public isEmulatorInPath = computed(() => this.systemService.isEmulatorInPath()); @@ -533,6 +548,19 @@ export class HomeComponent implements OnInit, OnDestroy { this.ocrKeyInput.set(ocrKey); } }); + // Default the device guide to whichever platform actually has devices. + effect(() => { + if (this.deviceGuidePlatformAutoSet) return; + if (this.systemService.selectedIosDevice()) { + this.deviceGuidePlatform.set('ios'); + this.deviceGuidePlatformAutoSet = true; + } else if (this.iosDevices().length > 0 && this.androidDevices().length === 0) { + this.deviceGuidePlatform.set('ios'); + this.deviceGuidePlatformAutoSet = true; + } else if (this.androidDevices().length > 0) { + this.deviceGuidePlatformAutoSet = true; + } + }); } ngOnInit(): void { @@ -1008,8 +1036,9 @@ export class HomeComponent implements OnInit, OnDestroy { this.systemService.dismissEmulatorStatus().subscribe(); } - public selectTargetDevice(serial: string): void { - this.systemService.selectDevice(serial).subscribe(); + public selectTargetDevice(dev: DeviceInfo): void { + this.deviceGuidePlatform.set(dev.platform === 'ios' ? 'ios' : 'android'); + this.systemService.selectDevice(dev.serial, dev.platform ?? 'android').subscribe(); } public getEmulatorCommand(avdName: string): string { diff --git a/apps/showcase_ui/src/app/services/agent.service.spec.ts b/apps/showcase_ui/src/app/services/agent.service.spec.ts index c17e334d4..980d065b3 100644 --- a/apps/showcase_ui/src/app/services/agent.service.spec.ts +++ b/apps/showcase_ui/src/app/services/agent.service.spec.ts @@ -180,6 +180,53 @@ describe('AgentService live LLM retry timeline', () => { expect(selectSpy).toHaveBeenCalledWith('new-session', false); }); + it('sends platform and device_serial when an iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { + selectedIosDevice: () => ({ serial: 'UDID-1', platform: 'ios' }) + }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.platform).toBe('ios'); + expect(postedPayload.device_serial).toBe('UDID-1'); + }); + + it('keeps the Android payload unchanged when no iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { selectedIosDevice: () => null }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.goal).toBe('test goal'); + expect('platform' in postedPayload).toBeFalse(); + expect('device_serial' in postedPayload).toBeFalse(); + }); + it('keeps the paused state when the backend says there is nothing to resume', () => { const service = createServiceWithoutPolling(); (service as any).http = { post: () => of({ status: 'not_paused' }) }; diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index b063c99b8..2ee14c602 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -23,6 +23,7 @@ import { ProTuningDefaults, ProTuningOptions } from '../core/models/pro-tuning.m import { StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE, PersistedCheckerStream, StreamSegment } from '../core/models/stream.model'; import { extractStepReplayFrames } from '../utils/action-formatter.util'; import { persistedStreamToSegments } from '../utils/stream-aggregator.util'; +import { SystemService } from './system.service'; export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice }; const SESSION_CACHE_KEY = 'artemis.sessions.v1'; @@ -70,6 +71,7 @@ interface SessionVideoResponse { export class AgentService { private http = inject(HttpClient); private zone = inject(NgZone); + private systemService = inject(SystemService); private activePauseCardKey: string | null = null; // Signals to expose state to components @@ -158,7 +160,8 @@ export class AgentService { start_time: at.acquired_at ? (new Date(at.acquired_at).getTime() / 1000) : (Date.now() / 1000), status: 'running', model_info: this.activeModel() || undefined, - device_serial: at.device_id || null + device_serial: at.device_id || null, + platform: at.platform || null }; sessionMap.set(sid, newSession); this.activeSessionTracking.set(sid, newSession); @@ -430,6 +433,13 @@ export class AgentService { }; this.pendingStartupProgress.set([submittedEvent]); const payload: any = { goal, profile }; + const iosDevice = this.systemService?.selectedIosDevice?.() ?? null; + if (iosDevice) { + // iOS submissions carry the simulator UDID + platform explicitly; + // Android keeps resolving through the ADB probe target. + payload.platform = 'ios'; + payload.device_serial = iosDevice.serial; + } if (expectedOutput && expectedOutput.trim()) { payload.expected_output = expectedOutput.trim(); } @@ -1546,7 +1556,8 @@ export class AgentService { initial_goal: item.goal || '', start_time: item.start_time || item.created_at || (Date.now() / 1000 + index), status: item.status || 'pending', - device_serial: item.device_serial || item.device_id || null + device_serial: item.device_serial || item.device_id || null, + platform: item.platform || null }; } return { diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts index bd2c6d51c..f75ac47fe 100644 --- a/apps/showcase_ui/src/app/services/system.service.spec.ts +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -145,6 +145,331 @@ describe('SystemService readiness polling', () => { expect(service.adbServerStatus()).toBeNull(); }); + it('merges iOS simulators into the device list with platform tags', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(60), + probes: [ + { + id: 'android_adb', + category: 'device', + title: 'Android Device', + status: 'pass', + is_blocker: true, + summary: 'Device Ready', + description: '', + metadata: { + devices: [ + { + serial: 'emulator-5554', + state: 'device', + model: 'Pixel_8', + product: 'sdk', + android_version: '14', + screen_resolution: '1080x2400', + is_locked: false, + is_emulator: true + } + ] + }, + actions: [] + }, + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + }, + { + udid: 'AAAA1111-2222-3333-4444-555566667777', + name: 'iPhone 17', + state: 'Shutdown', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-26-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + const devices = service.connectedDevices(); + expect(devices.length).toBe(3); + expect(devices[0].platform).toBe('android'); + const booted = devices.find(d => d.serial === 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2'); + expect(booted?.platform).toBe('ios'); + expect(booted?.model).toBe('iPhone 18 Pro'); + expect(booted?.state).toBe('device'); + expect(booted?.product).toBe('iOS 27.0'); + expect(booted?.is_emulator).toBeTrue(); + const shutdown = devices.find(d => d.serial === 'AAAA1111-2222-3333-4444-555566667777'); + expect(shutdown?.state).toBe('Shutdown'); + }); + + it('selects an iOS simulator by posting its platform and tracks it locally', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(70), + probes: [ + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'UDID-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + service.selectDevice('UDID-1', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'UDID-1', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'UDID-1', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('UDID-1'); + expect(service.selectedDeviceSerial()).toBe('UDID-1'); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('clears the iOS selection when an Android device is selected instead', () => { + service.selectedIosDevice.set({ + serial: 'UDID-1', + state: 'device', + model: 'iPhone', + product: 'iOS 27.0', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios' + }); + + service.selectDevice('emulator-5554', 'android').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'emulator-5554', platform: 'android' }); + request.flush({ status: 'success', selected_serial: 'emulator-5554', report: report(80) }); + + expect(service.selectedIosDevice()).toBeNull(); + }); + + const iosProbeWith = (metadata: any) => ({ + id: 'ios_simulators', + category: 'device', + title: 'iOS Devices', + status: 'pass', + is_blocker: false, + summary: 'iOS ready', + description: '', + metadata, + actions: [] + }); + + const envProbe = (id: string, status = 'pass') => ({ + id, + category: 'runtime', + title: id, + status, + is_blocker: true, + summary: status, + description: '', + metadata: {}, + actions: [] + }); + + it('merges paired physical iOS devices alongside simulators', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(90), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ], + connected_physical_devices: [ + { + udid: 'PHYS-1', + name: 'Office iPhone', + os_version: '27.1', + product_type: 'iPhone17,2' + }, + { name: 'No UDID — skipped', os_version: '27.0' } + ] + }) + ] + }); + + const devices = service.connectedDevices(); + const physical = devices.find(d => d.serial === 'PHYS-1'); + expect(devices.filter(d => d.platform === 'ios').length).toBe(2); + expect(physical?.platform).toBe('ios'); + expect(physical?.is_emulator).toBeFalse(); + expect(physical?.state).toBe('device'); + expect(physical?.product).toBe('iOS 27.1'); + expect(physical?.android_version).toBeNull(); + }); + + it('selects a physical iOS device by serial and posts its platform', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(91), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + connected_physical_devices: [ + { udid: 'PHYS-9', name: 'Office iPhone', os_version: '27.0' } + ] + }) + ] + }); + + service.selectDevice('PHYS-9', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'PHYS-9', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'PHYS-9', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('PHYS-9'); + expect(service.selectedIosDevice()?.is_emulator).toBeFalse(); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('reverts the optimistic iOS selection when the server rejects it', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(92), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'r', + isAvailable: true + } + ] + }) + ] + }); + + service.selectDevice('SIM-2', 'ios').subscribe({ error: () => {} }); + expect(service.selectedIosDevice()?.serial).toBe('SIM-2'); // optimistic + http + .expectOne('/api/system/devices/select') + .flush('boom', { status: 500, statusText: 'Internal Server Error' }); + expect(service.selectedIosDevice()).toBeNull(); + }); + + it('gates iOS readiness on macOS + Xcode 27, not the ADB toolchain', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(93), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-3', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + expect(service.isEnvironmentReady()).toBeFalse(); // no iOS selection yet + + service.selectDevice('SIM-3', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeTrue(); + }); + + it('keeps an iOS selection not-ready when Xcode is too old', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(94), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + iosProbeWith({ + xcode_27_or_newer: false, + simulators: [ + { udid: 'SIM-4', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-4', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeFalse(); + }); + + it('restores ADB gates when an Android device is selected', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(95), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + { ...envProbe('toolchain'), is_blocker: false }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-5', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-5', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.isEnvironmentReady()).toBeTrue(); + + service.selectDevice('emulator-5554', 'android').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.selectedIosDevice()).toBeNull(); + expect(service.isEnvironmentReady()).toBeFalse(); + }); + it('restores the standard local ADB server explicitly', () => { service.useLocalAdbServer(true).subscribe(); diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index b69697d9f..af1e916a3 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -56,6 +56,7 @@ export class SystemService { public configProbe = computed(() => this.probes().find(p => p.id === 'system_config') || null); public toolchainProbe = computed(() => this.probes().find(p => p.id === 'toolchain') || null); public adbProbe = computed(() => this.probes().find(p => p.id === 'android_adb') || null); + public iosProbe = computed(() => this.probes().find(p => p.id === 'ios_simulators') || null); public llmProbe = computed(() => this.probes().find(p => p.id === 'gemini_api_key' || p.id === 'llm_api_key') || null); public geminiProbe = computed(() => this.llmProbe()); public ocrProbe = computed(() => this.probes().find(p => p.id === 'vision_ocr_key' || p.id === 'ocr_api_key') || null); @@ -64,10 +65,16 @@ export class SystemService { public isEnvironmentReady = computed(() => { const py = this.pythonProbe(); const cfg = this.configProbe(); - const adb = this.adbProbe(); - const tc = this.toolchainProbe(); const pyOk = py?.status === 'pass'; const cfgOk = cfg?.status === 'pass'; + if (this.selectedIosDevice() != null) { + // An iOS target replaces the ADB toolchain gates: it needs macOS with + // Xcode 27+, never an ADB installation. + const iosOk = this.iosProbe()?.metadata?.['xcode_27_or_newer'] === true; + return pyOk && cfgOk && this.osType() === 'darwin' && iosOk; + } + const adb = this.adbProbe(); + const tc = this.toolchainProbe(); const adbInstalled = adb?.metadata?.['installed'] ?? (adb?.status !== 'fail'); const tcOk = tc?.status === 'pass' || tc?.is_blocker === false; return pyOk && cfgOk && adbInstalled && tcOk; @@ -85,7 +92,8 @@ export class SystemService { public isDeviceReady = computed(() => { const adb = this.adbProbe(); - return adb?.status === 'pass'; + // A selected iOS simulator satisfies the device requirement on its own. + return adb?.status === 'pass' || this.selectedIosDevice() != null; }); // Step-level counting matching the 3-step onboarding guide @@ -122,15 +130,64 @@ export class SystemService { } - // Device list from metadata + // Device list from metadata: Android devices plus available iOS simulators. public connectedDevices = computed(() => { - const meta = this.adbProbe()?.metadata; - if (meta && Array.isArray(meta['devices'])) { - return meta['devices'] as DeviceInfo[]; + const devices: DeviceInfo[] = []; + const adbMeta = this.adbProbe()?.metadata; + if (adbMeta && Array.isArray(adbMeta['devices'])) { + for (const d of adbMeta['devices'] as DeviceInfo[]) { + devices.push({ ...d, platform: d.platform ?? 'android' }); + } } - return []; + const iosMeta = this.iosProbe()?.metadata; + if (iosMeta && Array.isArray(iosMeta['simulators'])) { + for (const s of iosMeta['simulators'] as any[]) { + if (!s?.udid) continue; + devices.push({ + serial: s.udid, + state: s.state === 'Booted' ? 'device' : (s.state || 'unknown'), + model: s.name ?? null, + product: String(s.runtime ?? '') + .replace('com.apple.CoreSimulator.SimRuntime.', '') + .replace(/^([A-Za-z]+)-(.+)$/, (_m: string, name: string, ver: string) => `${name} ${ver.replace(/-/g, '.')}`) || null, + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios', + }); + } + } + if (iosMeta && Array.isArray(iosMeta['connected_physical_devices'])) { + for (const p of iosMeta['connected_physical_devices'] as any[]) { + if (!p?.udid) continue; + devices.push({ + serial: p.udid, + state: 'device', + model: p.name ?? null, + product: p.os_version ? `iOS ${p.os_version}` : 'iOS physical device', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: false, + platform: 'ios', + }); + } + } + return devices; }); + public iosDevices = computed( + () => this.connectedDevices().filter(d => d.platform === 'ios') + ); + + // iOS selection lives client-side: iOS has no ADB probe to retarget, so + // the chosen simulator is carried in the run payload instead. + public selectedIosDevice = signal(null); + public selectedDeviceSerial = computed( + () => this.selectedIosDevice()?.serial ?? this.activeDevice()?.serial ?? null + ); + // Installed local AVD emulators from metadata public installedAvds = computed(() => { const meta = this.adbProbe()?.metadata; @@ -496,13 +553,34 @@ export class SystemService { } /** - * Select a specific connected device serial as active target + * Select a specific connected device or iOS simulator as active target */ - public selectDevice(serial: string): Observable { + public selectDevice(serial: string, platform: 'android' | 'ios' = 'android'): Observable { + // iOS selection is client-side state (the run payload carries the binding); + // apply it optimistically since UDID validation can take seconds and the + // error path below reverts it if the server rejects the pick. + if (platform === 'ios') { + const dev = + this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? null; + this.selectedIosDevice.set(dev); + } this.isLoading.set(true); - return this.http.post('/api/system/devices/select', { serial }).pipe( + return this.http.post('/api/system/devices/select', { serial, platform }).pipe( tap({ next: (res) => { + if (platform === 'ios') { + this.selectedIosDevice.set( + res?.status === 'success' + ? this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? this.selectedIosDevice() + : null + ); + } else if (platform === 'android') { + this.selectedIosDevice.set(null); + } if (res?.report) { this.applyReadinessReport(res.report); } @@ -510,6 +588,9 @@ export class SystemService { }, error: (err) => { console.error('Failed to select active device:', err); + if (platform === 'ios') { + this.selectedIosDevice.set(null); + } this.isLoading.set(false); } }) diff --git a/apps/showcase_ui/tsconfig.json b/apps/showcase_ui/tsconfig.json index 5525117c6..3e01b56f1 100644 --- a/apps/showcase_ui/tsconfig.json +++ b/apps/showcase_ui/tsconfig.json @@ -4,6 +4,7 @@ "compileOnSave": false, "compilerOptions": { "outDir": "./dist/out-tsc", + "rootDir": "./src", "strict": true, "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 52dff2ca2..93cc2b41b 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -67,6 +67,7 @@ ) from artemis.utils.visualization import format_minimal_list_with_elements from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -255,6 +256,8 @@ def _load_prompts() -> dict[str, str]: def probes_enabled(ctx: ArtemisContext) -> bool: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + return False setup = getattr(ctx, "execution_setup", None) return not (setup and getattr(setup, "disable_device_probes", False)) @@ -842,7 +845,7 @@ async def run_final_check( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 13d80317c..b87e7e7f2 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -54,6 +54,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -72,6 +73,8 @@ def __init__(self, ctx: ArtemisContext): self._llm = None def _is_device_online(self) -> bool: + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + return getattr(self.ctx, "_active_driver", None) is not None try: if self.ctx.adb_client is None: return False @@ -111,6 +114,9 @@ def _build_traced_tools(self) -> list: *get_history_tools(self.ctx), get_submit_answer_tool(self.ctx), ] + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + # logcat-backed analysis can never succeed on iOS. + all_tools = [t for t in all_tools if t.name not in ("run_adb_command", "analyze_logs")] if not self.is_device_online: logger.info( "Diagnoser running in OFFLINE environment: stripping adb short" @@ -227,7 +233,7 @@ def _build_initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) diff --git a/artemis/agents/explorer/native_runner.py b/artemis/agents/explorer/native_runner.py index 4b29bd1b0..f9de2d2b4 100644 --- a/artemis/agents/explorer/native_runner.py +++ b/artemis/agents/explorer/native_runner.py @@ -28,6 +28,7 @@ from collections.abc import Awaitable, Callable import json import os +from pathlib import Path import time from typing import TYPE_CHECKING, Any @@ -44,6 +45,7 @@ retry_policy_for, ) from artemis.services.llm import _record_llm_event, _record_llm_retry +from artemis.utils.image_mime import image_mime_type from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -188,12 +190,13 @@ async def get_image_part(file_path: str) -> types.Part: return types.Part( file_data=types.FileData( file_uri=file_ref.uri, - mime_type=file_ref.mime_type or "image/jpeg", + mime_type=file_ref.mime_type + or image_mime_type(Path(file_path).read_bytes()), ) ) with open(file_path, "rb") as f: img_bytes = f.read() - return types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg") + return types.Part.from_bytes(data=img_bytes, mime_type=image_mime_type(img_bytes)) return get_image_part diff --git a/artemis/agents/explorer/screen_index.py b/artemis/agents/explorer/screen_index.py index 837082a2f..a91b0b2ff 100644 --- a/artemis/agents/explorer/screen_index.py +++ b/artemis/agents/explorer/screen_index.py @@ -24,7 +24,7 @@ import re from typing import Any, Literal -from artemis.utils.visualization import parse_bounds +from artemis.utils.visualization import get_hit_point, parse_bounds Bounds = tuple[int, int, int, int] ElementSource = Literal["xml", "ocr"] @@ -84,6 +84,8 @@ class ScreenElement: @property def center(self) -> tuple[int, int]: + if (point := get_hit_point(self.node)) is not None: + return point left, top, right, bottom = self.bounds return (left + right) // 2, (top + bottom) // 2 @@ -173,8 +175,12 @@ def add(element: ScreenElement) -> None: for node in fused_xml or []: if not isinstance(node, dict): continue + if "hit_point" in node and get_hit_point(node, width, height) is None: + node = {key: value for key, value in node.items() if key != "hit_point"} class_name = node.get("class") or node.get("className") - resource_id = node.get("resource-id") or node.get("resourceId") + resource_id = ( + node.get("resource-id") or node.get("resourceId") or node.get("resource_id") + ) interactive = any(_is_truthy_attr(node.get(key)) for key in INTERACTION_KEYS) for ocr in node.get("ocr_elements") or []: diff --git a/artemis/agents/explorer/universal_runner.py b/artemis/agents/explorer/universal_runner.py index c5b5bc89a..48f0dbe2c 100644 --- a/artemis/agents/explorer/universal_runner.py +++ b/artemis/agents/explorer/universal_runner.py @@ -36,6 +36,7 @@ from artemis.graph.state import State from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -172,7 +173,7 @@ def _build_universal_messages( }, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] @@ -266,9 +267,7 @@ def _build_tool_images_message(self, images: list[tuple[str, str]]) -> HumanMess continue if tool_name not in tool_names: tool_names.append(tool_name) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if not blocks: return None header = { diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 3ea3f6865..12b689eaf 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -95,6 +95,7 @@ parse_swipe_parameters, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -103,6 +104,22 @@ " action tool or 'report_task_status'." ) +_REPEATED_THOUGHT_NOTICE = ( + "You returned the same response again without calling a tool. Do not repeat" + " reasoning — call an action tool or 'report_task_status' now." +) + +#: Consecutive identical tool-less responses that mark a deterministic loop: +#: at temperature zero a healthy model never repeats verbatim, so this many +#: identical replies in a row mean the agent is stuck rather than reasoning. +MAX_IDENTICAL_THOUGHT_TURNS = 3 + +#: Consecutive tool-less turns regardless of content. Paraphrased repetition +#: (the same plan re-stated with different wording) evades the identical-text +#: guard, so a plain streak cap is the backstop — healthy runs interleave +#: occasional thinking turns with actions, never this many in a row. +MAX_SILENT_TURN_STREAK = 8 + _FINAL_TURN_WARNING = "[WARNING] This is your final turn; only 'report_task_status' is available." @@ -296,7 +313,11 @@ def _render_system_prompt(self, tools_declaration: list) -> str: prompt_path = Path(__file__).parent / "flash_runner.md" prompt_template = prompt_path.read_text(encoding="utf-8") available_tools = frozenset(t.name for t in tools_declaration) - return Template(prompt_template).render(goal=self.goal, available_tools=available_tools) + from artemis.agents.platform_guidance import device_action_guidance + + return device_action_guidance(self.ctx) + Template(prompt_template).render( + goal=self.goal, available_tools=available_tools + ) # ------------------------------------------------------------------ # Per-turn helpers (observe / think) @@ -369,9 +390,7 @@ def _build_tail( if img_bytes: img_b64 = base64.b64encode(img_bytes).decode("utf-8") blocks.append({"type": "text", "text": "--- Current Screenshot ---"}) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if xml_list: blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"}) ephemeral: list[int] = [] @@ -1056,6 +1075,9 @@ async def run(self, state: State) -> dict: current_xml_list = xml_list previous_turn: _TurnRecord | None = None pending_notices: list[str] = [] + last_thought = "" + identical_thoughts = 0 + silent_turns = 0 while limit is None or turns < limit: turns += 1 @@ -1131,12 +1153,45 @@ async def run(self, state: State) -> dict: if is_final: final_report = {"status": "failed", "explanation": raw_text} break - pending_notices.append(_NO_TOOL_CALL_NOTICE) + thought = " ".join((raw_text or native_text).split()) or "" + if thought == last_thought: + identical_thoughts += 1 + else: + identical_thoughts = 0 + last_thought = thought + silent_turns += 1 + if identical_thoughts + 1 >= MAX_IDENTICAL_THOUGHT_TURNS: + final_report = { + "status": "failed", + "explanation": ( + f"The model returned the identical response" + f" {identical_thoughts + 1} turns in a row without calling" + " a tool; the run is stalled rather than reasoning." + f" Last response: {raw_text[:200]}" + ), + } + break + if silent_turns >= MAX_SILENT_TURN_STREAK: + final_report = { + "status": "failed", + "explanation": ( + f"The model produced {silent_turns} consecutive turns" + " without calling a tool; the run is stalled rather" + f" than reasoning. Last response: {raw_text[:200]}" + ), + } + break + pending_notices.append( + _REPEATED_THOUGHT_NOTICE if identical_thoughts else _NO_TOOL_CALL_NOTICE + ) ledger.stage_turn(messages[turn_base:]) previous_turn = _TurnRecord() continue # Process tool calls + last_thought = "" + identical_thoughts = 0 + silent_turns = 0 turn = _TurnRecord() ( final_report_from_calls, diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 5b8a3cf98..c577717dd 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -41,6 +41,7 @@ from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -430,7 +431,7 @@ async def _attempt(self, key: JobKey) -> bool: } ) content_blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}} + {"type": "image_url", "image_url": {"url": image_data_uri(b64_pre)}} ) if post_bytes: @@ -449,7 +450,7 @@ async def _attempt(self, key: JobKey) -> bool: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_post}"}, + "image_url": {"url": image_data_uri(b64_post)}, } ) diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 4ce526984..2ee22724b 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -35,6 +35,7 @@ from artemis.services.llm import get_llm from artemis.utils.python_executor import PythonExecutor from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -118,7 +119,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: {"type": "text", "text": "Here is the target image. Begin writing your code."}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] ), @@ -186,9 +187,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: image_blocks.append( { "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{b64_data}" - }, + "image_url": {"url": image_data_uri(b64_data)}, } ) image_blocks.append({"type": "text", "text": f"\n{label}\n"}) diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index b2acb4f5f..dae8f8e82 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -26,6 +26,7 @@ from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -107,7 +108,7 @@ async def _run_object_detection( image_bytes: bytes | str | Path | None = None, queries: list[str] | None = None, templates: list[str] | None = None, - mime_type: str = "image/jpeg", + mime_type: str | None = None, global_timeout: float = 30.0, image_path: str | Path | None = None, ) -> dict: @@ -120,6 +121,7 @@ async def _run_object_detection( image_data = Path(target_img).read_bytes() else: image_data = target_img + mime_type = mime_type or image_mime_type(image_data) queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d24536270..4514a2b2c 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -72,7 +72,15 @@ } # Bare key names press_key accepts (case-insensitive, optional KEYCODE_ prefix). -SUPPORTED_PRESS_KEYS = ("ENTER", "BACK", "HOME", "APP_SWITCH") +SUPPORTED_PRESS_KEYS = ( + "ENTER", + "BACK", + "HOME", + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN", +) from artemis.agents.operator.prompts import ( OPERATOR_MAX_TOOL_ITERATIONS, @@ -919,7 +927,9 @@ async def __call__(self, state: State): # The analyzer is always available: the "output truncated, use # analyze_task_output" hint arrives mid-turn, and tools are bound once per # turn, so a conditional mount would be one turn late. - if not any(t.name == "analyze_task_output" for t in all_tools): + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) != "ios" and not any( + t.name == "analyze_task_output" for t in all_tools + ): all_tools.append(analyze_task_output_wrapper.tool_fn_getter(self.ctx)) traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools] diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de234..d74659d0e 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -40,6 +40,7 @@ from artemis.agents.prompt_assembly import render_tool_enum, resolve_available from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER +from artemis.utils.image_mime import image_data_uri @lru_cache(maxsize=1) @@ -203,6 +204,8 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]: # video_recording_tools_enabled); the prompt must not advertise it when the # tool is not actually available this run. available = set(OPERATOR_PROMPT_TOOLSET) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + available.difference_update({"run_adb_command", "manage_task", "analyze_task_output"}) setup = getattr(ctx, "execution_setup", None) if not (setup and getattr(setup, "video_recording_tools_enabled", False)): available.discard("video_analyzer") @@ -316,6 +319,9 @@ def render_transcript_static_system( available = resolve_operator_prompt_tools(ctx) static_template = apply_operator_prompt_contract(static_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + static_template = device_action_guidance(ctx) + static_template return Template(static_template).render( initial_goal=state.initial_goal, subgoals_status="", @@ -354,6 +360,9 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte available = resolve_operator_prompt_tools(ctx) prompt_template = apply_operator_prompt_contract(prompt_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + prompt_template = device_action_guidance(ctx) + prompt_template plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.") @@ -384,7 +393,7 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte builder.add_human_content( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 9b75b427f..88259b63d 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -46,6 +46,7 @@ from artemis.utils.task_tree import get_active_subgoal_hashes from third_party.mobile_use.utils.logger import get_logger from pydantic import BaseModel +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -274,7 +275,7 @@ def _initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 02be5c7c4..e33745057 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -54,6 +54,7 @@ from artemis.utils.cython_compat import CyFunctionDetector from third_party.mobile_use.utils.decorators import agent_lifecycle_logging from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -279,7 +280,7 @@ async def __call__(self, state: State): human_message_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) messages = [ diff --git a/artemis/agents/platform_guidance.py b/artemis/agents/platform_guidance.py new file mode 100644 index 000000000..9e191b408 --- /dev/null +++ b/artemis/agents/platform_guidance.py @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared platform constraints for the Flash and Pro device-action prompts.""" + + +def device_action_guidance(ctx) -> str: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) != "ios": + return "" + return ( + "Target platform: iOS. For input_text, explicitly set clear_exist=false; " + "type into an empty field or at its existing cursor. After typing into a URL " + "or search field, submit it with press_key(enter) or by tapping the " + "keyboard's Go/Search button; typing alone does not navigate. Whole-field " + "clearing, Android keycodes, Back/Delete keys, ADB commands, and app " + "locking are unavailable. iOS has no Back button: never call press_key " + "with BACK; return to the previous screen with the app's back chevron " + "(top-left) instead. press_key supports enter, home, power, volume_up, " + "volume_down, and app_switch. manage_app accepts installed iOS display " + "names or bundle identifiers. Action coordinates use the screenshot's " + "normalized 0-1000 space.\n\n" + ) diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 80a542fb8..5cf32a4c8 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -36,6 +36,7 @@ from artemis.services.llm import acomplete_structured from artemis.utils import visualization from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -120,12 +121,12 @@ def _build_messages( {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"}, + "image_url": {"url": image_data_uri(orig_b64)}, }, {"type": "text", "text": "[Image 2 (Current State)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{live_b64}"}, + "image_url": {"url": image_data_uri(live_b64)}, }, ] if state: diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index fba8dd751..b2d30b46f 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -45,6 +45,7 @@ READ_NOTE_DOCSTRING, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -79,7 +80,7 @@ def to_langchain_tool_message(self) -> ToolMessage: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + "image_url": {"url": image_data_uri(b64_img)}, } ) return ToolMessage( diff --git a/artemis/agents/video_analyzer/native_coordinator.py b/artemis/agents/video_analyzer/native_coordinator.py index 6cfab85d9..b0994f89a 100644 --- a/artemis/agents/video_analyzer/native_coordinator.py +++ b/artemis/agents/video_analyzer/native_coordinator.py @@ -34,6 +34,7 @@ from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -171,10 +172,11 @@ def _append_ledger_diff(analyzer, contents: list, last_ledger_index: int) -> int image_parts.append( types.Part.from_text(text=f"PROOF: {Path(screenshot_file).name}") ) + image_bytes = Path(screenshot_file).read_bytes() image_parts.append( types.Part.from_bytes( - data=Path(screenshot_file).read_bytes(), - mime_type="image/jpeg", + data=image_bytes, + mime_type=image_mime_type(image_bytes), ) ) lines.append(line) diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 96ade3fe7..4eaf8fe28 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -42,6 +42,7 @@ ) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -159,7 +160,7 @@ async def _build_universal_user_blocks( user_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_str}"}, + "image_url": {"url": image_data_uri(b64_str)}, } ) diff --git a/artemis/context.py b/artemis/context.py index 57360c8c8..c97386882 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -198,6 +198,17 @@ class ArtemisContext(DeviceClientAccessors, BaseModel): async def __aenter__(self) -> ArtemisContext: return self + async def disconnect_driver(self) -> None: + """Release the cached driver, including partially initialized transports.""" + driver = self._active_driver + self._active_driver = None + self._mobile_controller = None + if driver is not None: + try: + await driver.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"Device driver disconnect failed: {exc}", exc_info=True) + async def __aexit__(self, exc_type, exc_val, exc_tb): # Close the in-process action session before draining background tasks so its # owner task exits cleanly rather than being cancelled below. @@ -283,6 +294,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception as exc: logger.debug(f"DataEngine shutdown failed; skipped: {exc}", exc_info=True) + await self.disconnect_driver() + from artemis.data_engine.engine import DataEngine diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index d0dc1d035..82a74ed44 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -19,12 +19,13 @@ import subprocess import tempfile import time -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 from artemis.config.paths import get_temp_dir from artemis.context import ArtemisContext from artemis.drivers.factory import get_driver +from artemis.drivers.ios.recording import IosRecordingSession from artemis.utils.video import ( ANDROID_RECORDING_SEGMENT_SECONDS, await_scrcpy_first_frame, @@ -48,6 +49,9 @@ set_active_session, ) +if TYPE_CHECKING: + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + logger = get_logger(__name__) @@ -56,6 +60,18 @@ def __init__(self, ctx: ArtemisContext): super().__init__(ctx, get_driver(ctx)) self._segment_cache: dict[tuple[str, int, float, float], VideoRecordingResult] = {} + async def open_url(self, url: str) -> bool: + if self.ctx.device.mobile_platform == "ios": + return await self._driver.open_url(url) + return await super().open_url(url) + + async def erase_text(self, nb_chars: int | None = None) -> bool: + if self.ctx.device.mobile_platform == "ios": + # No native clear/Backspace key exists; the driver raises + # NotImplementedError for unsupported clear operations. + return await self._driver.input_text("", clear_existing=True) + return await super().erase_text(nb_chars) + @staticmethod async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process: kwargs: dict[str, Any] = { @@ -131,6 +147,21 @@ async def extract_segment_metadata( ) return cached_res + if isinstance(session, IosRecordingSession): + result = await self._extract_ios_segment(session, start_time, end_time, output_path) + if ( + cache_key is not None + and result.success + and result.video_path + and result.video_path.exists() + ): + # Sealing bumps the generation; key the entry under the + # post-seal generation a repeated lookup will observe. + self._segment_cache[ + (cache_key[0], session.generation, cache_key[2], cache_key[3]) + ] = result + return result + try: mkv_path = session.local_video_path if not mkv_path: @@ -397,8 +428,10 @@ async def start_video_recording( output_dir: Path | None = None, max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> VideoRecordingResult: - """Start screen recording on Android device using scrcpy.""" + """Start screen recording using the platform's native capture.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return await self._start_ios_recording(output_dir, max_duration_seconds) device_id = self._get_device_id() # Check mock driver first @@ -517,8 +550,10 @@ async def start_video_recording( return abort_recording(device_id, "start", e) async def stop_video_recording(self) -> VideoRecordingResult: - """Stop scrcpy recording and return the converted MP4 video file.""" + """Stop recording and return the finalized MP4 video file.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return await self._stop_ios_recording() device_id = self._get_device_id() # Check mock driver first @@ -645,5 +680,242 @@ async def stop_video_recording(self) -> VideoRecordingResult: self._record_recording_failure(session, str(e)) return abort_recording(device_id, "stop", e) + def _ios_recording_driver(self) -> "XcodeSimulatorDriver | None": + """The active driver when it supports native iOS capture, else ``None``.""" + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + return self._driver if isinstance(self._driver, XcodeSimulatorDriver) else None + + async def _start_ios_recording( + self, output_dir: Path | None, max_duration_seconds: int + ) -> VideoRecordingResult: + """Start native capture through the iOS driver (simctl or devicectl).""" + device_id = self._get_device_id() + if already_active := recording_already_active(device_id): + return already_active + driver = self._ios_recording_driver() + if driver is None: + return VideoRecordingResult( + success=False, + message="iOS recording requires the iOS driver.", + ) + try: + await driver.start_video_recording( + output_dir, max_duration_seconds=max_duration_seconds + ) + except Exception as exc: + session = driver.recording_session + if session is None: + session = IosRecordingSession( + video_id=uuid4(), + device_id=device_id, + start_time=time.time(), + local_video_path=(Path(output_dir) / "recording.mov" if output_dir else None), + is_active=False, + ) + if session.data_engine_start_time is None and self.ctx and self.ctx.data_engine: + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time or session.start_time + ) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path or "", + start_time=session.start_time, + ) + self._record_recording_failure(session, f"iOS recording failed to start: {exc}") + return VideoRecordingResult( + success=False, message=f"iOS recording failed to start: {exc}" + ) + session = driver.recording_session + if session is None: + return VideoRecordingResult(success=False, message="iOS recorder returned no session") + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time + if (self.ctx and self.ctx.data_engine) + else session.start_time + ) + set_active_session(device_id, session) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path, + start_time=session.start_time, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording started on {device_id}", + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:active", + warning=warning, + ) + + async def _stop_ios_recording(self) -> VideoRecordingResult: + """Finalize the driver's recorder and report the sealed manifest.""" + device_id = self._get_device_id() + driver = self._ios_recording_driver() + session = get_active_session(device_id) + if session is None: + session = driver.recording_session if driver is not None else None + if session is None or session.video_id is None: + return no_active_recording(device_id) + if driver is None: + message = "iOS recording stop requires the Xcode simulator driver." + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + async def _finalize() -> str | None: + session.is_active = False + return await driver.stop_video_recording() + + try: + final_path = await asyncio.shield(_finalize()) + except asyncio.CancelledError: + raise + except Exception as exc: + self._record_recording_failure(session, f"iOS recording stop failed: {exc}") + return VideoRecordingResult(success=False, message=f"iOS recording stop failed: {exc}") + finally: + remove_active_session(device_id) + + missing = [ + record + for record in session.segments + if not ( + record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ) + ] + if final_path is None or not session.segments or missing: + message = "; ".join(session.errors) or "iOS recording finalized with no valid segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + final_video_path = Path(final_path) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_stop( + video_id=session.video_id, + device_id=device_id, + local_video_path=final_video_path, + start_time=session.start_time, + end_time=session.start_time + session.sealed_until, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording stopped, saved {len(session.segments)} segments", + video_path=final_video_path, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:ready", + warning=warning, + ) + + async def _extract_ios_segment( + self, + session: IosRecordingSession, + start_time: float, + end_time: float | None, + output_path: Path | None, + ) -> VideoRecordingResult: + """Render a clip from sealed iOS MP4 segments only. + + The open .mov is never handed to FFmpeg: the driver seals the segment + at the requested recording-relative end first, then finalized segments + render through the shared timeline renderer. Recorder restarts leave + their gaps black rather than stretching adjacent frames. + """ + device_id = self._get_device_id() + anchor = session.data_engine_start_time + shift = (session.start_time - anchor) if anchor is not None else 0.0 + recording_start = float(start_time) - shift + recording_end = float(end_time) - shift if end_time is not None else None + + driver = self._ios_recording_driver() + if session.is_active and recording_end is not None and driver is not None: + await driver.seal_recording_segment(through_time=recording_end) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + + warning = None + available_end = session.sealed_until + if recording_end is None or recording_end > available_end: + if recording_end is not None: + warning = ( + "Requested end clipped to latest sealed iOS capture " + f"({available_end + shift:.1f}s of session time)" + ) + recording_end = available_end + + records = [dict(record) for record in session.segments] + first_capture = min((float(record["start"]) for record in records), default=None) + if first_capture is not None and recording_start < first_capture: + leading = "Requested start preceded the first captured frame; clipped to capture start" + warning = f"{warning}; {leading}" if warning else leading + recording_start = first_capture + + if not records or recording_end <= recording_start: + message = "Requested range does not overlap sealed iOS recording" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + timeline = [ + { + "path": str(record["output_path"]), + "start": float(record["start"]), + "end": float(record["end"]), + } + for record in records + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and float(record["end"]) > recording_start + and float(record["start"]) < recording_end + ] + timeline.sort(key=lambda segment: segment["start"]) + if not timeline: + message = "No finalized iOS MP4 segments overlap the requested range" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + trim_output_path = ( + Path(output_path) + if output_path is not None + else Path(tempfile.mkdtemp(prefix="video_trimmed_", dir=get_temp_dir("trimmed_videos"))) + / "segment.mp4" + ) + success = await render_timeline_clip( + timeline, recording_start, recording_end, trim_output_path + ) + if not success or not trim_output_path.exists(): + message = "Failed to render clip from sealed iOS segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + actual_start = recording_start + shift + actual_end = recording_end + shift + file_size_mb = trim_output_path.stat().st_size / (1024 * 1024) + return VideoRecordingResult( + success=True, + message=( + f"iOS clip rendered for session range {actual_start:.1f}s to {actual_end:.1f}s" + ), + video_path=trim_output_path, + file_size_mb=round(file_size_mb, 2), + duration_seconds=round(recording_end - recording_start, 2), + actual_start_relative_time=actual_start, + warning=warning, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=(f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"), + ) + async def cleanup(self) -> None: await self._driver.disconnect() diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 3c43c9cf4..00be7f8de 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -56,7 +56,9 @@ import time from typing import Any, TypeVar +from artemis.runtime.device_target import normalize_device_platform from artemis.runtime.device_lock import DeviceExecutionLock +from artemis.runtime.ios_observation import observe_ios_controller from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -96,8 +98,55 @@ def _uiautomator_fix(serial: str | None) -> list[str]: ] -def fix_for_error(error: str | None, serial: str | None) -> list[str]: +def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: + """iOS repair steps (simctl/Xcode for simulators, devicectl/WDA for hardware).""" + if not error: + return [] + s = serial or "" + text = (error or "").lower() + if "busy" in text or "another task" in text: + return [ + "Wait for the running task to finish, or stop it with mobile_manage_task(action='stop', ...).", + "Then rerun the smoke test.", + ] + if "approv" in text or "authoriz" in text or "permission" in text: + return [ + "Approve Artemis's agent access in Xcode (the first run opens a workspace approval prompt; choose persistent approval).", + "Confirm the permitted agent and workspace under Xcode > Settings > Agentic Tools.", + ] + if "xcode" in text or "macos" in text or "darwin" in text: + return [ + "Install Xcode 27 or later and select it: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ] + if ( + "webdriveragent" in text + or "wda" in text + or "physical" in text + or "pair" in text + or "trust" in text + or "devicectl" in text + or "developer mode" in text + ): + return [ + f"List paired devices: xcrun devicectl list devices; check capture: xcrun devicectl device capture screenshot --device {s} --destination /tmp/ios.png", + "Pair the device (plug in, tap Trust), enable Developer Mode under Settings > Privacy & Security, and ensure WebDriverAgent is installed " + "(build-for-testing via Xcode, or set ARTEMIS_IOS_WDA_URL to a running server).", + ] + if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: + return [ + f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", + "Create a simulator in Xcode > Settings > Platforms if none exist.", + ] + return [ + "Check native capture manually: xcrun simctl io booted screenshot /tmp/ios.png", + 'Verify Xcode MCP access: echo \'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}\' | xcrun mcpbridge', + ] + + +def fix_for_error(error: str | None, serial: str | None, platform: str = "android") -> list[str]: """Map an error string to concrete repair steps (substring match, case-insensitive).""" + if normalize_device_platform(platform, strict=False) == "ios": + return _ios_fix_for_error(error, serial) if not error: return [] text = error.lower() @@ -260,19 +309,31 @@ def _decoded_length(b64: Any) -> int | None: async def smoke_test_device( - device_serial: str | None = None, timeout_seconds: float = 20.0 + device_serial: str | None = None, + timeout_seconds: float = 20.0, + platform: str = "android", ) -> dict[str, Any]: """Observe the device exactly like ``mobile_get_device_state`` and report a verdict. + ``platform="ios"`` exercises the native iOS path (``simctl`` + mcpbridge + for simulators, ``devicectl`` + WebDriverAgent for physical devices) + instead of ADB/UIAutomator. + Never raises. Returns:: {"ok": bool, "serial": str | None, "elapsed_seconds": float, "screenshot_bytes": int | None, "element_count": int | None, - "hierarchy_backend": "helper" | "uiautomator" | None, + "hierarchy_backend": "helper" | "uiautomator" | "xcode" | "wda" | None, "error": str | None, "fix": list[str]} """ started = time.monotonic() - requested_serial = _resolve_requested_serial(device_serial) + is_ios = normalize_device_platform(platform, strict=False) == "ios" + # iOS targets are UDIDs; ADB_DEVICE_SERIAL is Android-only fallback. + requested_serial = ( + (device_serial or os.environ.get("ARTEMIS_DEVICE_ID")) + if is_ios + else _resolve_requested_serial(device_serial) + ) result: dict[str, Any] = { "ok": False, "serial": requested_serial, @@ -289,10 +350,14 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["elapsed_seconds"] = round(time.monotonic() - started, 3) result["error"] = error result["ok"] = error is None - result["fix"] = fix_for_error(cause if cause is not None else error, result["serial"]) + result["fix"] = fix_for_error( + cause if cause is not None else error, result["serial"], platform + ) return result - busy = _find_busy_owner(requested_serial) + # iOS skips this unscoped sweep: observe_ios_controller performs atomic, + # scope-correct lease admission on the resolved UDID itself. + busy = None if is_ios else _find_busy_owner(requested_serial) if busy is not None: key, owner = busy description = getattr(owner, "description", "") or "unknown task" @@ -306,20 +371,33 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any try: from artemis.mcp import adb_server - controller = await _run_in_daemon_thread( - lambda: adb_server._get_controller(device_serial=device_serial), - timeout_seconds, - "controller-init", - ) + if is_ios: + # iOS controller construction enumerates simctl/devicectl lazily; + # offload the sync factory instead of the daemon-thread hop used + # for the blocking Android UIAutomator handshake. + controller = await asyncio.wait_for( + asyncio.to_thread( + adb_server._get_controller, + device_serial=device_serial, + target_platform="ios", + ), + timeout_seconds, + ) + else: + controller = await _run_in_daemon_thread( + lambda: adb_server._get_controller(device_serial=device_serial), + timeout_seconds, + "controller-init", + ) except TimeoutError: - return _finish( - f"UIAutomator/controller initialization did not respond within {timeout_seconds:g}s" - ) + label = "iOS driver" if is_ios else "UIAutomator/controller" + return _finish(f"{label} initialization did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught cause = _describe_failure(exc) - return _finish(f"Failed to initialize Android device controller: {cause}", cause) + target = "iOS device" if is_ios else "Android device" + return _finish(f"Failed to initialize {target} controller: {cause}", cause) controller_ctx = getattr(controller, "ctx", None) device = getattr(controller_ctx, "device", None) @@ -329,13 +407,25 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any remaining = max(0.5, timeout_seconds - (time.monotonic() - started)) try: - device_data = await _run_in_daemon_thread( - lambda: _run_coroutine_blocking(controller.get_screen_data), - remaining, - "screen-data", - ) + if is_ios: + # Runs on the caller's loop: the helper owns lease + connect + + # capture + disconnect + release and is cancellation-safe, unlike + # the Android coroutine which must be bounced onto a private + # thread/loop because UIAutomator blocks the loop thread. + device_data = await asyncio.wait_for(observe_ios_controller(controller), remaining) + else: + + async def _observe() -> Any: + return await controller.get_screen_data() + + device_data = await _run_in_daemon_thread( + lambda: _run_coroutine_blocking(_observe), + remaining, + "screen-data", + ) except TimeoutError: - return _finish(f"UIAutomator/screen capture did not respond within {timeout_seconds:g}s") + label = "iOS session capture" if is_ios else "UIAutomator/screen capture" + return _finish(f"{label} did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught @@ -344,9 +434,24 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None)) result["element_count"] = _count_elements(getattr(device_data, "elements", None)) - from artemis.clients.screen_client_factory import describe_backend + if is_ios: + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + # Report the canonical UDID the observation lease actually resolved. + resolved_serial = getattr(device, "device_id", None) + if isinstance(resolved_serial, str) and resolved_serial: + result["serial"] = resolved_serial + result["hierarchy_backend"] = ( + "wda" + if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) + else "xcode" + ) + else: + from artemis.clients.screen_client_factory import describe_backend - result["hierarchy_backend"] = describe_backend(getattr(controller_ctx, "ui_adb_client", None)) + result["hierarchy_backend"] = describe_backend( + getattr(controller_ctx, "ui_adb_client", None) + ) if result["screenshot_bytes"] is None: return _finish("Screen capture returned no screenshot data") @@ -355,9 +460,8 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any f"Screenshot capture failed (driver returned a {result['screenshot_bytes']}-byte placeholder image)" ) if not result["element_count"]: - return _finish( - "UIAutomator hierarchy dump returned no UI elements (screenshot worked, hierarchy did not)" - ) + backend = "Xcode accessibility tree" if is_ios else "UIAutomator hierarchy dump" + return _finish(f"{backend} returned no UI elements (screenshot worked, hierarchy did not)") return _finish(None) diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index 97f9d20ca..56b93cf83 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -27,6 +27,7 @@ LLMCredentialsProbe, VisionOCRProbe, ) +from artemis.core.diagnostics.probes.ios_probe import IosDeviceProbe from artemis.core.diagnostics.probes.runtime_probe import ( PythonRuntimeProbe, SystemConfigProbe, @@ -67,6 +68,7 @@ def __init__(self): self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() + self._ios_probe = IosDeviceProbe() self._report_cache: SystemReadinessReport | None = None self._report_cache_time = 0.0 self._report_cache_generation = -1 @@ -80,6 +82,7 @@ def __init__(self): self.register_probe(self._credentials_probe) self.register_probe(self._ocr_probe) self.register_probe(self._adb_probe) + self.register_probe(self._ios_probe) def register_probe(self, probe: BaseProbe) -> None: """Register a new diagnostic probe.""" diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py new file mode 100644 index 000000000..bb4c78658 --- /dev/null +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -0,0 +1,170 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Device Readiness Probe. + +Informational only (``is_blocker = False``): iOS support is opt-in per task, +so a missing Xcode toolchain must never block the Android default path. The +probe reports whether this host could run an iOS task today — macOS, Xcode +27+, and at least one available simulator or paired+connected physical +device. +""" + +import asyncio + +from artemis.core.diagnostics.probes.base import BaseProbe +from artemis.core.diagnostics.schema import ( + ProbeAction, + ProbeCategory, + ProbeResult, + ProbeStatus, +) +from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices, + list_ios_simulators, + parse_xcode_version, + physical_ios_ready, + run_xcrun, + simctl_available, +) + + +class IosDeviceProbe(BaseProbe): + """Reports iOS automation capability: Xcode 27+, simulators, physical.""" + + @property + def probe_id(self) -> str: + # Historical id kept stable: persisted reports reference it. + return "ios_simulators" + + @property + def category(self) -> ProbeCategory: + return ProbeCategory.DEVICE + + @property + def is_blocker(self) -> bool: + return False + + async def probe(self) -> ProbeResult: + if not simctl_available(): + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.SKIPPED, + is_blocker=self.is_blocker, + summary="Unavailable", + description=( + "iOS automation requires macOS with Xcode 27+; " + "this host cannot enumerate iOS devices." + ), + metadata={"platform_supported": False}, + ) + + xcode_ok = False + xcode_version: str | None = None + try: + raw = await asyncio.wait_for(run_xcrun("xcodebuild", "-version"), timeout=15.0) + xcode_version = parse_xcode_version(raw) + if xcode_version: + xcode_ok = int(xcode_version.split(".")[0]) >= 27 + except (OSError, RuntimeError, TimeoutError): + xcode_version = None + + simulators = await list_ios_simulators() + booted = [d for d in (simulators or []) if d.get("state") == "Booted"] + core_devices = await list_core_devices() + physical = [d for d in (core_devices or []) if is_physical_ios(d)] + connected = [d for d in physical if physical_ios_ready(d)] + metadata = { + "platform_supported": True, + "xcode_version": xcode_version, + "xcode_27_or_newer": xcode_ok, + "simulator_count": len(simulators or []), + "booted_udids": [d.get("udid") for d in booted], + "simulators": simulators or [], + "physical_device_count": len(physical), + "connected_physical_devices": [ + { + "udid": d.get("udid"), + "name": d.get("name"), + "os_version": d.get("os_version"), + "product_type": d.get("product_type"), + } + for d in connected + ], + } + + if not xcode_ok: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="Xcode 27+ Required", + description=( + "Native iOS automation requires Xcode 27 or later" + + (f" (found {xcode_version})." if xcode_version else "; none detected.") + ), + metadata=metadata, + actions=[ + ProbeAction( + action_type="hint", + label="Select Xcode 27", + payload="sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ), + ], + ) + + if not simulators and not connected: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="No Devices", + description=( + "Xcode is installed but no available iOS simulators or connected " + "physical devices were found. Create a simulator in Xcode > " + "Settings > Platforms, or attach a paired iPhone/iPad." + ), + metadata=metadata, + ) + + parts = [] + if booted: + parts.append(f"{len(booted)} simulator(s) booted") + elif simulators: + parts.append(f"{len(simulators)} simulator(s) available") + if connected: + parts.append(f"{len(connected)} physical device(s) connected") + summary = ", ".join(parts) + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.PASS, + is_blocker=self.is_blocker, + summary=summary, + description=( + f"Xcode {xcode_version}: {summary}. " + "Submit a task with platform='ios' and a device UDID to use them." + ), + metadata=metadata, + ) + + +# Backward-compatible name from when the probe only covered simulators. +IosSimulatorProbe = IosDeviceProbe diff --git a/artemis/core/diagnostics/readiness.py b/artemis/core/diagnostics/readiness.py index 8f8f08533..03c2e8fb0 100644 --- a/artemis/core/diagnostics/readiness.py +++ b/artemis/core/diagnostics/readiness.py @@ -41,8 +41,9 @@ "integration_host": 2, "gemini_api_key": 3, "android_adb": 4, - "toolchain": 5, - "vision_ocr_key": 6, + "ios_simulators": 5, + "toolchain": 6, + "vision_ocr_key": 7, } @@ -70,7 +71,7 @@ def base_verdict(results: Sequence[ProbeResult]) -> Verdict: blockers = [r for r in results if r.is_blocker] if not blockers or any(r.status is not ProbeStatus.PASS for r in blockers): return "blocked" - if any(r.status is not ProbeStatus.PASS for r in results): + if any(r.status not in (ProbeStatus.PASS, ProbeStatus.SKIPPED) for r in results): return "degraded" return "ready" diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index d6283a1c6..e997b8131 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -31,8 +31,57 @@ logger = get_logger(__name__) +def ios_driver_class(device_id: str | None) -> type[BaseDeviceDriver]: + """Pick the driver class for an iOS serial by device reality. + + A serial resolving to a CoreDevice physical entry maps to + ``PhysicalIosDriver``; simulators, ``booted``, and unknown serials keep + ``XcodeSimulatorDriver`` so its existing validation and error paths apply. + """ + from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + if device_id and device_id.strip().lower() != BOOTED_SIMULATOR_ID: + from artemis.drivers.ios.discovery import ( + device_matches_identifier, + find_physical_ios_device_sync, + list_ios_simulators_sync, + ) + + simulators = list_ios_simulators_sync() + if simulators is None or not any( + device_matches_identifier(device, device_id) for device in simulators + ): + if find_physical_ios_device_sync(device_id) is not None: + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + return PhysicalIosDriver + return XcodeSimulatorDriver + + +def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: + """Instantiate the simulator or physical-device driver for an iOS target.""" + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS support is local only; cloud mode targets Android.") + config = getattr(ctx, "agent_config", None) + workspace_path = getattr(config, "ios_workspace_path", None) if config is not None else None + driver_class = ios_driver_class(ctx.device.device_id) + return driver_class( + device_id=ctx.device.device_id, + width=ctx.device.device_width, + height=ctx.device.device_height, + workspace_path=workspace_path, + ) + + def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: """Instantiates the appropriate BaseDeviceDriver based on the runtime context.""" + mobile_platform = getattr(ctx.device, "mobile_platform", "android") + if mobile_platform == "ios": + return _create_ios_driver(ctx) + if mobile_platform not in {"android", "mock"}: + raise ValueError(f"Unsupported mobile platform: {mobile_platform}") + # 1. Cloud mode check. Cloud devices are reached through the gateway's # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there # because the Accessibility Helper needs a local adb forward. diff --git a/artemis/drivers/ios/__init__.py b/artemis/drivers/ios/__init__.py new file mode 100644 index 000000000..40da1ece4 --- /dev/null +++ b/artemis/drivers/ios/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS drivers: Xcode simulators and paired physical devices (loaded only when iOS is selected).""" diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py new file mode 100644 index 000000000..105db8d5f --- /dev/null +++ b/artemis/drivers/ios/bridge.py @@ -0,0 +1,228 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Task-safe access to Xcode's MCP bridge using Artemis's existing MCP SDK. + +The SDK handles initialization, notifications and response IDs. A single owner +task enters and exits the SDK's AnyIO contexts, and serializes device commands. +A timeout retires the bridge; a possibly executed input is never retried. +""" + +import asyncio +import json +import os +from pathlib import Path +import sys +from typing import Any + +import anyio +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import get_default_environment, stdio_client +from mcp.shared.exceptions import McpError + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +_APPROVAL_MESSAGES = ( + "agent isn't approved", + "agent is not approved", + "waiting for the user to approve", +) + + +class XcodeApprovalRequiredError(RuntimeError): + """Xcode refused a tool because this interpreter lacks agent/folder approval.""" + + def __init__( + self, + tool_name: str, + native_message: str, + *, + workspace_path: str | Path | None = None, + ): + super().__init__(f"Xcode tool {tool_name} failed: {native_message}") + self.tool_name = tool_name + self.native_message = native_message + self.workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + + @property + def guidance(self) -> str: + """Human steps to approve this interpreter; never executes them.""" + return xcode_approval_guidance(self.workspace_path) + + +def xcode_approval_guidance(workspace_path: str | Path | None = None) -> str: + """Human-facing Xcode approval steps; printing is the only action taken.""" + workspace = str(workspace_path) if workspace_path else "not supplied" + return ( + "Xcode approval is required for the Python interpreter running Artemis.\n" + f"Interpreter: {Path(sys.executable).resolve()}\n" + f"Workspace: {workspace}\n\n" + "Open the Xcode MCP menu bar icon and approve this interpreter and the selected " + "project folder. Choose Always Allow or persistent approval there if offered. " + "Alternatively, inspect the relevant pending request IDs and approve only those " + "entries from your own terminal:\n" + " xcrun mcp-server status\n" + " sudo xcrun mcp-server approve --always\n\n" + "Artemis never runs sudo or changes global access settings. After approval, " + "rerun the task. Interpreter upgrades, a different project folder, or expiring " + "grants can require approval again.\n" + "If Xcode has not recorded a request yet, rerun with " + "--platform ios --ios-workspace /absolute/path/YourApp.xcodeproj " + "(or pass workspace_path to for_ios_device())." + ) + + +_ERRORS = ( + OSError, + ValueError, + RuntimeError, + McpError, + ExceptionGroup, + anyio.EndOfStream, + anyio.ClosedResourceError, + anyio.BrokenResourceError, +) + + +class XcodeBridge: + """Own one native bridge subprocess for one simulator driver.""" + + def __init__(self, timeout_seconds: float = 120.0): + self.timeout_seconds = timeout_seconds + self._owner: asyncio.Task[None] | None = None + self._ready: asyncio.Future[set[str]] | None = None + self._requests: asyncio.Queue[Any] = asyncio.Queue() + self._active: asyncio.Future[Any] | None = None + self._start_lock = asyncio.Lock() + self.tools: set[str] = set() + + @property + def connected(self) -> bool: + return self._owner is not None and not self._owner.done() + + async def start(self) -> None: + async with self._start_lock: + if self._owner is not None and not self._owner.done(): + return + self._ready = asyncio.get_running_loop().create_future() + self._owner = asyncio.create_task(self._run(), name="artemis-xcode-bridge") + try: + self.tools = await asyncio.wait_for( + asyncio.shield(self._ready), self.timeout_seconds + ) + except (*_ERRORS, TimeoutError, asyncio.CancelledError): + await self.close() + raise + + async def _run(self) -> None: + try: + # mcpbridge only needs a clean base env plus the Xcode pinning + # vars — do not forward the full environment (API keys) to a + # long-lived child process. + env = get_default_environment() + for key in ("DEVELOPER_DIR", "MCP_XCODE_PID"): + if value := os.environ.get(key): + env[key] = value + params = StdioServerParameters(command="xcrun", args=["mcpbridge"], env=env) + async with stdio_client(params) as (reader, writer): + async with ClientSession(reader, writer) as session: + await session.initialize() + names: set[str] = set() + cursor = None + while True: + page = await session.list_tools(cursor=cursor) + names.update(tool.name for tool in page.tools) + cursor = page.nextCursor + if not cursor: + break + if self._ready is not None and not self._ready.done(): + self._ready.set_result(names) + while True: + request = await self._requests.get() + if request is None: + return + future, name, arguments = request + if future.cancelled(): + continue + self._active = future + result = await session.call_tool(name, arguments) + if not future.done(): + future.set_result(result) + self._active = None + except _ERRORS as exc: + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(RuntimeError(f"Cannot connect to Xcode MCP: {exc}")) + if self._active is not None and not self._active.done(): + self._active.set_exception(RuntimeError(f"Xcode MCP connection failed: {exc}")) + finally: + error = RuntimeError( + "Xcode MCP bridge has closed. Reconnect the driver before retrying." + ) + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(error) + if self._active is not None and not self._active.done(): + self._active.set_exception(error) + self._active = None + while not self._requests.empty(): + request = self._requests.get_nowait() + if request is not None and not request[0].done(): + request[0].set_exception(error) + + async def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + if self._owner is None or self._owner.done(): + raise RuntimeError("Xcode MCP bridge is not connected.") + future = asyncio.get_running_loop().create_future() + self._requests.put_nowait((future, name, arguments)) + try: + result = await asyncio.wait_for(future, self.timeout_seconds) + except (TimeoutError, asyncio.CancelledError): + await self.close() + raise + if result.isError: + message = " ".join(block.text for block in result.content if block.type == "text") + if any(marker in message.casefold() for marker in _APPROVAL_MESSAGES): + raise XcodeApprovalRequiredError(name, message) + raise RuntimeError(f"Xcode tool {name} failed: {message}") + if isinstance(result.structuredContent, dict): + return result.structuredContent + # Xcode also supports older clients with JSON text content. + for block in result.content: + if block.type == "text": + try: + data = json.loads(block.text) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + raise RuntimeError(f"Xcode tool {name} returned no structured result.") + + async def close(self) -> None: + owner, self._owner = self._owner, None + if owner is None: + return + if not owner.done(): + owner.cancel() + try: + await owner + except asyncio.CancelledError: + pass + except Exception as exc: + # Owner failures already propagate to requesters via futures; a + # close() must never mask the error its caller is handling. + logger.debug(f"Xcode bridge owner exited with an error: {exc}") + if self._ready is not None and self._ready.done() and not self._ready.cancelled(): + self._ready.exception() # consume an initialization error on cancellation + self.tools.clear() diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py new file mode 100644 index 000000000..1860a599f --- /dev/null +++ b/artemis/drivers/ios/discovery.py @@ -0,0 +1,439 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared iOS device enumeration: ``simctl`` for simulators, ``devicectl`` for physical. + +Both the Xcode driver (target validation) and the runtime pools (discovery, +explicit-serial validation, auto-selection) read the same +``xcrun simctl list devices --json`` output through this module so parsing +stays in one place; paired physical hardware is enumerated through +``xcrun devicectl list devices``. Every function fails closed to +``None``/``[]`` on missing tooling so Android-only hosts never see iOS errors. +""" + +import asyncio +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time +from typing import Any + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +SIMCTL_QUERY_TIMEOUT = 15.0 + +# ``simctl`` wildcard that resolves to whichever simulator is currently +# booted; also the public sentinel callers pass to mean "the booted one". +BOOTED_SIMULATOR_ID = "booted" + +# ``simctl list devices`` / ``devicectl list devices`` take seconds on a busy +# host and every iOS consumer (readiness probe, device pool validation, +# /api/devices) enumerates them. Share one result briefly so polling UIs do +# not spawn back-to-back tool invocations. +_ENUMERATION_CACHE_TTL = 10.0 + + +class _TtlCache: + """Monotonic-TTL cache holding one enumeration result (or nothing).""" + + def __init__(self) -> None: + self._devices: list[dict[str, Any]] | None = None + self._time = 0.0 + + def clear(self) -> None: + self._devices = None + self._time = 0.0 + + def store(self, devices: list[dict[str, Any]]) -> list[dict[str, Any]]: + self._devices = devices + self._time = time.monotonic() + return devices + + def get(self) -> list[dict[str, Any]] | None: + if self._devices is None: + return None + if time.monotonic() - self._time > _ENUMERATION_CACHE_TTL: + return None + return self._devices + + +_simulator_cache = _TtlCache() +_core_device_cache = _TtlCache() + + +def clear_ios_simulator_cache() -> None: + """Drop the cached enumeration (e.g. after the driver boots a simulator).""" + _simulator_cache.clear() + + +async def reap_process(process: asyncio.subprocess.Process) -> None: + """Kill a still-running child and drain its pipes; never raises.""" + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() + + +async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: + """Run argv directly, reporting native errors and reaping cancelled children.""" + process = await asyncio.create_subprocess_exec( + "xcrun", + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except (TimeoutError, asyncio.CancelledError): + await reap_process(process) + raise + if process.returncode: + raise RuntimeError( + f"xcrun {' '.join(arguments[:3])} failed: {stderr.decode(errors='replace').strip()}" + ) + return stdout + + +async def plist_to_json(payload: bytes, timeout: float = 30.0) -> bytes: + """Convert an OpenStep/XML/binary plist payload to JSON via ``plutil``. + + ``simctl listapps`` emits OpenStep (ASCII) plists that ``plistlib`` + cannot read; ``plutil -convert json`` accepts every plist flavor. + """ + process = await asyncio.create_subprocess_exec( + "plutil", + "-convert", + "json", + "-o", + "-", + "--", + "-", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(input=payload), timeout) + except (TimeoutError, asyncio.CancelledError): + await reap_process(process) + raise + if process.returncode != 0: + raise RuntimeError(f"plutil conversion failed: {stderr.decode(errors='replace').strip()}") + return stdout + + +def simctl_available() -> bool: + """Whether this host can enumerate iOS simulators at all.""" + return sys.platform == "darwin" and shutil.which("xcrun") is not None + + +def parse_xcode_version(payload: bytes | str) -> str | None: + """Extract the Xcode version string from ``xcodebuild -version`` output.""" + text = payload.decode(errors="replace") if isinstance(payload, bytes) else payload + match = re.search(r"Xcode\s+(\d+(?:\.\d+)*)", text) + return match.group(1) if match else None + + +def parse_simctl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``simctl list devices --json`` into available iOS entries.""" + return [ + { + "udid": device.get("udid"), + "name": device.get("name"), + "state": device.get("state"), + "runtime": runtime, + } + for runtime, entries in payload.get("devices", {}).items() + if ".iOS-" in runtime + for device in entries + if device.get("isAvailable") and device.get("udid") + ] + + +async def list_ios_simulators( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """All available iOS simulators, or ``None`` when enumeration fails. + + Successful enumerations are cached for ``_ENUMERATION_CACHE_TTL`` seconds; + failures are never cached so callers retry against live simctl. + """ + if not simctl_available(): + return None + if not force_refresh: + cached = _simulator_cache.get() + if cached is not None: + return cached + try: + raw = await run_xcrun("simctl", "list", "devices", "--json", timeout=SIMCTL_QUERY_TIMEOUT) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + try: + return _simulator_cache.store(parse_simctl_devices(json.loads(raw))) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None + + +def list_ios_simulators_sync( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """Synchronous variant for non-async callers (e.g. replay device lists).""" + if not simctl_available(): + return None + if not force_refresh: + cached = _simulator_cache.get() + if cached is not None: + return cached + try: + completed = subprocess.run( + ["xcrun", "simctl", "list", "devices", "--json"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=SIMCTL_QUERY_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return _simulator_cache.store(parse_simctl_devices(json.loads(completed.stdout))) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None + + +# --- CoreDevice (physical iPhone/iPad) enumeration ------------------------- + +DEVICECTL_ENUMERATE_TIMEOUT = 20.0 + + +async def devicectl_screenshot(device_id: str, destination: Path, timeout: float) -> None: + """Capture one PNG frame from a paired device via ``devicectl``.""" + await run_xcrun( + "devicectl", + "device", + "capture", + "screenshot", + "--device", + device_id, + "--destination", + str(destination), + timeout=timeout, + ) + + +def _device_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read a property across devicectl's current and deprecated JSON shapes. + + Xcode marks ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + deprecated in favor of a nested ``properties`` dictionary; accept both. + Sections may be present-but-null, so guard every hop. + """ + parent = device.get(section) + value = parent.get(key) if isinstance(parent, dict) else None + if value is not None: + return value + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None + + +def _modern_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read ``properties.
.`` from Xcode 27's devicectl shape. + + The modern ``properties`` map uses short section names (``hardware``, + ``software``, ``state``, ``connection``) rather than the deprecated + ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + spellings. Sections may be present-but-null, so guard every hop. + """ + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None + + +def _modern_os_version(device: dict[str, Any]) -> Any: + """``properties.software.osVersionNumber`` may carry a stringValue map.""" + version = _modern_property(device, "software", "osVersionNumber") + if isinstance(version, dict): + return version.get("stringValue") or version.get("string") + return version + + +def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``devicectl list devices --json-output -`` into device entries.""" + devices = [] + result = payload.get("result") + if not isinstance(result, dict): + return devices + entries = result.get("devices") + if not isinstance(entries, list): + return devices + for device in entries: + if not isinstance(device, dict): + continue + udid = ( + _device_property(device, "hardwareProperties", "udid") + or _modern_property(device, "hardware", "udid") + or device.get("identifier") + ) + if not udid: + continue + devices.append( + { + "udid": udid, + "name": _device_property(device, "deviceProperties", "name") + or _modern_property(device, "state", "name"), + "os_version": _device_property(device, "deviceProperties", "osVersionNumber") + or _modern_os_version(device), + "platform": _device_property(device, "hardwareProperties", "platform") + or _modern_property(device, "hardware", "platform"), + "reality": _device_property(device, "hardwareProperties", "reality") + or _modern_property(device, "hardware", "reality"), + "product_type": _device_property(device, "hardwareProperties", "productType") + or _modern_property(device, "hardware", "productType"), + "connection_state": _device_property(device, "connectionProperties", "tunnelState") + or _modern_property(device, "connection", "state"), + "pairing_state": _device_property(device, "connectionProperties", "pairingState") + or _modern_property(device, "connection", "pairingState"), + "visibility": device.get("visibilityClass") + or _modern_property(device, "state", "visibilityClass"), + } + ) + return devices + + +async def list_core_devices( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """All devices known to CoreDevice (physical and simulated), cached briefly.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _core_device_cache.get() + if cached is not None: + return cached + try: + raw = await run_xcrun( + "devicectl", + "list", + "devices", + "--json-output", + "-", + timeout=DEVICECTL_ENUMERATE_TIMEOUT, + ) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + try: + return _core_device_cache.store(parse_devicectl_devices(json.loads(raw))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def list_core_devices_sync( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """Synchronous variant for the driver factory and other sync callers.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _core_device_cache.get() + if cached is not None: + return cached + try: + completed = subprocess.run( + ["xcrun", "devicectl", "list", "devices", "--json-output", "-"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=DEVICECTL_ENUMERATE_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return _core_device_cache.store(parse_devicectl_devices(json.loads(completed.stdout))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def is_physical_ios(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a physical iPhone/iPad (not a simulator).""" + return device.get("platform") in ("iOS", "iPadOS") and device.get("reality") == "physical" + + +def physical_ios_ready(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a paired, reachable physical iOS device. + + ``connection_state`` (CoreDevice ``tunnelState``) may be absent on paired + USB devices running older iOS — an absent value is acceptable; + ``"disconnected"`` is not. Every consumer (pool validators, readiness + probe, replay manager, driver) must agree on this rule. + """ + if not is_physical_ios(device): + return False + if device.get("pairing_state") != "paired": + return False + return device.get("connection_state") in ("connected", None) + + +def device_matches_identifier(device: dict[str, Any], identifier: str) -> bool: + """Match a device entry by case-insensitive UDID or exact name.""" + if not identifier: + return False + return ( + str(device.get("udid") or "").lower() == identifier.lower() + or device.get("name") == identifier + ) + + +def find_physical_ios_device_sync(identifier: str) -> dict[str, Any] | None: + """Match a physical iOS device by UDID or exact name, or ``None``. + + Raises ``ValueError`` when the identifier matches more than one device — + silently picking the first would drive an arbitrary phone. + """ + devices = list_core_devices_sync() + if devices is None or not identifier: + return None + matches = [ + device + for device in devices + if is_physical_ios(device) and device_matches_identifier(device, identifier) + ] + if len(matches) > 1: + udids = sorted(str(device.get("udid") or "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {identifier!r} " + f"({', '.join(udids)}); use the device UDID instead." + ) + return matches[0] if matches else None diff --git a/artemis/drivers/ios/hierarchy.py b/artemis/drivers/ios/hierarchy.py new file mode 100644 index 000000000..b2d12fdfd --- /dev/null +++ b/artemis/drivers/ios/hierarchy.py @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert Xcode's textual accessibility frames and hit points to screen pixels.""" + +import re +from typing import Any + +_NUMBER = r"(-?\d+(?:\.\d+)?)" +_FRAME = re.compile( + r"\{\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\},\s*\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\}\}" +) +_HIT = re.compile(r"hitPoint:\s*\{\s*" + _NUMBER + r",\s*" + _NUMBER + r"\}") +_BUNDLE = re.compile(r"Application bundle identifier:\s*(\S+)") + + +def application_bundle(hierarchy: str) -> str | None: + bundles = set(_BUNDLE.findall(hierarchy)) + return next(iter(bundles)) if len(bundles) == 1 else None + + +def pixel_element( + *, text: str, resource_id: str, class_name: str, left: int, top: int, right: int, bottom: int +) -> dict[str, Any]: + """Element dict in the shared Android-shape format used by both iOS parsers.""" + return { + "text": text, + "resource_id": resource_id, + "class": class_name, + "bounds": f"[{left},{top}][{right},{bottom}]", + "parsed_bounds": {"left": left, "top": top, "right": right, "bottom": bottom}, + } + + +def parse_hierarchy( + hierarchy: str, width: int, height: int +) -> tuple[list[dict[str, Any]], tuple[float, float]]: + """Determine scale from the screen window; never assume a Retina factor. + + UIKit hierarchy geometry is in the native interaction coordinate space. + Artemis observations/actions use full screenshot pixels. Missing or rotated + geometry is an error rather than silently tapping with an unverified scale. + """ + lines = hierarchy.splitlines() + windows = [] + for line in lines: + match = _FRAME.search(line) + if match and re.match(r"\s*(?:UI)?Window\b", line): + x, y, w, h = map(float, match.groups()) + if x == 0 and y == 0 and w > 0 and h > 0: + windows.append((w, h)) + matching = [(w, h) for w, h in windows if abs(width / w - height / h) < 0.05] + if windows and not matching: + raise RuntimeError( + "Xcode hierarchy has no screen window matching the screenshot. Recapture before interacting." + ) + # Xcode 27 exports the full screenshot at logical screen dimensions. An + # inaccessible custom canvas can have no AX Window; visual targeting still + # works in this documented native coordinate space. + w, h = max(matching, key=lambda size: size[0] * size[1]) if matching else (width, height) + scale = width / w, height / h + elements = [] + for line in lines: + match = _FRAME.search(line) + if not match: + continue + x, y, w, h = map(float, match.groups()) + if w <= 0 or h <= 0: + continue + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right <= 0 or bottom <= 0 or left >= width or top >= height: + continue + label = re.search(r"label:\s*'((?:\\.|[^'])*)'", line) + quoted = re.search(r'"([^"\n]*)"', line[: match.start()]) + identifier = re.search(r"identifier:\s*'((?:\\.|[^'])*)'", line) + placeholder = re.search(r"placeholderValue:\s*'((?:\\.|[^'])*)'", line) + value = re.search(r"value:\s*'((?:\\.|[^'])*)'", line) + if value: + value_text = value.group(1) + else: + # Xcode elides long values and drops quotes, e.g. `value: Text...`, + # so read until the next native metadata delimiter verbatim. + unquoted = re.search( + r"\bvalue:\s*(.*?)(?=,\s*(?:Keyboard Focused\b|Selected\b|Disabled\b" + r"|hitPoint:|activationBundleId:|identifier:|label:|placeholderValue:)|$)", + line, + ) + value_text = unquoted.group(1).strip() if unquoted else None + hit = _HIT.search(line) + activation = re.search(r"activationBundleId:\s*(\S+)", line) + text = "" + if label: + text = label.group(1) + elif quoted: + text = quoted.group(1) + elif value_text: + text = value_text + elif placeholder and placeholder.group(1): + text = placeholder.group(1) + element = pixel_element( + text=text, + resource_id=identifier.group(1) if identifier else "", + class_name=line.strip().split(",", 1)[0].split(" ", 1)[0], + left=left, + top=top, + right=right, + bottom=bottom, + ) + if hit: + hx, hy = map(float, hit.groups()) + element["hit_point"] = [round(hx * scale[0]), round(hy * scale[1])] + if activation: + element["activation_bundle_id"] = activation.group(1) + if placeholder: + element["placeholder"] = placeholder.group(1) + if value_text is not None: + element["value"] = value_text + elements.append(element) + return elements, scale diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py new file mode 100644 index 000000000..59328ae82 --- /dev/null +++ b/artemis/drivers/ios/physical_driver.py @@ -0,0 +1,797 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iPhone/iPad support via ``devicectl`` plus WebDriverAgent. + +Lifecycle operations (install, launch, terminate, app list, URL open, +screenshots) run through ``xcrun devicectl`` against a paired device. Xcode's +``DeviceInteraction*`` MCP tools accept simulators only, so UI observation and +input on hardware go through WebDriverAgent (WDA), the XCUITest bridge Appium +uses: the driver finds a signed WDA runner on the device, launches it through +``devicectl``, and talks to its HTTP endpoint over the CoreDevice tunnel, a +LAN address, or a forwarded port. + +Prerequisites surface as actionable errors: the device must appear in +``devicectl list devices`` as ``paired`` and ``connected`` (USB or network), +Developer Mode must be on, and a WebDriverAgent runner must be installed — +set ``ARTEMIS_IOS_WDA_URL`` to reach an existing server directly. +""" + +import asyncio +import base64 +from io import BytesIO +import json +import os +from pathlib import Path +import plistlib +import re +import tempfile +from typing import Any, Literal + +from PIL import Image + +from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection +from artemis.drivers.ios.discovery import ( + BOOTED_SIMULATOR_ID, + device_matches_identifier, + devicectl_screenshot, + is_physical_ios, + list_core_devices, + reap_process, + run_xcrun, +) +from artemis.drivers.ios.physical_recording import PhysicalIosRecorder +from artemis.drivers.ios.wda import ( + WdaClient, + WdaUnavailableError, + parse_wda_elements, + probe_wda, + wda_url_candidates, +) +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.controllers.types import ElementQuery +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session + +logger = get_logger(__name__) + +DEVICECTL_OP_TIMEOUT = 30.0 +DEVICECTL_LAUNCH_TIMEOUT = 60.0 +DEVICECTL_INSTALL_TIMEOUT = 300.0 +WDA_START_TIMEOUT = 45.0 +WDA_RUNNER_PATTERN = re.compile(r"webdriveragent", re.IGNORECASE) +WDA_BUNDLE_ENV = "ARTEMIS_IOS_WDA_BUNDLE_ID" +WDA_XCTESTRUN_ENV = "ARTEMIS_IOS_WDA_XCTESTRUN" + +_WDA_SETUP_HINT = ( + "Physical iOS UI automation needs WebDriverAgent on the device. Build it once with " + "'xcodebuild build-for-testing -project /WebDriverAgent.xcodeproj " + "-scheme WebDriverAgentRunner -destination id= -allowProvisioningUpdates " + "DEVELOPMENT_TEAM=', install the produced WebDriverAgentRunner-Runner.app via " + "'xcrun devicectl device install app', point ARTEMIS_IOS_WDA_XCTESTRUN at the " + "generated .xctestrun for Artemis to host it, or expose a running server through " + "ARTEMIS_IOS_WDA_URL (iproxy, pymobiledevice3, or the device LAN address)." +) + + +class PhysicalIosDriver(XcodeSimulatorDriver): + """One paired physical iOS device driven by devicectl and WebDriverAgent. + + Shares lifecycle conventions with ``XcodeSimulatorDriver`` but replaces the + interaction core entirely: CoreDevice handles discovery and app lifecycle, + WDA supplies hierarchy, screenshots, taps, swipes, text, and keys. The + simulator-only Xcode MCP bridge is never started for a physical target. + """ + + def __init__( + self, + device_id: str, + width: int = 0, + height: int = 0, + *, + workspace_path: str | Path | None = None, + ): + super().__init__( + device_id=device_id, width=width, height=height, workspace_path=workspace_path + ) + self._launched_pids: dict[str, int] = {} + self._wda: WdaClient | None = None + self._wda_runner_pid: int | None = None + self._wda_test_process: asyncio.subprocess.Process | None = None + + # --- Resolution and connection --- + + async def _resolve_device(self) -> dict[str, Any]: + self._validate_workspace() + if not self._device_id.strip() or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + raise ValueError( + "Physical iOS devices require --device-serial ; " + "'booted' only selects simulators. Find UDIDs via 'xcrun devicectl list devices'." + ) + await self._require_ios_host() + devices = await list_core_devices(force_refresh=True) + if devices is None: + raise RuntimeError( + "Could not enumerate physical devices; 'xcrun devicectl list devices' failed." + ) + matches = [ + device + for device in devices + if is_physical_ios(device) and device_matches_identifier(device, self._device_id) + ] + if not matches: + needle = self._device_id.lower() + simulator = [ + device + for device in devices + if device.get("udid", "").lower() == needle and device.get("reality") == "simulated" + ] + if simulator: + raise ValueError( + f"UDID {self._device_id} is an iOS Simulator, not a physical device." + ) + raise ValueError( + f"No paired physical iOS device matches {self._device_id!r}. " + "Attach it, trust this Mac, and verify 'xcrun devicectl list devices'." + ) + if len(matches) > 1: + udids = sorted(device.get("udid", "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {self._device_id!r} " + f"({', '.join(udids)}); target the device UDID instead." + ) + candidate = matches[0] + self._device_id = candidate["udid"] + return candidate + + async def _prepare_device(self, candidate: dict[str, Any]) -> None: + """Verify the paired device is reachable instead of booting it.""" + if candidate.get("pairing_state") != "paired": + raise RuntimeError( + f"iOS device {self._device_id} is not paired. Connect it and tap Trust." + ) + connection_state = candidate.get("connection_state") + # Older iOS versions over USB expose no CoreDevice tunnelState — an + # absent value on a paired device is acceptable; "disconnected" is not. + if connection_state is not None and connection_state != "connected": + raise RuntimeError( + f"iOS device {self._device_id} ({candidate.get('name') or 'unknown'}) is not " + "connected. Attach it over USB or ensure network pairing is reachable; " + "on iOS 16+ also enable Developer Mode in Settings > Privacy & Security." + ) + + async def connect(self) -> None: + async with self._connect_lock: + if self._session_key: + return + candidate = await self._resolve_device() + await self._prepare_device(candidate) + connected = False + try: + self._wda = await self._ensure_wda() + # The reachable WDA endpoint must belong to THIS device before + # we open a session or send input. WDA reports the product + # family name ("iPhone"), not the personalized devicectl name + # ("Dana's iPhone"), and uuid is identifierForVendor — so a + # *specific* conflicting name is the wrong-device signal; + # generic family names are accepted. + info = await self._wda.device_info() + expected_name = (candidate.get("name") or "").strip() + wda_name = str(info.get("name") or "").strip() + generic_names = { + "iphone", + "ipad", + "ipod touch", + "apple watch", + "apple tv", + } + name_conflict = ( + expected_name + and wda_name + and wda_name.lower() not in generic_names + and wda_name != expected_name + ) + if info.get("isSimulator") is not False or name_conflict: + raise RuntimeError( + f"The WebDriverAgent at {self._wda.base_url} does not report " + f"the selected physical device {self._device_id} " + f"(expected name {expected_name!r}, got " + f"{info.get('name')!r}, isSimulator={info.get('isSimulator')!r}). " + "Point ARTEMIS_IOS_WDA_URL at a WDA server running on the " + "selected device." + ) + # A session auto-created by a runner WE launched is ours to + # adopt; an endpoint discovered via env/probe keeps the + # foreign-session refusal. + runner_owned = ( + self._wda_runner_pid is not None or self._wda_test_process is not None + ) + self._session_key = await self._wda.open_session(adopt_existing=runner_owned) + await self.get_screen_data(skip_settling=True) + connected = True + finally: + # Any incomplete setup must release a half-started WDA + # runner, xcodebuild session, or server-side session. + if not connected: + try: + await self.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + logger.warning( + "Could not release the WDA session after a connection failure: " + f"{cleanup_error}", + ) + + async def disconnect(self) -> None: + try: + if self._recorder is not None: + session = self._recorder.session + if session is not None: + # A failed (non-active) session still owns frames and + # conversions — finalize it too so nothing is lost. + try: + await self._recorder.stop() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.error( + f"Physical iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + async with self._operation_lock: + self._session_key = None + self._scale = None + self._launched_pids.clear() + client, self._wda = self._wda, None + runner_pid, self._wda_runner_pid = self._wda_runner_pid, None + test_process, self._wda_test_process = self._wda_test_process, None + # Each cleanup step is isolated so a failure in one never skips + # the owned WDA session, the xcodebuild child, or the runner pid. + if client is not None: + try: + await client.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA session close failed during disconnect: {exc}") + if test_process is not None and test_process.returncode is None: + try: + test_process.terminate() + await asyncio.wait_for(test_process.wait(), timeout=10.0) + except (TimeoutError, OSError): + # Graceful terminate failed — reap (kill + drain) the + # xcodebuild child so it cannot outlive the driver. + try: + await reap_process(test_process) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA xcodebuild reap failed: {exc}") + if runner_pid is not None: + try: + await self._terminate_pid(runner_pid) + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"WDA runner termination failed: {exc}") + + def _require_connected(self) -> None: + if not self._session_key or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + raise RuntimeError("Connect the physical iOS driver before interacting.") + + def _require_wda(self) -> WdaClient: + """Bound WDA client for input paths; fails clearly when detached.""" + self._require_connected() + client = self._wda + if client is None: + raise RuntimeError("Connect the physical iOS driver before interacting.") + return client + + async def _ensure_wda(self) -> WdaClient: + """Attach to a reachable WDA server, starting one when possible. + + Order: probe known endpoints, then try launching an installed runner + app directly (newer WDA builds self-host), then — when + ``ARTEMIS_IOS_WDA_XCTESTRUN`` names a ``.xctestrun`` file from a + ``build-for-testing`` — spawn ``xcodebuild test-without-building``, + which is the canonical way to boot the XCTest session that hosts + WDA's HTTP server. + """ + candidates = wda_url_candidates(tunnel_ip=await self._tunnel_ip()) + client = await probe_wda(candidates) + if client is not None: + return client + xctestrun = os.environ.get(WDA_XCTESTRUN_ENV) + runner = await self._wda_runner_bundle() + if xctestrun: + await self._start_xctest_session(xctestrun) + elif runner is not None: + try: + self._wda_runner_pid = await self._launch_bundle(runner, terminate_existing=False) + logger.info( + f"Launched WebDriverAgent runner {runner} on {self._device_id}; " + "waiting for its HTTP server" + ) + except (OSError, RuntimeError, TimeoutError) as exc: + # A stale runner may already be hosting the server — the + # probe loop below still gets a chance to attach. + logger.debug(f"WDA runner launch failed ({runner}): {exc}") + if runner is None and not xctestrun: + raise RuntimeError( + f"No WebDriverAgent server answers on {self._device_id} and no WDA " + f"runner is installed. {_WDA_SETUP_HINT}" + ) + deadline = asyncio.get_running_loop().time() + WDA_START_TIMEOUT + while True: + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + raise RuntimeError( + f"WebDriverAgent did not answer within {WDA_START_TIMEOUT:.0f}s " + f"on {self._device_id} at {candidates}. If the device is showing " + "a passcode prompt to enable UI Automation, enter it on the " + f"device first. {_WDA_SETUP_HINT}" + ) + client = await probe_wda(candidates, timeout=min(5.0, remaining)) + if client is not None: + logger.info(f"WebDriverAgent attached at {client.base_url} for {self._device_id}") + return client + await asyncio.sleep(min(1.0, remaining)) + + async def _start_xctest_session(self, xctestrun: str) -> None: + """Hold WDA alive through ``xcodebuild test-without-building``.""" + path = Path(xctestrun).expanduser() + if path.suffix != ".xctestrun" or not path.is_file(): + raise RuntimeError( + f"{WDA_XCTESTRUN_ENV} must point at an existing .xctestrun file " + "produced by 'xcodebuild build-for-testing'." + ) + if self._wda_test_process is not None and self._wda_test_process.returncode is None: + return + self._wda_test_process = await asyncio.create_subprocess_exec( + "xcodebuild", + "test-without-building", + "-xctestrun", + str(path), + "-destination", + f"id={self._device_id}", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + + async def _tunnel_ip(self) -> str | None: + """CoreDevice's managed tunnel address for this device, when present.""" + try: + details = await self._devicectl_json("info", "details") + except (OSError, RuntimeError, TimeoutError, ValueError): + return None + # The tunnel address has moved across Xcode releases: probe every + # observed shape, canonical first. + paths = ( + ((details.get("connectionProperties") or {}), "tunnelIPAddress"), + ( + ((details.get("properties") or {}).get("connection") or {}), + "tunnelIPAddressString", + ), + ((details.get("tunnel") or {}), "ipAddress"), + ) + for section, key in paths: + address = section.get(key) + if isinstance(address, str) and ":" in address: + return address + return None + + async def _wda_runner_bundle(self) -> str | None: + override = os.environ.get(WDA_BUNDLE_ENV) + try: + apps = await self._devicectl_json("info", "apps") + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"Could not list apps while looking for a WDA runner: {exc}") + return override or None + bundles = { + app.get("bundleIdentifier") or "" + for app in apps.get("apps", []) + if isinstance(app, dict) + } + if override: + return override if override in bundles else None + matches = sorted(b for b in bundles if WDA_RUNNER_PATTERN.search(b)) + return matches[0] if matches else None + + async def _launch_bundle(self, bundle: str, terminate_existing: bool = True) -> int | None: + arguments = [ + "devicectl", + "device", + "process", + "launch", + "--device", + self._device_id, + ] + if terminate_existing: + arguments.append("--terminate-existing") + arguments += ["--json-output", "-", bundle] + raw = await run_xcrun(*arguments, timeout=DEVICECTL_LAUNCH_TIMEOUT) + return self._launched_pid(raw) + + async def _devicectl_json(self, *arguments: str) -> dict[str, Any]: + """Run a ``devicectl`` info subcommand and return its ``result`` JSON. + + ``--json-output -`` still writes a human table to stdout for ``info`` + subcommands, so the JSON must go to a scratch file. + """ + with tempfile.TemporaryDirectory(prefix="artemis-devicectl-") as tmp: + target = Path(tmp) / "out.json" + await run_xcrun( + "devicectl", + "device", + *arguments, + "--device", + self._device_id, + "--json-output", + str(target), + timeout=DEVICECTL_OP_TIMEOUT, + ) + payload = json.loads(target.read_text(encoding="utf-8")) + result = payload.get("result") + return result if isinstance(result, dict) else payload + + # --- Observation --- + + async def _screenshot_png(self) -> bytes: + """WDA screenshot first (same framebuffer as the hierarchy); devicectl fallback.""" + if self._wda is not None: + try: + return await self._wda.screenshot_png() + except (RuntimeError, WdaUnavailableError, OSError) as exc: + logger.debug(f"WDA screenshot failed, falling back to devicectl: {exc}") + with tempfile.TemporaryDirectory(prefix="artemis-shot-") as tmp: + target = Path(tmp) / "shot.png" + await devicectl_screenshot(self._device_id, target, timeout=DEVICECTL_OP_TIMEOUT) + data = target.read_bytes() + if not data: + raise RuntimeError("devicectl produced an empty screenshot.") + return data + + async def _capture(self) -> ScreenData: + if self._wda is None: + raise RuntimeError("Connect the physical iOS driver before observing.") + screenshot = await self._screenshot_png() + with Image.open(BytesIO(screenshot)) as image: + self._width, self._height = image.size + win_w, win_h = await self._wda.window_size() + if win_w <= 0 or win_h <= 0: + raise RuntimeError("WebDriverAgent reported an unusable window size.") + if abs(self._width / win_w - self._height / win_h) > 0.05: + self._scale = None + raise RuntimeError( + "The device screenshot and hierarchy disagree on orientation. " + "Capture again after the rotation settles." + ) + scale = (self._width / win_w, self._height / win_h) + if not (0.9 <= scale[0] <= 4.5 and 0.9 <= scale[1] <= 4.5): + # iOS displays render at 1x-3x; a wildly off scale means the WDA + # window is not full-screen (e.g. iPad multitasking) and every + # element bound would be wrong. + self._scale = None + raise RuntimeError( + f"WebDriverAgent window {win_w}x{win_h} does not match the " + f"{self._width}x{self._height} framebuffer (scale {scale}); " + "bring the session app full-screen before interacting." + ) + self._scale = scale + tree = await self._wda.source_json() + elements = parse_wda_elements(tree, self._scale, self._width, self._height) + return ScreenData( + screenshot_bytes=screenshot, + screenshot_base64=base64.b64encode(screenshot).decode("ascii"), + ui_elements=elements, + width=self._width, + height=self._height, + platform="ios", + ) + + # --- Input --- + + async def tap( + self, x: int, y: int, duration_ms: int = 100, times: int = 1, delay_ms: int = 100 + ) -> bool: + if times < 1 or duration_ms < 0 or delay_ms < 0: + raise ValueError("Tap count must be positive and durations nonnegative.") + async with self._operation_lock: + wda = self._require_wda() + await self._capture_unchanged("tapping") + point = self._scaled_point(x, y) + for index in range(times): + await wda.tap(*point, hold_ms=duration_ms) + if index < times - 1: + await asyncio.sleep(delay_ms / 1000) + return True + + async def swipe( + self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 + ) -> bool: + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + wda = self._require_wda() + await self._capture_unchanged("swiping") + start = self._scaled_point(start_x, start_y) + end = self._scaled_point(end_x, end_y) + await wda.swipe(*start, *end, duration_ms) + return True + + async def swipe_direction( + self, + direction: SwipeDirection | Literal["up", "down", "left", "right"], + duration_ms: int = 800, + ) -> bool: + direction = SwipeDirection(direction).value + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + # Rotation-tolerant like the simulator path: recapture and compute + # from the *current* size rather than refusing like tap. + wda = self._require_wda() + await self._capture() + sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) + start = self._scaled_point(sx, sy) + end = self._scaled_point(ex, ey) + await wda.swipe(*start, *end, duration_ms) + return True + + async def input_text(self, text: str, clear_existing: bool = True) -> bool: + if clear_existing: + raise NotImplementedError( + "Physical iOS typing appends to the focused field; clear it through the " + "UI or pass clear_existing=False. This matches the simulator behavior." + ) + async with self._operation_lock: + wda = self._require_wda() + await wda.type_text(text) + return True + + async def press_key(self, key: KeyCode | str | int) -> bool: + key = key.value if isinstance(key, KeyCode) else str(key).lower() + async with self._operation_lock: + wda = self._require_wda() + if key in ("home", "app_switch"): + presses = 2 if key == "app_switch" else 1 + for index in range(presses): + if not await wda.press_button("home"): + if key == "app_switch": + # A backend that cannot press Home cannot + # double-press for app switching — never report + # success from a homescreen fallback. + raise NotImplementedError( + "App switching is unavailable from this WebDriverAgent backend." + ) + await wda.homescreen() + if index < presses - 1: + await asyncio.sleep(0.4) + elif key == "enter": + await wda.type_text("\n") + else: + buttons = { + "volume_up": "volumeUp", + "volume_down": "volumeDown", + "power": "power", + } + if key not in buttons or not await wda.press_button(buttons[key]): + raise NotImplementedError( + f"Key {key!r} is not supported by the physical iOS driver." + ) + return True + + async def tap_element( + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 + ) -> bool: + async with self._operation_lock: + wda = self._require_wda() + data = await self._capture() + element, center, error = await self.find_element(query, data) + if error or element is None or center is None: + return False + point = self._scaled_point(*center) + await wda.tap(*point, hold_ms=duration_ms if long_press else 0) + return True + + # --- App lifecycle via devicectl --- + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + pid = await self._launch_bundle(package_name) + if pid is not None: + self._launched_pids[package_name] = pid + return True + + @staticmethod + def _launched_pid(raw: bytes) -> int | None: + """Best-effort process id from devicectl's launch JSON or text output.""" + try: + payload = json.loads(raw) + except (ValueError, TypeError): + payload = None + if isinstance(payload, dict): + result = payload.get("result") + result = result if isinstance(result, dict) else {} + process = result.get("process") + candidates = [c for c in (process, result, payload) if isinstance(c, dict)] + for candidate in candidates: + pid = candidate.get("processIdentifier") or candidate.get("pid") + if isinstance(pid, int): + return pid + if isinstance(pid, str) and pid.isdigit(): + return int(pid) + match = re.search(r"pid[:= ]+(\d+)", raw.decode(errors="replace")) + return int(match.group(1)) if match else None + + @staticmethod + def _normalize_executable(raw: Any) -> str: + """devicectl reports executables as file:// URLs or paths; normalize.""" + value = raw if isinstance(raw, str) else str(raw or "") + if isinstance(raw, dict): + value = str(raw.get("url") or raw.get("path") or "") + value = value.removeprefix("file://") + return value.removeprefix("/private") or value + + async def _resolve_pid(self, package_name: str) -> int: + """Verified live pid: the executable must sit under the app's own URL. + + A cached launch pid only prioritizes among processes already proven to + belong to the app — a recycled pid must never select a foreign process. + """ + # ``--include-default-apps`` keeps system apps (Safari, Settings) + # resolvable; the default view lists developer-installed apps only. + apps = await self._devicectl_json("info", "apps", "--include-default-apps") + url_prefix = "" + for app in apps.get("apps", []): + if isinstance(app, dict) and app.get("bundleIdentifier") == package_name: + url_prefix = self._normalize_executable(app.get("url")).rstrip("/") + break + if not url_prefix: + raise ValueError( + f"{package_name!r} is not installed on {self._device_id} or its app URL " + "could not be determined; cannot verify a process to terminate." + ) + processes = await self._devicectl_json("info", "processes") + candidates: list[int] = [] + for process in processes.get("runningProcesses", []): + if not isinstance(process, dict): + continue + executable = self._normalize_executable(process.get("executable")) + if executable != url_prefix and not executable.startswith(url_prefix + "/"): + continue + pid = process.get("processIdentifier") or process.get("pid") or process.get("processID") + if isinstance(pid, int): + candidates.append(pid) + if not candidates: + raise ValueError( + f"No running process found for {package_name!r} on {self._device_id}; " + "launch it with launch_app before stopping." + ) + tracked = self._launched_pids.get(package_name) + if tracked is not None and tracked in candidates: + return tracked + return candidates[0] + + async def _terminate_pid(self, pid: int, kill: bool = False) -> None: + arguments = [ + "devicectl", + "device", + "process", + "terminate", + "--device", + self._device_id, + "--pid", + str(pid), + ] + if kill: + arguments.append("--kill") + await run_xcrun(*arguments, timeout=DEVICECTL_OP_TIMEOUT) + + async def stop_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + # Verify the live pid first — a recycled cached pid could belong to + # a different app now, and terminating it would kill the wrong app. + pid = await self._resolve_pid(package_name) + await self._terminate_pid(pid, kill=True) + self._launched_pids.pop(package_name, None) + return True + + async def install_app(self, app_path: Path) -> str: + self._require_connected() + path = app_path.expanduser().resolve() + if path.suffix.lower() == ".ipa": + if not path.is_file(): + raise ValueError("iOS installation requires an existing .ipa file.") + bundle = self._ipa_bundle_id(path) or path.stem + elif path.suffix == ".app" and path.is_dir(): + with (path / "Info.plist").open("rb") as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + if not isinstance(bundle, str) or not bundle: + raise ValueError("The .app has no CFBundleIdentifier in Info.plist.") + else: + raise ValueError( + "Physical iOS installation requires a signed .app directory or .ipa built " + "for a device (arm64) with a valid provisioning profile." + ) + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "install", + "app", + "--device", + self._device_id, + str(path), + timeout=DEVICECTL_INSTALL_TIMEOUT, + ) + return bundle + + @staticmethod + def _ipa_bundle_id(path: Path) -> str | None: + """Read CFBundleIdentifier from an IPA's embedded app Info.plist.""" + import zipfile + + try: + with zipfile.ZipFile(path) as archive: + for name in archive.namelist(): + if name.startswith("Payload/") and name.endswith(".app/Info.plist"): + with archive.open(name) as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + except (OSError, zipfile.BadZipFile, KeyError, plistlib.InvalidFileException): + return None + return None + + async def list_apps(self) -> dict[str, str]: + async with self._operation_lock: + self._require_connected() + # ``info apps`` defaults to developer-installed apps only, which + # hides Safari and friends; include the system defaults so app + # resolution can find and launch them. + apps = await self._devicectl_json( + "info", "apps", "--include-default-apps" + ) + result = {} + for app in apps.get("apps", []): + bundle = app.get("bundleIdentifier") or app.get("bundleID") + if not bundle: + continue + result[bundle] = app.get("name") or bundle + return result + + async def open_url(self, url: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "process", + "openURL", + "--device", + self._device_id, + url, + timeout=DEVICECTL_OP_TIMEOUT, + ) + return True + + async def get_current_package(self) -> str | None: + async with self._operation_lock: + self._require_connected() + if self._wda is not None: + return await self._wda.active_app() + return None + + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: + raise NotImplementedError("Android shell commands are unavailable on iOS devices.") + + # --- Recording (devicectl screenshot polling) --- + + def _new_recorder(self) -> PhysicalIosRecorder: + """Physical recorder: devicectl screenshot polling + ffconcat encode.""" + return PhysicalIosRecorder(self._device_id) diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py new file mode 100644 index 000000000..06cc9f512 --- /dev/null +++ b/artemis/drivers/ios/physical_recording.py @@ -0,0 +1,493 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iOS device recording via polled ``devicectl`` screenshots. + +``simctl io recordVideo`` is simulator-only, so physical capture polls +``xcrun devicectl device capture screenshot`` into timestamped PNG frames and +assembles each contiguous run of equal-sized frames into a CFR MP4 with the +ffconcat demuxer, preserving real per-frame timing. Segments roll on rotation +(dimension change), max duration, or capture failures. Physical capture has no +audio and a lower frame rate than simulator recording (~1-3 fps bounded by +devicectl round-trip time). +""" + +import asyncio +import contextlib +from pathlib import Path +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from PIL import Image + +from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import devicectl_screenshot, reap_process +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + SEGMENT_PROBE_TIMEOUT_SECONDS, + IosRecordingSession, +) +from artemis.utils.video import get_ffmpeg_path, probe_video_segment, write_recording_manifest +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +CAPTURE_TIMEOUT_SECONDS = 15.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +# devicectl frame polls flake more often than the managed simctl process, so +# the failure cap and the ffmpeg budget are looser than in recording.py. +MAX_CONSECUTIVE_FAILURES = 5 + + +FFMPEG_TIMEOUT_SECONDS = 300.0 + + +async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: + """Seam for segment assembly so tests can intercept the ffmpeg process.""" + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + _stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=FFMPEG_TIMEOUT_SECONDS + ) + except TimeoutError: + await reap_process(process) + return -1, b"ffmpeg timed out" + except asyncio.CancelledError: + await reap_process(process) + # Drop only our own in-flight .part output before propagating. + part = Path(str(arguments[-1])) + if part.suffix == ".mp4" and ".part" in part.name: + part.unlink(missing_ok=True) + raise + rc = process.returncode + if rc is None: + raise RuntimeError("ffmpeg completed without an exit code") + return rc, stderr + + +class IosPhysicalRecordingSession(IosRecordingSession): + """Recording session for polled physical-device frame capture.""" + + poll_task: asyncio.Task | None = None + frames_dir: Path | None = None + frames: list[dict[str, Any]] = [] + frame_index: int = 0 + + +class PhysicalIosRecorder: + """Owns one screenshot-poll recording lifecycle for one physical UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosPhysicalRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS + self._consecutive_failures = 0 + + @property + def session(self) -> IosPhysicalRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _segment_output_path(self, session: IosPhysicalRecordingSession, index: int) -> Path: + assert self._output_dir is not None + return self._output_dir / f"segment_{index:04d}.mp4" + + async def _capture_frame(self, session: IosPhysicalRecordingSession) -> None: + """Append one timestamped screenshot frame to the open segment.""" + assert session.frames_dir is not None + session.frame_index += 1 + path = session.frames_dir / f"frame_{session.frame_index:06d}.png" + await devicectl_screenshot(self._device_id, path, timeout=CAPTURE_TIMEOUT_SECONDS) + captured_at = time.monotonic() + if not path.exists() or path.stat().st_size == 0: + raise RuntimeError("devicectl produced no screenshot frame") + with Image.open(path) as image: + width, height = image.size + if session.capture_width is None: + session.capture_width, session.capture_height = width, height + session.segment_started_monotonic = captured_at + session.segment_started_at = time.time() + if session.anchor_monotonic is None: + session.anchor_monotonic = captured_at + session.start_time = session.segment_started_at + session.frames.append({"path": path, "at": captured_at}) + if (width, height) != (session.capture_width, session.capture_height): + await self._roll(session, captured_at, "rotation") + + async def _poll_loop(self, session: IosPhysicalRecordingSession) -> None: + """Capture frames back-to-back; devicectl's own latency paces the loop.""" + try: + while session.is_active: + failed = False + async with self._lock: + if not session.is_active: + return + try: + await self._capture_frame(session) + self._consecutive_failures = 0 + except asyncio.CancelledError: + raise + except Exception as exc: + # Count any unexpected failure (PIL errors, devicectl + # surprises) toward the limit so the loop can never + # die silently with the session still active. + failed = True + self._consecutive_failures += 1 + session.errors.append(f"Frame capture failed: {exc}") + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + session.errors.append( + "Frame capture failure limit reached; session stopped" + ) + session.is_active = False + return + # Yield every iteration: callers and the watchdog share this loop. + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS if failed else 0) + except asyncio.CancelledError: + return + + async def _start_segment(self, session: IosPhysicalRecordingSession) -> None: + assert self._output_dir is not None + session.frames = [] + # frame_index stays monotonic for the whole session: rolled leftover + # frames keep their original names in the new segment directory, so a + # fresh index range can never collide with them. + session.capture_width = session.capture_height = None + session.segment_started_monotonic = None + session.frames_dir = self._output_dir / f"frames_{session.segment_index:04d}" + session.frames_dir.mkdir(parents=True, exist_ok=True) + + async def _poll_first_frame(self, session: IosPhysicalRecordingSession) -> None: + """Block until the first frame anchors the recording timeline.""" + deadline = time.monotonic() + CAPTURE_TIMEOUT_SECONDS * 2 + while session.anchor_monotonic is None: + if time.monotonic() > deadline: + raise RuntimeError("Timed out waiting for the first devicectl screenshot frame.") + if not session.is_active: + raise RuntimeError( + session.errors[-1] + if session.errors + else "Frame capture stopped before the first frame." + ) + await asyncio.sleep(0.05) + + def _seal_current_segment( + self, session: IosPhysicalRecordingSession, end_monotonic: float + ) -> dict[str, Any] | None: + frames = [f for f in session.frames if f["at"] < end_monotonic] + session.frames = [f for f in session.frames if f["at"] >= end_monotonic] + if not frames: + return None + record = { + "frames": frames, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, frames[0]["at"] - (session.anchor_monotonic or 0.0)), + "end": max(0.0, end_monotonic - (session.anchor_monotonic or 0.0)), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + return record + + @staticmethod + def _ffconcat_file_line(path: Path) -> str: + """Quote a frame path for ffconcat: ' inside a quoted name -> '\\''.""" + escaped = path.as_posix().replace("'", "'\\''") + return f"file '{escaped}'" + + def _write_concat(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> Path: + """Write an ffconcat list preserving real capture timestamps.""" + frames = record["frames"] + end_monotonic = record["end"] + (session.anchor_monotonic or 0.0) + lines = ["ffconcat version 1.0"] + for index, frame in enumerate(frames): + lines.append(self._ffconcat_file_line(Path(frame["path"]))) + following = frames[index + 1]["at"] if index + 1 < len(frames) else end_monotonic + lines.append(f"duration {max(0.001, following - frame['at']):.3f}") + # The concat demuxer ignores the final frame's duration unless the file + # line repeats, so list the last frame twice to keep its tail visible. + lines.append(self._ffconcat_file_line(Path(frames[-1]["path"]))) + concat_path = Path(frames[0]["path"]).parent / "frames.txt" + concat_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return concat_path + + async def _convert_record( + self, session: IosPhysicalRecordingSession, record: dict[str, Any] + ) -> None: + output = Path(record["output_path"]) + width = max(2, int(record["width"] or 1080)) // 2 * 2 + height = max(2, int(record["height"] or 1920)) // 2 * 2 + temporary = output.with_name(f"{output.stem}.part.mp4") + try: + if temporary.exists(): + temporary.unlink() + concat = self._write_concat(session, record) + returncode, stderr = await _run_ffmpeg( + [ + "-y", + "-f", + "concat", + "-safe", + "0", + "-i", + str(concat), + "-vf", + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + "color=black,setsar=1", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary), + ] + ) + metadata = await probe_video_segment( + temporary, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS + ) + valid = ( + returncode == 0 + and temporary.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary.replace(output) + record["conversion_done"] = True + return + record["conversion_error"] = "finalization produced no valid MP4" + logger.error( + f"Physical iOS recording finalization failed (code {returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except TimeoutError as exc: + # A bounded probe that expires records a failure — never report a + # segment whose metadata could not be verified as converted. + record["conversion_error"] = f"metadata probe timed out: {exc}" + logger.error(f"Physical iOS recording probe timed out: {exc}") + except OSError as exc: + record["conversion_error"] = str(exc) + logger.error(f"Physical iOS recording finalization failed: {exc}") + try: + if temporary.exists(): + temporary.unlink() + except OSError: + pass + session.errors.append(f"Segment {record['output_path']} failed finalization; frames kept") + + async def _roll( + self, session: IosPhysicalRecordingSession, end_monotonic: float, reason: str + ) -> None: + """Seal the current frames and open the next segment directory.""" + record = self._seal_current_segment(session, end_monotonic) + leftover = session.frames + if record is not None: + logger.info(f"Rolling physical iOS recording segment after {reason}") + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.is_active: + await self._start_segment(session) + frames_dir = session.frames_dir + if frames_dir is None: + raise RuntimeError("Physical iOS recording roll found no frame directory.") + for frame in leftover: + target = frames_dir / Path(frame["path"]).name + Path(frame["path"]).rename(target) + frame["path"] = target + session.frames = leftover + if leftover: + with Image.open(leftover[0]["path"]) as image: + session.capture_width, session.capture_height = image.size + session.segment_started_monotonic = leftover[0]["at"] + + async def _watchdog(self, session: IosPhysicalRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + poll = session.poll_task + if poll is not None and poll.done() and not poll.cancelled(): + exc = poll.exception() + if exc is not None: + session.errors.append(f"Frame capture task died: {exc}") + session.is_active = False + return + if ( + session.anchor_monotonic is not None + and time.monotonic() - session.anchor_monotonic + >= self._max_duration_seconds + ): + session.errors.append("Recording reached its duration limit") + session.is_active = False + return + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"Physical iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, + ) -> IosPhysicalRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError(f"iOS recording is already active on {self._device_id}") + output = ( + Path(output_dir) + if output_dir is not None + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosPhysicalRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + is_active=True, + ) + self._session = session + await self._start_segment(session) + session.poll_task = asyncio.create_task(self._poll_loop(session)) + # The first-frame wait must run outside the lock: the poll loop takes + # the same lock to append captured frames. + completed = False + try: + await self._poll_first_frame(session) + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + completed = True + return session + finally: + if not completed: + session.is_active = False + if session.poll_task and not session.poll_task.done(): + session.poll_task.cancel() + if session.poll_task is not None: + with contextlib.suppress( + asyncio.CancelledError, OSError, ValueError, RuntimeError, TimeoutError + ): + await session.poll_task + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment so its final MP4 can be read safely.""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosPhysicalRecordingSession) -> Path | None: + session.is_active = False + for task in (session.poll_task, session.watchdog_task): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"Physical iOS recording task ended with an error: {exc}") + async with self._lock: + record = self._seal_current_segment(session, time.monotonic()) + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized physical iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the recording manifest.""" + session = self._session + if session is None: + return None + # _finalize takes _lock itself for the seal — do not hold it here. + finalize = asyncio.ensure_future(self._finalize(session)) + try: + # Shield so a cancelled caller cannot abort mid-finalize and + # lose already-captured frames (mirrors IosScreenRecorder). + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize + raise diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py new file mode 100644 index 000000000..128e47308 --- /dev/null +++ b/artemis/drivers/ios/recording.py @@ -0,0 +1,687 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS Simulator recording via ``xcrun simctl io recordVideo``. + +simctl writes a variable frame-rate H.264 ``.mov`` and announces the first +processed frame on stderr as ``Recording started``; that marker anchors the +recording timeline, not process spawn. Each owned ``.mov`` is finalized to a +browser-safe CFR MP4 whose last frame is cloned across the segment's static +tail, so recorded event timestamps are never stretched. iOS capture has no +audio. Segments roll on display-dimension change, max duration, or recorder +exit; restart gaps stay gaps in rendered clips. +""" + +import asyncio +import os +from pathlib import Path +import re +import signal +import subprocess +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import reap_process +from artemis.utils.video import ( + get_ffmpeg_path, + probe_video_segment, + write_recording_manifest, +) +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import RecordingSession + +logger = get_logger(__name__) + +RECORDING_STARTED_MARKER = "Recording started" +# Longest a single iOS capture may run before auto-stopping (15 minutes). +DEFAULT_MAX_DURATION_SECONDS = 900 +STARTUP_TIMEOUT_SECONDS = 30.0 +SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 +TERMINATE_TIMEOUT_SECONDS = 3.0 +PROBES_TIMEOUT_SECONDS = 3.0 +STALE_RECORDER_GRACE_SECONDS = 3.0 +SEGMENT_PROBE_TIMEOUT_SECONDS = 30.0 +# Finalization re-encodes the whole capture, so the budget scales with the +# segment's wall span: encode on a loaded host can run slower than realtime. +FFMPEG_TIMEOUT_SECONDS = 180.0 +FFMPEG_TIMEOUT_PER_SPAN_SECOND = 2.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +MAX_CONSECUTIVE_FAILURES = 3 +MIN_HEALTHY_SEGMENT_SECONDS = 10.0 +MAX_CONCURRENT_CONVERSIONS = 2 +STDERR_BUFFER_LINES = 200 + + +class IosRecordingSession(RecordingSession): + """Recording session for native iOS ``simctl`` capture.""" + + segments: list[dict[str, Any]] = [] + segment_index: int = 0 + segment_started_at: float | None = None + segment_started_monotonic: float | None = None + anchor_monotonic: float | None = None + stderr_task: asyncio.Task | None = None + stderr_lines: list[str] = [] + conversion_tasks: list[asyncio.Task] = [] + + +def _parse_display_dimensions(text: str) -> tuple[int, int] | None: + """Largest ``IOSurface port`` (width, height) from ``simctl io enumerate``. + + ``recordVideo`` captures the device LCD without ``--display``; only that + framebuffer reports an ``IOSurface port`` (external scene displays carry + only ``Default width``/``height``), and its dimensions swap on rotation, + so the largest reported surface is the segment's coded size. + """ + best: tuple[int, int] | None = None + best_area = 0 + for match in re.finditer( + r"IOSurface port:\s*\n\s*width\s*=\s*(\d+)\s*\n\s*height\s*=\s*(\d+)", text + ): + width, height = int(match.group(1)), int(match.group(2)) + if width * height > best_area: + best, best_area = (width, height), width * height + return best + + +async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: + """Read the recordVideo target display's pixel size via simctl.""" + try: + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + device_id, + "enumerate", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + except OSError: + return None + try: + stdout, _stderr = await asyncio.wait_for(process.communicate(), PROBES_TIMEOUT_SECONDS) + except TimeoutError: + await _reap_probe(process) + return None + except asyncio.CancelledError: + await _reap_probe(process) + raise + if process.returncode != 0: + return None + return _parse_display_dimensions(stdout.decode(errors="replace")) + + +_reap_probe = reap_process + + +async def finalize_mov_to_mp4( + source_path: Path, + output_path: Path, + width: int, + height: int, + wall_span_seconds: float, +) -> bool: + """Atomically finalize an owned .mov to a fixed-canvas CFR MP4. + + ``tpad`` clones the final frame across the segment's remaining wall span + so sparse VFR tails pad without stretching any recorded event time. + """ + if not source_path.exists() or source_path.stat().st_size == 0: + return False + width = max(2, int(width)) // 2 * 2 + height = max(2, int(height)) // 2 * 2 + span = max(0.001, float(wall_span_seconds)) + # fps resamples the raw VFR timeline before scale: scale buffers a frame + # and feeding it to fps first drops the final frame on sparse sources. + video_filter = ( + "setpts=PTS-STARTPTS,fps=30," + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + f"color=black,setsar=1," + f"tpad=stop_mode=clone:stop_duration={span:.3f}" + ) + temporary_path = output_path.with_name(f"{output_path.stem}.part.mp4") + if temporary_path.exists(): + temporary_path.unlink() + try: + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + "-y", + "-i", + str(source_path), + "-map", + "0:v:0", + "-vf", + video_filter, + "-t", + f"{span:.3f}", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + timeout = max(FFMPEG_TIMEOUT_SECONDS, span * FFMPEG_TIMEOUT_PER_SPAN_SECOND) + _stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + metadata = await probe_video_segment( + temporary_path, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS + ) + except TimeoutError: + await reap_process(process) + raise RuntimeError("iOS recording finalization timed out") + except asyncio.CancelledError: + # Reap the owned ffmpeg child and drop the partial .mp4 before + # cancellation propagates — never leave an orphan encoder behind. + await reap_process(process) + if temporary_path.exists(): + temporary_path.unlink() + raise + valid = ( + process.returncode == 0 + and temporary_path.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary_path.replace(output_path) + return True + logger.error( + f"iOS recording finalization failed (code {process.returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except OSError as exc: + logger.error(f"iOS recording finalization failed: {exc}") + except RuntimeError as exc: + logger.error(f"iOS recording finalization failed: {exc}") + if temporary_path.exists(): + temporary_path.unlink() + return False + + +class IosScreenRecorder: + """Owns one simctl recording lifecycle for one pinned simulator UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS + self._consecutive_failures = 0 + self._conversion_semaphore = asyncio.Semaphore(MAX_CONCURRENT_CONVERSIONS) + + @property + def session(self) -> IosRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _output_root(self) -> Path: + if self._output_dir is None: + raise RuntimeError("iOS recording has no output directory") + return self._output_dir + + def _segment_source_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + while True: + name = "recording.mov" if index == 0 else f"recording_{index:03d}.mov" + candidate = output_dir / name + if not candidate.exists(): + return candidate + index += 1 + session.segment_index = index + + def _segment_output_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + if index == 0: + return output_dir / "recording.mp4" + return output_dir / f"recording_{index:03d}.mp4" + + async def _drain_stderr( + self, + process: asyncio.subprocess.Process, + session: IosRecordingSession, + first_frame: asyncio.Future, + ) -> None: + try: + assert process.stderr is not None + while True: + line = await process.stderr.readline() + if not line: + if not first_frame.done(): + tail = "; ".join(session.stderr_lines[-3:]) + first_frame.set_exception( + RuntimeError( + "simctl recordVideo exited before 'Recording started'" + + (f": {tail}" if tail else "") + ) + ) + return + text = line.decode(errors="replace").rstrip() + session.stderr_lines.append(text) + if len(session.stderr_lines) > STDERR_BUFFER_LINES: + del session.stderr_lines[: len(session.stderr_lines) - STDERR_BUFFER_LINES] + if RECORDING_STARTED_MARKER in text and not first_frame.done(): + first_frame.set_result((time.time(), time.monotonic())) + except asyncio.CancelledError: + raise + except Exception as exc: + if not first_frame.done(): + first_frame.set_exception(exc) + + async def _reap_stale_recorders(self, *, any_device: bool = False) -> list[int]: + """SIGINT orphaned ``simctl recordVideo`` processes still holding the host lock. + + ``recordVideo``'s "Host recording is already in progress" lock is shared + host-wide, not per-device, so a recorder orphaned by a session that ended + without reaping it (a killed task or console) blocks every later spawn — + on this device or a sibling simulator. SIGINT lets the orphan flush its + ``.mov``; a survivor is escalated to SIGKILL. + """ + if any_device: + pattern = r"simctl io [0-9A-Fa-f-]+ recordVideo" + else: + pattern = f"simctl io {self._device_id} recordVideo" + try: + probe = await asyncio.to_thread( + subprocess.run, + ["pgrep", "-f", pattern], + capture_output=True, + text=True, + timeout=PROBES_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError): + return [] + pids = [int(token) for token in probe.stdout.split() if token.isdigit()] + reaped: list[int] = [] + for pid in pids: + try: + os.kill(pid, signal.SIGINT) + except (ProcessLookupError, PermissionError): + continue + reaped.append(pid) + if not reaped: + return [] + logger.warning( + f"Reaping {len(reaped)} stale simctl recordVideo process(es) on " + f"{self._device_id}: {reaped}" + ) + deadline = time.monotonic() + STALE_RECORDER_GRACE_SECONDS + for pid in reaped: + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + await asyncio.sleep(0.1) + else: + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + return reaped + + async def _reap(self, process: asyncio.subprocess.Process) -> float: + """SIGINT to flush, escalate to terminate/kill; returns end monotonic.""" + end_monotonic = time.monotonic() + if process.returncode is not None: + return end_monotonic + try: + process.send_signal(signal.SIGINT) + await asyncio.wait_for(process.wait(), SIGINT_FLUSH_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + try: + process.terminate() + except ProcessLookupError: + return end_monotonic + try: + await asyncio.wait_for(process.wait(), TERMINATE_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.wait() + return end_monotonic + + async def _spawn_recorder(self, session: IosRecordingSession) -> None: + """Spawn simctl recordVideo and anchor at its first-frame marker.""" + await self._reap_stale_recorders() + try: + await self._spawn_once(session) + except RuntimeError as exc: + # The host recording lock is shared across simulators, so an orphan + # on a different UDID can still be the contender; sweep all stale + # recorders and give the spawn one more chance. + if "already in progress" not in str(exc): + raise + await self._reap_stale_recorders(any_device=True) + await self._spawn_once(session) + + async def _spawn_once(self, session: IosRecordingSession) -> None: + index = session.segment_index + source_path = self._segment_source_path(session, index) + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + self._device_id, + "recordVideo", + "--codec=h264", + str(source_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + first_frame: asyncio.Future = asyncio.get_running_loop().create_future() + session.stderr_task = asyncio.create_task(self._drain_stderr(process, session, first_frame)) + started = False + try: + marker_wall, marker_monotonic = await asyncio.wait_for( + asyncio.shield(first_frame), STARTUP_TIMEOUT_SECONDS + ) + session.process = process + session.local_video_path = source_path + session.segment_started_at = marker_wall + session.segment_started_monotonic = marker_monotonic + if session.anchor_monotonic is None: + session.start_time = marker_wall + session.anchor_monotonic = marker_monotonic + dimensions = await probe_display_dimensions(self._device_id) + if dimensions: + session.capture_width, session.capture_height = dimensions + logger.info( + f"iOS recording segment {index} first frame " + f"{marker_wall - session.start_time:.2f}s after anchor on {self._device_id}" + ) + started = True + finally: + if not started: + # Startup never completed: the spawned recorder is still + # owned by this coroutine — drain stderr and reap it before + # the original exception/cancellation propagates. + if not first_frame.done(): + first_frame.cancel() + session.stderr_task.cancel() + try: + await session.stderr_task + except asyncio.CancelledError: + pass + except (OSError, RuntimeError, ValueError, TimeoutError) as stderr_error: + logger.debug(f"iOS recorder stderr drain ended with an error: {stderr_error}") + reap = asyncio.ensure_future(self._reap(process)) + try: + await asyncio.shield(reap) + except asyncio.CancelledError: + await reap # finish reaping even when the caller was cancelled + + def _seal_current_segment( + self, session: IosRecordingSession, end_monotonic: float + ) -> dict[str, Any]: + if session.segment_started_monotonic is None or session.anchor_monotonic is None: + raise RuntimeError("No started iOS recording segment to seal") + record = { + "path": session.local_video_path, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, session.segment_started_monotonic - session.anchor_monotonic), + "end": max(0.0, end_monotonic - session.anchor_monotonic), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + session.segment_started_monotonic = None + session.segment_started_at = None + return record + + async def _convert_record(self, session: IosRecordingSession, record: dict[str, Any]) -> None: + source = Path(record["path"]) + output = Path(record["output_path"]) + span = max(0.001, float(record["end"]) - float(record["start"])) + async with self._conversion_semaphore: + ok = await finalize_mov_to_mp4( + source, output, int(record["width"] or 1080), int(record["height"] or 1920), span + ) + if ok: + record["conversion_done"] = True + else: + record["conversion_error"] = "finalization produced no valid MP4" + session.errors.append( + f"Segment {record['output_path']} failed finalization; raw capture kept" + ) + + async def _roll(self, session: IosRecordingSession, end_monotonic: float, reason: str) -> None: + """Seal the current segment and start the next recorder first.""" + record = None + if session.segment_started_monotonic is not None: + process = session.process + if process is not None and process.returncode is None: + end_monotonic = await self._reap(process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + record = self._seal_current_segment(session, end_monotonic) + logger.info(f"Rolling iOS recording segment after {reason}") + restarted = False + if session.is_active: + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + # Spawns that keep dying young are a crash loop: stop + # respawning (the sealed segment still converts below). + session.is_active = False + session.errors.append("Recording recovery limit reached; session stopped") + else: + try: + await self._spawn_recorder(session) + restarted = True + except asyncio.CancelledError: + raise + except (OSError, RuntimeError, TimeoutError) as exc: + self._consecutive_failures += 1 + session.errors.append(f"Recorder restart failed: {exc}") + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if ( + not restarted + and session.is_active + and self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES + ): + session.errors.append("Recording recovery limit reached; session stopped") + session.is_active = False + + async def _watchdog(self, session: IosRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + # Re-verify the trigger under the lock: a concurrent roll + # may have already replaced the dead process. + process = session.process + crashed = process is None or process.returncode is not None + dimensions = await probe_display_dimensions(self._device_id) + rotated = bool( + dimensions + and session.capture_width + and session.capture_height + and dimensions != (session.capture_width, session.capture_height) + ) + age = time.monotonic() - (session.segment_started_monotonic or time.monotonic()) + overdue = age >= self._max_duration_seconds + if crashed and age < MIN_HEALTHY_SEGMENT_SECONDS: + # A respawn that dies before surviving a healthy interval + # is a crash loop, not recovery — count it even though a + # successful respawn would otherwise reset the counter. + self._consecutive_failures += 1 + elif not crashed and age >= MIN_HEALTHY_SEGMENT_SECONDS: + self._consecutive_failures = 0 + if not crashed and not rotated and not overdue: + continue + reason = ( + "recorder exit" if crashed else "rotation" if rotated else "duration limit" + ) + await self._roll(session, time.monotonic(), reason) + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, + ) -> IosRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError(f"iOS recording is already active on {self._device_id}") + output = ( + Path(output_dir) + if output_dir is not None + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + local_video_path=output / "recording.mov", + is_active=True, + ) + self._session = session + started = False + try: + await self._spawn_recorder(session) + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + started = True + return session + finally: + if not started: + session.is_active = False + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment through ``through_time`` (recording-relative).""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + if ( + session.segment_started_monotonic is not None + and end_monotonic <= session.segment_started_monotonic + ): + return + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosRecordingSession) -> Path | None: + if session.is_active: + session.is_active = False + if session.watchdog_task and not session.watchdog_task.done(): + session.watchdog_task.cancel() + try: + await session.watchdog_task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"iOS recording watchdog ended with an error: {exc}") + end_monotonic = time.monotonic() + if session.process is not None and session.process.returncode is None: + end_monotonic = await self._reap(session.process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + if session.segment_started_monotonic is not None: + record = self._seal_current_segment(session, end_monotonic) + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the version-2 manifest.""" + async with self._lock: + session = self._session + if session is None: + return None + finalize = asyncio.ensure_future(self._finalize(session)) + try: + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize # let cleanup finish before propagating + raise diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py new file mode 100644 index 000000000..e46b8b048 --- /dev/null +++ b/artemis/drivers/ios/wda.py @@ -0,0 +1,620 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WebDriverAgent HTTP client for physical iOS UI automation. + +Xcode's ``DeviceInteraction*`` MCP tools accept simulators only, so physical +devices are driven through WebDriverAgent — the same XCUITest bridge Appium +uses. The client speaks plain HTTP to the WDA server running on the device +and has no third-party dependencies; every call is offloaded to a thread so +the driver stays fully async. + +Reaching the device-side server needs one of: + +- ``ARTEMIS_IOS_WDA_URL``: an explicit endpoint such as + ``http://127.0.0.1:8100`` for ``iproxy``/``pymobiledevice3`` forwards or + ``http://:8100`` when the phone shares the LAN. +- The CoreDevice tunnel address from ``devicectl device info details`` — + paired devices already hold a managed IPv6 tunnel that routes TCP. +- ``ARTEMIS_IOS_WDA_HOST``: just a host/IP; port 8100 is assumed. +""" + +import asyncio +import base64 +import http.client +import ipaddress +import json +import os +from typing import Any +import urllib.error +import urllib.request + +from artemis.drivers.ios.hierarchy import pixel_element +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +WDA_URL_ENV = "ARTEMIS_IOS_WDA_URL" +WDA_HOST_ENV = "ARTEMIS_IOS_WDA_HOST" +WDA_DEFAULT_PORT = 8100 +WDA_REQUEST_TIMEOUT = 30.0 + +#: Server-side session invalidation markers — WDA answers these once the +#: session's target app died, so the request provably did not execute and is +#: safe to retry after rebinding. +_SESSION_LOST_MARKERS = ( + "invalid session id", + "no such session", + "stale session", + "session does not exist", + "session id is not valid", + # Anchor-app death — WDA reports "invalid element state: The application + # under test with bundle id ... is not running, possibly crashed" rather + # than a session error. Match the message body, not the generic error + # code, which also covers legitimately unhittable elements. + "application under test", + "possibly crashed", + "is not running", +) + + +class WdaUnavailableError(RuntimeError): + """The WebDriverAgent server could not be reached or did not respond.""" + + +class WdaClient: + """Minimal WebDriverAgent client covering Artemis's interaction surface.""" + + def __init__(self, base_url: str, timeout: float = WDA_REQUEST_TIMEOUT): + self._base = base_url.rstrip("/") + self._timeout = timeout + self._session_id: str | None = None + self._reopen_lock = asyncio.Lock() + + @property + def base_url(self) -> str: + return self._base + + @property + def session_id(self) -> str | None: + return self._session_id + + @property + def connected(self) -> bool: + return self._session_id is not None + + def _sync_request( + self, method: str, path: str, payload: dict[str, Any] | None, timeout: float + ) -> Any: + body = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request( + f"{self._base}{path}", + data=body, + method=method, + headers={"Content-Type": "application/json"} if body else {}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + raw = response.read() + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace")[:500] + raise RuntimeError(f"WebDriverAgent {method} {path} failed: HTTP {error.code} {detail}") + except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException) as error: + # HTTPException covers garbage services answering probed ports — + # BadStatusLine is not an OSError, so it needs mapping here. + raise WdaUnavailableError( + f"WebDriverAgent at {self._base} is unreachable: {error}. " + "Check the device connection, the WDA runner process, and any " + "port forwarding (iproxy or 'pymobiledevice3 remote')." + ) + try: + payload_out = json.loads(raw) + except ValueError: + return raw + if isinstance(payload_out, dict) and "value" in payload_out: + value = payload_out["value"] + if isinstance(value, dict) and value.get("error"): + # Keep both fields in the message — recovery matching keys off + # the error code ("invalid session id") while humans need the + # readable message. + message = value.get("message") or "" + error_code = value["error"] + detail = f"{error_code}: {message}" if message else str(error_code) + raise RuntimeError(f"WebDriverAgent {method} {path} failed: {detail}") + # WDA reports the live session in the outer /status envelope, not + # inside value — surface it so a foreign session is detectable. + if ( + path == "/status" + and isinstance(value, dict) + and isinstance(payload_out.get("sessionId"), str) + and payload_out["sessionId"] + ): + value["sessionId"] = payload_out["sessionId"] + return value + return payload_out + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + timeout: float | None = None, + _recovered: bool = False, + ) -> Any: + request_task = asyncio.create_task( + asyncio.to_thread(self._sync_request, method, path, payload, timeout or self._timeout) + ) + try: + return await asyncio.shield(request_task) + except asyncio.CancelledError: + # The blocking urllib call keeps running on its thread; drain it + # before propagating so a cancelled input cannot still land on the + # device after the caller (and its device lease) moved on. + try: + await request_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA request drain failed while cancelling: {drain_error}") + raise + except RuntimeError as error: + if ( + _recovered + or self._is_session_admin(method, path) + or not self._is_session_loss(error) + ): + raise + # The dead session never ran the command, so any method is safe + # to replay after rebinding. + path = await self._reopen_session_with_path(path) + return await self._request(method, path, payload, timeout, _recovered=True) + except WdaUnavailableError: + if _recovered or method != "GET": + raise + # A timed-out write may have executed device-side, so only reads + # recover transparently; writes surface the error and the next + # observation rebinds if the session really is gone. + path = await self._reopen_session_with_path(path, suppress_errors=True) + return await self._request(method, path, payload, timeout, _recovered=True) + + async def _reopen_session_with_path( + self, path: str, suppress_errors: bool = False + ) -> str: + """Rebind the session and rewrite the old session id inside ``path``.""" + old_id = self._session_id + await self._reopen_session(suppress_errors=suppress_errors) + if old_id and self._session_id and old_id != self._session_id: + path = path.replace(old_id, self._session_id) + return path + + def _is_session_loss(self, error: RuntimeError) -> bool: + message = str(error).casefold() + return any(marker in message for marker in _SESSION_LOST_MARKERS) + + @staticmethod + def _is_session_admin(method: str, path: str) -> bool: + # Session create/delete must not trigger recovery: creating during + # recovery recurses, and deleting a dead session should fail quietly. + return (method == "POST" and path == "/session") or ( + method == "DELETE" and path.startswith("/session/") + ) + + async def _reopen_session(self, suppress_errors: bool = False) -> None: + """Rebind a dead WDA session, anchored to the current foreground app. + + Binding to the foreground bundle keeps the device's visible app + unchanged (a Preferences anchor would pull Settings to the front + mid-task); SpringBoard is not activatable as a session target, so a + home-screen foreground falls back to Preferences. + """ + async with self._reopen_lock: + anchor = await self._foreground_bundle() + if not anchor or anchor == "com.apple.springboard": + anchor = "com.apple.Preferences" + # Drop the zombie first so the preflight does not see it as a + # foreign session and refuse the replacement. + old_id = self._session_id + if old_id: + self._session_id = None + try: + await asyncio.to_thread( + self._sync_request, "DELETE", f"/session/{old_id}", None, 5.0 + ) + except (RuntimeError, WdaUnavailableError, OSError): + pass + try: + self._session_id = await self._create_owned_session( + adopt_existing=False, bundle_id=anchor + ) + logger.info( + f"WebDriverAgent session rebound to {self._session_id} " + f"(anchor {anchor}) on {self._base}" + ) + except Exception as error: + self._session_id = None + if not suppress_errors: + raise WdaUnavailableError( + f"WebDriverAgent session recovery failed on {self._base}: {error}" + ) from error + logger.debug(f"WDA session rebound attempt failed: {error}") + + async def _foreground_bundle(self) -> str | None: + """Session-free probe of the device's foreground app bundle id.""" + try: + value = await asyncio.to_thread( + self._sync_request, "GET", "/wda/activeAppInfo", None, 5.0 + ) + except (RuntimeError, WdaUnavailableError, OSError): + return None + if isinstance(value, dict): + bundle = value.get("bundleId") or value.get("bundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + return None + + # --- Session lifecycle --- + + async def status(self, timeout: float = 5.0) -> dict[str, Any] | None: + try: + value = await self._request("GET", "/status", timeout=timeout) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA status probe failed for {self._base}: {error}") + return None + # Reject non-JSON/foreign bodies instead of wrapping them: a proxy or + # unrelated service answering on this port is not WebDriverAgent. + return value if isinstance(value, dict) else None + + async def device_info(self, timeout: float = 10.0) -> dict[str, Any]: + """GET /wda/device/info — available without a session. + + ``uuid`` here is ``identifierForVendor``, not the device UDID, so + identity checks must rely on ``name``/``isSimulator`` instead. + """ + value = await self._request("GET", "/wda/device/info", timeout=timeout) + if not isinstance(value, dict): + raise RuntimeError( + f"WebDriverAgent /wda/device/info returned a non-JSON response: {value!r:.300}" + ) + return value + + async def _create_owned_session( + self, adopt_existing: bool, bundle_id: str = "com.apple.Preferences" + ) -> str: + """Status preflight, POST /session, and ID assignment as one unit.""" + if self._session_id: + return self._session_id + status = await self.status() + if status is None: + raise WdaUnavailableError(f"WebDriverAgent at {self._base} did not answer /status.") + active = status.get("sessionId") + if isinstance(active, str) and active: + if adopt_existing: + # The driver launched this runner itself, so its auto-created + # session is ours to reuse — not a foreign client to protect. + self._session_id = active + return active + raise RuntimeError( + "Refusing to replace the active WebDriverAgent session " + f"({active}), which this client does not own — POST /session " + "would kill it. Point ARTEMIS_IOS_WDA_URL at a dedicated WDA " + "server or close the existing session first." + ) + payload = { + "capabilities": { + # A bare session binds to an ephemeral pid.0 application that + # dies instantly ("stale element reference" on first command). + # com.apple.springboard cannot be activated as an app target; + # the anchor must be an installed, activatable app so the + # session binds to a real process. + "alwaysMatch": { + "platformName": "iOS", + "bundleId": bundle_id, + }, + "firstMatch": [{}], + } + } + value = await self._request("POST", "/session", payload, timeout=60.0) + session_id = None + if isinstance(value, dict): + session_id = value.get("sessionId") + if session_id is None and isinstance(value.get("capabilities"), dict): + session_id = value.get("capabilities", {}).get("sessionId") + if not isinstance(session_id, str) or not session_id: + raise RuntimeError( + f"WebDriverAgent did not return a session id (response: {value!r:.300})." + ) + self._session_id = session_id + return session_id + + async def open_session(self, adopt_existing: bool = False) -> str: + """Create a WDA session, refusing to take over a foreign one. + + ``adopt_existing=True`` reuses the active session reported by + ``/status``; only safe when the caller owns the WDA runner process + (auto-launched runners create a session on startup). Foreign or + user-provisioned endpoints must keep the default refusal. + + Cancellation-safe: when the caller is cancelled while creation is in + flight, the request is drained and any session it produced is closed + before the CancelledError propagates. + """ + create_task = asyncio.ensure_future(self._create_owned_session(adopt_existing)) + try: + return await asyncio.shield(create_task) + except asyncio.CancelledError: + session_id = None + try: + session_id = await create_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA session-create drain failed while cancelling: {drain_error}") + if session_id: + try: + await self.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as close_error: + # Best-effort teardown of the orphaned session. + logger.debug( + f"WDA session close after cancelled open_session failed: {close_error}" + ) + raise + + async def close_session(self) -> None: + session_id, self._session_id = self._session_id, None + if session_id is None: + return + try: + await self._request("DELETE", f"/session/{session_id}", timeout=10.0) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA session delete failed: {error}") + + # --- Observation --- + + async def screenshot_png(self) -> bytes: + value = await self._request("GET", "/screenshot") + if not isinstance(value, str) or not value: + raise RuntimeError("WebDriverAgent returned no screenshot data.") + return base64.b64decode(value) + + async def source_json(self) -> dict[str, Any]: + value = await self._request("GET", "/source?format=json") + if not isinstance(value, dict): + raise RuntimeError("WebDriverAgent returned a non-JSON hierarchy.") + return value + + async def window_size(self) -> tuple[float, float]: + path = f"/session/{self._session_id}/window/size" if self._session_id else "/window/size" + value = await self._request("GET", path) + if isinstance(value, dict) and "width" in value and "height" in value: + return float(value["width"]), float(value["height"]) + raise RuntimeError(f"WebDriverAgent returned an unexpected window size: {value!r}") + + async def active_app(self) -> str | None: + path = "/wda/activeAppInfo" + if self._session_id: + path = f"/session/{self._session_id}/wda/activeAppInfo" + try: + value = await self._request("GET", path) + except (RuntimeError, WdaUnavailableError): + return None + if isinstance(value, dict): + bundle = value.get("bundleId") or value.get("bundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + return None + + # --- Input --- + + async def _actions(self, pointer_actions: list[dict[str, Any]]) -> None: + session = self._require_session() + payload = { + "actions": [ + { + "type": "pointer", + "id": "artemis-finger", + "parameters": {"pointerType": "touch"}, + "actions": pointer_actions, + } + ] + } + await self._request("POST", f"/session/{session}/actions", payload) + + def _require_session(self) -> str: + if not self._session_id: + raise RuntimeError("WebDriverAgent session is not open.") + return self._session_id + + async def tap(self, x: float, y: float, hold_ms: int = 0) -> None: + actions: list[dict[str, Any]] = [ + {"type": "pointerMove", "duration": 0, "x": x, "y": y}, + {"type": "pointerDown", "button": 0}, + ] + if hold_ms > 0: + actions.append({"type": "pause", "duration": hold_ms}) + actions.append({"type": "pointerUp", "button": 0}) + await self._actions(actions) + + async def swipe(self, sx: float, sy: float, ex: float, ey: float, duration_ms: int) -> None: + await self._actions( + [ + {"type": "pointerMove", "duration": 0, "x": sx, "y": sy}, + {"type": "pointerDown", "button": 0}, + {"type": "pause", "duration": 50}, + {"type": "pointerMove", "duration": duration_ms, "x": ex, "y": ey}, + {"type": "pointerUp", "button": 0}, + ] + ) + + async def type_text(self, text: str) -> None: + session = self._require_session() + # ``value`` is a list of Unicode code points — newlines and non-ASCII + # text carry their literal values to the focused field. + payload = {"value": list(text)} + await self._request("POST", f"/session/{session}/wda/keys", payload) + + async def press_button(self, name: str) -> bool: + """WDA hardware buttons: home, volumeUp, volumeDown, power.""" + session = self._require_session() + try: + await self._request("POST", f"/session/{session}/wda/pressButton", {"name": name}) + return True + except WdaUnavailableError: + # The transport died — propagate so callers don't misreport an + # outage as an unsupported button. + raise + except RuntimeError: + return False + + async def homescreen(self) -> None: + await self._request("POST", "/wda/homescreen") + + async def lock(self) -> None: + await self._request("POST", "/wda/lock") + + async def unlock(self) -> None: + await self._request("POST", "/wda/unlock") + + +def normalize_wda_url(raw: str) -> str: + """Accept bare hosts, IPv6 literals, or full URLs and return a base URL.""" + value = raw.strip() + if not value: + return value + if "://" not in value: + # Bare IPv6 literals need brackets once a port is attached. A bare + # "v6:port" string is ambiguous — treat the last group as a port only + # when the address part parses as a real IPv6 literal. + if value.count(":") > 1 and not value.startswith("["): + try: + ipaddress.IPv6Address(value) + value = f"[{value}]" + except ValueError: + address, _, port = value.rpartition(":") + try: + ipaddress.IPv6Address(address) + value = f"[{address}]:{port}" + except ValueError: + value = f"[{value}]" + value = f"http://{value}" + value = value.rstrip("/") + scheme, _, remainder = value.partition("://") + host_port, _, path = remainder.partition("/") + # A port is present when ':' follows the host (or the IPv6 ']' bracket). + has_port = ( + host_port.rsplit("]", 1)[-1].startswith(":") if "]" in host_port else ":" in host_port + ) + if has_port: + return value + # The default port belongs to the authority, ahead of any path suffix. + suffix = f"/{path}" if path else "" + return f"{scheme}://{host_port}:{WDA_DEFAULT_PORT}{suffix}" + + +def wda_url_candidates( + env_url: str | None = None, + env_host: str | None = None, + tunnel_ip: str | None = None, +) -> list[str]: + """Ordered endpoints to probe for a running WebDriverAgent server.""" + candidates: list[str] = [] + for raw in ( + env_url if env_url is not None else os.environ.get(WDA_URL_ENV), + env_host if env_host is not None else os.environ.get(WDA_HOST_ENV), + ( + f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" + if ":" in tunnel_ip + else f"{tunnel_ip}:{WDA_DEFAULT_PORT}" + ) + if tunnel_ip + else None, + f"127.0.0.1:{WDA_DEFAULT_PORT}", + ): + if not raw: + continue + url = normalize_wda_url(raw) + if url and url not in candidates: + candidates.append(url) + return candidates + + +async def probe_wda(candidates: list[str], timeout: float = 5.0) -> WdaClient | None: + """Return a client bound to the first endpoint that answers ``/status``.""" + for url in candidates: + client = WdaClient(url) + status = await client.status(timeout=timeout) + if status is not None: + return client + return None + + +_ELEMENT_TYPE_PREFIX = "XCUIElementType" + + +def parse_wda_elements( + node: dict[str, Any], + scale: tuple[float, float], + width: int, + height: int, +) -> list[dict[str, Any]]: + """Flatten a WDA ``/source?format=json`` tree into Artemis ui_elements. + + Mirrors ``parse_hierarchy``'s output: ``text``, ``resource_id``, ``class``, + ``bounds``, ``parsed_bounds``, and ``hit_point`` in screenshot pixels. + WDA's ``isVisible`` is advisory; elements with usable geometry are kept so + downstream consumers see everything XCTest reports. + """ + elements: list[dict[str, Any]] = [] + + def visit(entry: dict[str, Any]) -> None: + rect = entry.get("rect") + if not isinstance(rect, dict): + rect = {} + try: + x = float(rect.get("x") or 0.0) + y = float(rect.get("y") or 0.0) + w = float(rect.get("width") or 0.0) + h = float(rect.get("height") or 0.0) + except (TypeError, ValueError, AttributeError): + x = y = w = h = 0.0 + if w > 0 and h > 0: + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right > 0 and bottom > 0 and left < width and top < height: + label = entry.get("label") + value = entry.get("value") + name = entry.get("name") + text = "" + if isinstance(label, str) and label: + text = label + elif value is not None and str(value): + text = str(value) + element_type = str(entry.get("type") or "") + element = pixel_element( + text=text, + resource_id=name if isinstance(name, str) else "", + class_name=element_type.removeprefix(_ELEMENT_TYPE_PREFIX), + left=left, + top=top, + right=right, + bottom=bottom, + ) + element["hit_point"] = [(left + right) // 2, (top + bottom) // 2] + element["visible"] = bool(entry.get("isVisible", True)) + if value is not None and str(value) != text: + element["value"] = str(value) + elements.append(element) + children = entry.get("children") + if isinstance(children, list): + for child in children: + if isinstance(child, dict): + visit(child) + + visit(node) + return elements diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py new file mode 100644 index 000000000..6262985f5 --- /dev/null +++ b/artemis/drivers/ios/xcode_driver.py @@ -0,0 +1,640 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator support using Xcode 27's native MCP and simctl tools.""" + +from __future__ import annotations + +import asyncio +import base64 +from io import BytesIO +import json +import re +from pathlib import Path +import plistlib +import sys +from typing import TYPE_CHECKING, Any, Literal +from uuid import uuid4 + +from PIL import Image + +from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge +from artemis.drivers.ios.discovery import ( + clear_ios_simulator_cache, + BOOTED_SIMULATOR_ID, + device_matches_identifier, + parse_simctl_devices, + parse_xcode_version, + plist_to_json, + run_xcrun, +) +from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + IosRecordingSession, + IosScreenRecorder, +) +from third_party.mobile_use.controllers.types import ElementQuery +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session + +if TYPE_CHECKING: + from artemis.drivers.ios.physical_recording import PhysicalIosRecorder + +logger = get_logger(__name__) + +# Xcode reports an orphaned interaction session as "already in use by a +# different session with key '