diff --git a/docs/COACH_HUD.md b/docs/COACH_HUD.md new file mode 100644 index 0000000..6a83ee6 --- /dev/null +++ b/docs/COACH_HUD.md @@ -0,0 +1,143 @@ +# Agent-guided demonstration HUD + +OpenAdapt Desktop already owns a native control overlay. This document extends that overlay into a coach for a first demonstration: a person does the clicks on the live system, and the agent they already use suggests the next move. + +The website isn't the runner. Citrix, RDP, a Windows EMR, and a local browser all use this overlay. + +## Decision: who actuates + +The person actuates. The agent suggests. + +For a novel GUI path, the calling agent isn't a production demonstration. Unsigned success is failure. A person admits later. The overlay must never look like `VERIFIED` or a Seal because a model proposed a click. + +Agent-with-confirm is a later governed-replay mode, after a certified bundle exists. Agent-solo is out of scope for this HUD. + +## Decision: where the hint lives + +The hint has more than one home, and they don't share a wire. + +The compact native overlay shows a closed-vocab status (`LOCAL CAPTURE`, `PAUSED`) plus, during recording, a short local hint and an optional "Your turn" badge. No overlay buttons while observing, recording, executing, verifying, pausing, resuming, or stopping. The window stays pointer-transparent, non-focusable, and capture-excluded. + +`overlay://coach` is the local-only coach channel. Hint text, turn, pause reason, operator response, and an optional bound target rect travel there. That payload is never written into evidence, `report.json`, a Seal, Cloud ingest, PostHog, or `overlay://frame`. Capture exclusion stays on even in presentation mode; a compositor that wants a demo derivative already subscribes to `overlay://frame`. + +The main Desktop window can show the pack playbook (step list, QR, pack URL) if the chat agent is weak. Hotkeys bind there, not on the click-through overlay. Pack presence on openadapt.ai stays claims-only. Desktop may fetch the pack URL locally. It must not POST live GUI state to that site. + +Chat stays the agent's own UI. OpenAdapt doesn't grow a second chat product. + +## Decision: how the existing agent drives it + +Coding agents on the machine (Claude Code, Cursor, a local MCP client) already sit next to Desktop. They drive the coach through the existing loopback IPC, not a new daemon. + +1. Desktop writes `~/.openadapt/desktop_ipc.json` = `{host, port, token, protocol_version}` when the engine starts. That file already exists for the tray. +2. The agent connects to `127.0.0.1` with the session token and sends newline JSON `{type, data, token}`. +3. `set_coach` / `get_coach` / `clear_coach` are engine commands on that socket (and on the Tauri sidecar JSON-lines wire). `set_coach` emits `engine://coach`. The overlay reduces it and emits `overlay://coach` for local subscribers. +4. The tray allowlist still drops `coach` events. A tray client must never receive hint text. + +ChatGPT / Claude in a browser can't run local MCP. They already fetched the pack. Desktop can load that pack URL itself and step the playbook in the main window. MCP in `openadapt-agent` is the later wrapper around these same commands; this slice leaves the payload stable so that wrapper can call `set_coach` without a Desktop redesign. + +A coach hint is a short instruction from the playbook or the agent, such as "Open the claim screen". It must not include a person name, a record id, chart text, a URL, or a typed value. + +## Decision: auth pause + +The person types secrets in the real application. OpenAdapt, the HUD, and the agent do not ask them to paste a password into chat. + +When the agent (or the person) marks a secret field, the overlay goes to `paused` and may become interactive. The card copy is exactly: + +> Type in the application. Continue here when done. + +Continue is an overlay control at that pause boundary. Capture pause/resume is still unimplemented in the recorder (`pause_recording` reports current status; Flow record sessions advertise `pause: false`). v1 coach pause is an overlay interaction boundary. Typing in the app still reaches the app because the overlay shrinks to a compact card and the rest of the screen is the live UI. If lossless capture pause lands later, this same `pause_reason: "auth"` path should request it. + +## Decision: feedback + +Also a pause boundary, never a click-through intercept. + +| Operator action | Coach field | Overlay | +|---|---|---| +| Continue | `operator_response: "continue"` | Leaves pause, back to recording visuals | +| That was wrong | `operator_response: "wrong"` | Stays paused until the agent sends the next hint | +| Skip | `operator_response: "skip"` | Stays paused; agent decides whether skip is legal | +| Done | `operator_response: "done"` | Stays paused; agent or the person then stops from Desktop | +| Secret field | `pause_reason: "secret_field"` | Same card as auth | + +Wrong-step, skip, and done do not compile, admit, or Seal anything. They are notes for the agent and for a later human admission. + +## Decision: what the agent may see + +v1: nothing from the live GUI. + +The agent already has the pack. It can `get_coach` to read the last hint and the last operator response. It does not receive screenshots, UIA/AX/AT-SPI text, OCR, URLs, or field values. Structural observers still persist beside capture events for compile, on the machine. They are not a coach feed. + +A later local-only snapshot of control *roles* (not values) could help an on-machine coding agent. That is a separate change and still must not leave the host. + +## Ghost target ring + +If `set_coach` includes a rect *and* an exact observation binding (`observation_hmac_sha256` of 64 hex chars, or a `media_frame` hash plus `frame_index`), the overlay may grow to the current monitor, stay click-through, and draw a dashed ring. Coordinate space is `top_level_viewport_normalized`. Omit the ring when the binding is missing. Do not invent a rect from a screenshot, a selector, or interpolation. + +While paused or terminal, the overlay shrinks back to a compact card so the person can type in the app. The ring is visual-only and capture-excluded. + +## Native window contract (unchanged, plus coach) + +During `observing`, `recording`, `executing`, `verifying`, `pausing`, `resuming`, `stopping`: pointer-transparent, non-focusable, capture-excluded, no overlay controls. Status, hint, badge, and ring may paint. Interactive controls only at `paused` or terminal. + +`overlay://frame` stays the Types v2 closed vocabulary. Coach fields do not appear in `state_id`, `status`, `workflow_label`, or `target_tracking` of that frame. Flow's runtime emitter is unchanged. + +Same overlay path for native, RDP, and Citrix. RDP/Citrix stay black-box: OCR, relational anchors, and fresh-frame verification on the runner, not fake UIA across the remote boundary. + +## Rejected shapes + +DOM or accessibility injection puts OpenAdapt inside the target app's tree. It intercepts the physical click we are trying to record, and it contaminates capture. The overlay already exists to avoid that. + +A website HUD cannot see Citrix or a local EMR. The pack page is claims. The runner is Desktop. + +WebRTC (or any pixel pipe) to Cloud or openadapt.ai is a PHI export. Do not add it. + +Phonetic coaching still needs a HUD, and it is a poor fit for a dense form. + +Screenshot-to-ChatGPT is the same export with extra steps. The person and the agent look at the real UI; they do not upload it. + +## `set_coach` payload + +Local only. Schema `openadapt.control-overlay-coach/v1`. Not an `openadapt-types` frame. + +```json +{ + "schema_version": "openadapt.control-overlay-coach/v1", + "hint": "Open the claim screen", + "turn": "your_turn", + "pause_reason": null, + "target": null, + "operator_response": null +} +``` + +`turn` is `your_turn`, `wait`, `auth`, or `feedback`. `pause_reason` is `auth`, `secret_field`, `wrong_step`, `skip`, `done`, or `next`. Hint max 80 characters after whitespace collapse. The engine drops a hint that contains a URL, `@`, or a run of six or more digits rather than display a maybe-identifying string. + +## Agent recipe (coding agent on the same machine) + +```python +import json, socket +from pathlib import Path + +disc = json.loads(Path.home().joinpath(".openadapt/desktop_ipc.json").read_text()) +sock = socket.create_connection((disc["host"], disc["port"])) +sock.sendall((json.dumps({ + "type": "set_coach", + "token": disc["token"], + "data": {"hint": "Open the claim screen", "turn": "your_turn"}, +}) + "\n").encode()) +print(sock.makefile().readline()) +``` + +`get_coach` on the same socket returns the current payload to that connection only. Poll it after you ask the person to continue. + +MCP in `openadapt-agent` should wrap these three commands when that repo is extended. Until then, this IPC is the interface. + +## v1 gaps + +Lossless capture pause/resume is still a recorder no-op. Coach pause doesn't pretend otherwise. + +The main window doesn't fetch a pack URL or bind hotkeys yet. The overlay is the part a person can feel during recording. + +`openadapt-agent` doesn't expose `set_coach` as an MCP tool yet. The payload above is what it should send. + +A bound target ring needs an observation HMAC the first-run agent usually does not have. Most first demonstrations show the hint without a ring, which is the correct omission. diff --git a/docs/CONTROL_OVERLAY.md b/docs/CONTROL_OVERLAY.md index 47ef8bf..91be33a 100644 --- a/docs/CONTROL_OVERLAY.md +++ b/docs/CONTROL_OVERLAY.md @@ -75,3 +75,15 @@ path (or switch to full-window capture) rather than claiming the native overlay was recorded directly. Injecting controls into the target application's DOM is not the normal path; a clearly labeled demo-only injection may be used only as a fallback when post-composition is unavailable. + +## Demonstration coach + +The overlay can also coach a first demonstration: the person clicks on the live +system, and a local agent suggests the next move. That design, the +`overlay://coach` payload, the auth-pause copy, and the loopback IPC recipe +live in [`COACH_HUD.md`](COACH_HUD.md). + +`overlay://coach` is local-only. Do not write it into evidence, `report.json`, a +Seal, Cloud ingest, PostHog, or this `overlay://frame` contract. Hint text is +not a Types field. A compositor that consumes `overlay://frame` already has +the closed vocabulary it needs. diff --git a/engine/coach.py b/engine/coach.py new file mode 100644 index 0000000..2023161 --- /dev/null +++ b/engine/coach.py @@ -0,0 +1,175 @@ +"""Local-only demonstration coach payload. + +The control overlay may show this on the operator's machine. It must never be +written into evidence, report.json, a Seal, Cloud ingest, PostHog, or the +closed ``overlay://frame`` Types contract. Capture exclusion stays on. +""" + +from __future__ import annotations + +import re +from typing import Any + +COACH_SCHEMA_VERSION = "openadapt.control-overlay-coach/v1" +HINT_MAX_CHARS = 80 +_TURNS = frozenset({"your_turn", "wait", "auth", "feedback"}) +_PAUSE_REASONS = frozenset( + {"auth", "secret_field", "wrong_step", "skip", "done", "next"} +) +_OPERATOR_RESPONSES = frozenset({"continue", "wrong", "skip", "done", "secret_field"}) +_URL_RE = re.compile(r"https?://|www\.", re.IGNORECASE) +_LONG_ID_RE = re.compile(r"\d{6,}") +_HMAC_RE = re.compile(r"^[a-f0-9]{64}$") + +EMPTY_COACH: dict[str, Any] = { + "schema_version": COACH_SCHEMA_VERSION, + "hint": None, + "turn": None, + "pause_reason": None, + "target": None, + "operator_response": None, +} + + +def empty_coach() -> dict[str, Any]: + """Return a fresh empty coach payload.""" + + return dict(EMPTY_COACH) + + +def sanitize_hint(raw: object) -> str | None: + """Return a short local hint, or None if the text looks identifying.""" + + if raw is None: + return None + text = " ".join(str(raw).split()) + if not text: + return None + if len(text) > HINT_MAX_CHARS: + text = text[:HINT_MAX_CHARS].rstrip() + if _URL_RE.search(text) or "@" in text or _LONG_ID_RE.search(text): + return None + return text + + +def coach_holds_pause(payload: dict[str, Any]) -> bool: + """True when the overlay must stay at a pause boundary for the operator.""" + + turn = payload.get("turn") + reason = payload.get("pause_reason") + return turn in {"auth", "feedback"} or reason in { + "auth", + "secret_field", + "wrong_step", + "skip", + "done", + } + + +def _normalized_rect(raw: object) -> dict[str, float] | None: + if not isinstance(raw, dict): + return None + try: + x = float(raw["x"]) + y = float(raw["y"]) + width = float(raw["width"]) + height = float(raw["height"]) + except (KeyError, TypeError, ValueError): + return None + if width <= 0 or height <= 0: + return None + if min(x, y) < 0 or max(x + width, y + height) > 1.0001: + return None + return {"x": x, "y": y, "width": width, "height": height} + + +def _binding(raw: object) -> dict[str, Any] | None: + if not isinstance(raw, dict): + return None + kind = raw.get("kind") + if kind == "observation_hmac_sha256": + digest = raw.get("observation_hmac_sha256") + if not isinstance(digest, str) or _HMAC_RE.fullmatch(digest) is None: + return None + return { + "kind": "observation_hmac_sha256", + "observation_hmac_sha256": digest, + } + if kind == "media_frame": + digest = raw.get("media_sha256") + index = raw.get("frame_index") + if not isinstance(digest, str) or _HMAC_RE.fullmatch(digest) is None: + return None + if not isinstance(index, int) or isinstance(index, bool) or index < 0: + return None + return { + "kind": "media_frame", + "media_sha256": digest, + "frame_index": index, + } + return None + + +def bind_coach_target(raw: object) -> dict[str, Any] | None: + """Return a ring rect only when an exact observation binding is present. + + A rect without a binding is omitted. Heuristic reconstruction is forbidden. + """ + + if not isinstance(raw, dict): + return None + if raw.get("coordinate_space") != "top_level_viewport_normalized": + return None + binding = _binding(raw.get("binding")) + rect = _normalized_rect(raw.get("rect")) + if binding is None or rect is None: + return None + return { + "coordinate_space": "top_level_viewport_normalized", + "rect": rect, + "binding": binding, + } + + +def apply_coach_update(current: dict[str, Any], params: dict[str, Any] | None) -> dict[str, Any]: + """Merge a set_coach params dict into the in-memory coach payload.""" + + incoming = dict(params or {}) + if incoming.get("clear") is True: + return empty_coach() + next_state = dict(current) if current else empty_coach() + next_state["schema_version"] = COACH_SCHEMA_VERSION + # Drop anything a caller might have stuffed in. Pack URLs, screenshots, + # and identities belong nowhere on this channel. + next_state.pop("pack_url", None) + next_state.pop("screenshot", None) + next_state.pop("image", None) + + if "hint" in incoming: + next_state["hint"] = sanitize_hint(incoming.get("hint")) + if "turn" in incoming: + turn = incoming.get("turn") + next_state["turn"] = turn if turn in _TURNS else None + if "pause_reason" in incoming: + reason = incoming.get("pause_reason") + next_state["pause_reason"] = reason if reason in _PAUSE_REASONS else None + if "target" in incoming: + next_state["target"] = bind_coach_target(incoming.get("target")) + if "operator_response" in incoming: + response = incoming.get("operator_response") + if response in _OPERATOR_RESPONSES: + next_state["operator_response"] = response + if response == "continue": + next_state["pause_reason"] = None + if next_state.get("turn") in {"auth", "feedback"}: + next_state["turn"] = "your_turn" + elif incoming.get("operator_response") is None: + next_state["operator_response"] = None + return { + "schema_version": COACH_SCHEMA_VERSION, + "hint": next_state.get("hint"), + "turn": next_state.get("turn"), + "pause_reason": next_state.get("pause_reason"), + "target": next_state.get("target"), + "operator_response": next_state.get("operator_response"), + } diff --git a/engine/dispatch.py b/engine/dispatch.py index fc91fa9..2855bab 100644 --- a/engine/dispatch.py +++ b/engine/dispatch.py @@ -226,8 +226,12 @@ def __init__( self._sync_paused = False self._flow_recording: _ActiveFlowRecording | None = None self._pending_run_memory: dict[tuple[str, str], tuple[Path, str, bool]] = {} + self._coach: dict[str, Any] = {} self._handlers: dict[str, Callable[..., dict | None]] = {} self._register() + from engine.coach import empty_coach + + self._coach = empty_coach() # ------------------------------------------------------------------ setup @@ -240,6 +244,9 @@ def _register(self) -> None: "pause_recording": self.pause_recording, "resume_recording": self.resume_recording, "get_status": self.get_status, + "set_coach": self.set_coach, + "get_coach": self.get_coach, + "clear_coach": self.clear_coach, # library / captures / workflows "get_workflows": self.get_workflows, "get_first_workflow_state": self.get_first_workflow_state, @@ -591,6 +598,7 @@ def stop_recording(self, **params: Any) -> dict: metadata = controller.stop() self.emit("recording_stopped", metadata) self.emit("status_update", self._status_dict(controller)) + self.clear_coach() stopped = {"capture_id": metadata.get("id"), **metadata} correction = self._stopped_teach_correction(stopped) if correction is not None: @@ -653,6 +661,7 @@ def _finalize_flow_capture(self, active: _ActiveFlowRecording, result: Any) -> d } self.emit("recording_stopped", metadata) self.emit("status_update", self._status_dict(self.services.controller)) + self.clear_coach() metadata["compile"] = self._compile_registered_capture( active.capture_id, automatic=True, @@ -679,6 +688,26 @@ def get_status(self, **params: Any) -> dict: return status return self._status_dict(self.services.controller) + def set_coach(self, **params: Any) -> dict: + """Update the local-only demonstration coach. Never persisted.""" + from engine.coach import apply_coach_update + + self._coach = apply_coach_update(self._coach, params) + self.emit("coach", dict(self._coach)) + return dict(self._coach) + + def get_coach(self, **params: Any) -> dict: + """Return the current local coach payload.""" + return dict(self._coach) + + def clear_coach(self, **params: Any) -> dict: + """Drop the in-memory coach payload and notify local listeners.""" + from engine.coach import empty_coach + + self._coach = empty_coach() + self.emit("coach", dict(self._coach)) + return dict(self._coach) + def _status_dict(self, controller: Any) -> dict: from engine.controller import RecordingState diff --git a/engine/presentation_export.py b/engine/presentation_export.py index 2741cdd..08df006 100644 --- a/engine/presentation_export.py +++ b/engine/presentation_export.py @@ -4,6 +4,10 @@ control-overlay timeline v2, binds it to the exact source-media hash, composites in memory over a rawvideo pipe, and atomically publishes a separate MP4 derivative. + +The local coach channel (``overlay://coach``, schema +``openadapt.control-overlay-coach/v1``) is not a timeline source. Do not +read, composite, or persist it here. """ from __future__ import annotations diff --git a/engine/socket_server.py b/engine/socket_server.py index d60828f..4413725 100644 --- a/engine/socket_server.py +++ b/engine/socket_server.py @@ -54,6 +54,11 @@ } ) +# Local coding-agent coach. Same loopback socket as the tray, different +# vocabulary: coach events are never forwarded to tray clients. +_LOCAL_AGENT_COMMANDS = frozenset({"set_coach", "get_coach", "clear_coach"}) +_SOCKET_COMMANDS = _TRAY_COMMANDS | _LOCAL_AGENT_COMMANDS + # Events the tray's IPCMessageType enum can decode. Anything else is dropped # before forwarding so the tray never hits a from_json ValueError. _TRAY_EVENTS = frozenset( @@ -225,7 +230,7 @@ def _handle_frame(self, conn: socket.socket, line: str) -> None: self._send(conn, "recording_error", {"error": "unauthorized"}) return cmd = frame.get("type") - if cmd not in _TRAY_COMMANDS: + if cmd not in _SOCKET_COMMANDS: logger.warning("Tray sent unsupported command: {c}", c=cmd) return params = frame.get("data") or {} @@ -240,6 +245,10 @@ def _handle_frame(self, conn: socket.socket, line: str) -> None: # so the tray (which only renders events) updates immediately. if cmd == "get_status" and result is not None: self._send(conn, "status_update", self._tray_event_data("status_update", result)) + # Coach replies stay on this connection. Never broadcast hint text + # to a tray client that cannot decode the type. + if cmd in _LOCAL_AGENT_COMMANDS and result is not None: + self._send(conn, "coach", result) # ------------------------------------------------------------- send/emit diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index c1cbe53..b2c4f76 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -13,12 +13,42 @@ // so the engine (W1) and tray (W3) agents can align on the same wire. use serde_json::{json, Value}; -use tauri::{AppHandle, Manager, State}; +use tauri::{ + AppHandle, LogicalSize, Manager, PhysicalPosition, PhysicalSize, Position, Size, State, +}; use tauri_plugin_shell::ShellExt; use url::Url; use crate::sidecar::SidecarHandle; +const OVERLAY_COMPACT_WIDTH: f64 = 440.0; +const OVERLAY_COMPACT_HEIGHT: f64 = 104.0; +const OVERLAY_PAUSED_WIDTH: f64 = 560.0; +const OVERLAY_PAUSED_HEIGHT: f64 = 168.0; + +pub(crate) fn place_control_overlay(app: &AppHandle) { + let Some(main) = app.get_webview_window("main") else { + return; + }; + let Some(overlay) = app.get_webview_window("control-overlay") else { + return; + }; + let monitor = main + .current_monitor() + .ok() + .flatten() + .or_else(|| app.primary_monitor().ok().flatten()); + let (Some(monitor), Ok(overlay_size)) = (monitor, overlay.outer_size()) else { + return; + }; + let monitor_position = monitor.position(); + let monitor_size = monitor.size(); + let margin = (24.0 * monitor.scale_factor()).round() as i32; + let x = monitor_position.x + margin; + let y = monitor_position.y + monitor_size.height as i32 - overlay_size.height as i32 - margin; + let _ = overlay.set_position(Position::Physical(PhysicalPosition::new(x, y))); +} + /// Generic bridge: forward an arbitrary engine command + params to the sidecar. /// /// This is the primary path the frontend uses. It keeps the Rust shell agnostic @@ -101,6 +131,59 @@ pub fn ensure_control_overlay_capture_excluded(app: AppHandle) -> Result<(), Str .map_err(|error| format!("this platform could not exclude the overlay: {error}")) } +/// Resize the overlay between a compact card, a pause card, and a click-through +/// monitor stage used only to paint a bound target ring. Capture exclusion is +/// never disabled here. +#[tauri::command] +pub fn set_control_overlay_layout(app: AppHandle, layout: String) -> Result<(), String> { + let window = app + .get_webview_window("control-overlay") + .ok_or_else(|| "control overlay window is unavailable".to_string())?; + match layout.as_str() { + "stage" => { + let Some(main) = app.get_webview_window("main") else { + return Err("main window is unavailable".to_string()); + }; + let monitor = main + .current_monitor() + .ok() + .flatten() + .or_else(|| app.primary_monitor().ok().flatten()) + .ok_or_else(|| "no monitor is available for the overlay stage".to_string())?; + let size = monitor.size(); + let position = monitor.position(); + window + .set_size(Size::Physical(PhysicalSize::new(size.width, size.height))) + .map_err(|error| format!("could not stage the control overlay: {error}"))?; + window + .set_position(Position::Physical(PhysicalPosition::new( + position.x, + position.y, + ))) + .map_err(|error| format!("could not place the overlay stage: {error}"))?; + } + "paused" => { + window + .set_size(Size::Logical(LogicalSize::new( + OVERLAY_PAUSED_WIDTH, + OVERLAY_PAUSED_HEIGHT, + ))) + .map_err(|error| format!("could not size the paused overlay: {error}"))?; + place_control_overlay(&app); + } + _ => { + window + .set_size(Size::Logical(LogicalSize::new( + OVERLAY_COMPACT_WIDTH, + OVERLAY_COMPACT_HEIGHT, + ))) + .map_err(|error| format!("could not size the compact overlay: {error}"))?; + place_control_overlay(&app); + } + } + Ok(()) +} + /// Open a URL in the user's default system browser. /// /// Used for the login deep-link ("open Settings -> Ingest tokens"), "Open cloud diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index c710944..c16e638 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -18,30 +18,7 @@ mod tray; use std::sync::Arc; use sidecar::{SidecarHandle, SidecarInner}; -use tauri::{Manager, PhysicalPosition, Position}; - -fn place_control_overlay(app: &tauri::AppHandle) { - let Some(main) = app.get_webview_window("main") else { - return; - }; - let Some(overlay) = app.get_webview_window("control-overlay") else { - return; - }; - let monitor = main - .current_monitor() - .ok() - .flatten() - .or_else(|| app.primary_monitor().ok().flatten()); - let (Some(monitor), Ok(overlay_size)) = (monitor, overlay.outer_size()) else { - return; - }; - let monitor_position = monitor.position(); - let monitor_size = monitor.size(); - let margin = (24.0 * monitor.scale_factor()).round() as i32; - let x = monitor_position.x + margin; - let y = monitor_position.y + monitor_size.height as i32 - overlay_size.height as i32 - margin; - let _ = overlay.set_position(Position::Physical(PhysicalPosition::new(x, y))); -} +use tauri::Manager; fn main() { let engine = Arc::new(SidecarInner::default()); @@ -101,7 +78,7 @@ fn main() { // when a retained target or protected region conflicts. The user // can still drag the native capsule, and show/hide transitions // never reposition or focus it. - place_control_overlay(app.handle()); + commands::place_control_overlay(app.handle()); // Spawn the frozen Python engine sidecar. Guarded: if the binary is // absent (frontend-only dev) the app still runs; the UI shows an @@ -123,6 +100,7 @@ fn main() { commands::set_control_overlay_visible, commands::set_control_overlay_interactive, commands::ensure_control_overlay_capture_excluded, + commands::set_control_overlay_layout, ffmpeg::ffmpeg_status, ffmpeg::retry_ffmpeg_provisioning, // typed convenience commands (forward to the sidecar) diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 6859ac9..d63fc2a 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -108,6 +108,10 @@ export const CMD = { PORTAL_DEVICES: "portal_devices", PORTAL_REVOKE_DEVICE: "portal_revoke_device", PORTAL_NOTIFICATION: "portal_notification", + // Local demonstration coach. Payload is never an evidence frame. + SET_COACH: "set_coach", + GET_COACH: "get_coach", + CLEAR_COACH: "clear_coach", } as const; /** Events the engine emits (delivered as Tauri events `engine://`). */ @@ -129,6 +133,8 @@ export const EVT = { PORTAL_STATE: "portal_state", // Carries only {title, body, open_count, route}; see attentionNotification.ts. ATTENTION_NOTIFICATION: "attention_notification", + // Local-only coach channel. Never persist this payload into evidence. + COACH: "coach", } as const; export type EngineEvent = (typeof EVT)[keyof typeof EVT]; @@ -190,6 +196,14 @@ export async function ensureControlOverlayCaptureExcluded(): Promise { await invoke("ensure_control_overlay_capture_excluded"); } +/** Compact card, taller pause card, or full-monitor click-through stage. */ +export async function setControlOverlayLayout( + layout: "compact" | "paused" | "stage", +): Promise { + if (!inTauri()) return; + await invoke("set_control_overlay_layout", { layout }); +} + /** Publish the post-composition preference; raw capture stays excluded. */ const FFMPEG_BROWSER_FALLBACK: FfmpegRuntimeStatus = { phase: "ready", diff --git a/src/overlay/ControlOverlay.test.tsx b/src/overlay/ControlOverlay.test.tsx index ef7b490..97822ab 100644 --- a/src/overlay/ControlOverlay.test.tsx +++ b/src/overlay/ControlOverlay.test.tsx @@ -4,11 +4,16 @@ import { engineInvoke, engineTry, ensureControlOverlayCaptureExcluded, + onEngineEvent, setControlOverlayInteractive, + setControlOverlayLayout, setControlOverlayVisible, } from "../lib/engine"; +import { AUTH_PAUSE_COPY } from "./coach"; import { ControlOverlay } from "./ControlOverlay"; +const engineHandlers: Record void> = {}; + vi.mock("../lib/engine", async (importOriginal) => { const original = await importOriginal(); return { @@ -17,8 +22,12 @@ vi.mock("../lib/engine", async (importOriginal) => { engineTry: vi.fn(), ensureControlOverlayCaptureExcluded: vi.fn(() => Promise.resolve()), inTauri: vi.fn(() => false), - onEngineEvent: vi.fn(() => Promise.resolve(() => {})), + onEngineEvent: vi.fn((event, handler) => { + engineHandlers[event] = handler; + return Promise.resolve(() => {}); + }), setControlOverlayInteractive: vi.fn(() => Promise.resolve()), + setControlOverlayLayout: vi.fn(() => Promise.resolve()), setControlOverlayVisible: vi.fn(() => Promise.resolve()), }; }); @@ -27,6 +36,7 @@ afterEach(() => { cleanup(); vi.clearAllMocks(); window.localStorage.clear(); + for (const key of Object.keys(engineHandlers)) delete engineHandlers[key]; }); it("keeps an active overlay capture-excluded and hides its controls", async () => { @@ -52,3 +62,125 @@ it("keeps an active overlay capture-excluded and hides its controls", async () = expect(screen.queryByRole("button")).toBeNull(); expect(engineInvoke).not.toHaveBeenCalled(); }); + +it("paints a local coach hint during recording without taking clicks", async () => { + vi.mocked(engineTry).mockResolvedValue({ + recording: true, + paused: false, + controls: { pause: false, resume: false, stop: true }, + }); + + render(); + expect(await screen.findByText("Recording demonstration")).toBeTruthy(); + await waitFor(() => expect(onEngineEvent).toHaveBeenCalled()); + + engineHandlers.coach?.({ + hint: "Open the claim screen", + turn: "your_turn", + }); + + expect(await screen.findByText("Open the claim screen")).toBeTruthy(); + expect(screen.getByText("Your turn")).toBeTruthy(); + expect(screen.getByText("LOCAL CAPTURE")).toBeTruthy(); + expect(screen.queryByText("VERIFIED")).toBeNull(); + expect(screen.queryByRole("button")).toBeNull(); + await waitFor(() => + expect(setControlOverlayInteractive).toHaveBeenCalledWith(false), + ); + expect(setControlOverlayInteractive).not.toHaveBeenCalledWith(true); + expect(document.querySelector(".overlay-target-ring")).toBeNull(); +}); + +it("opens auth pause controls only at a pause boundary", async () => { + vi.mocked(engineTry).mockResolvedValue({ + recording: true, + paused: false, + controls: { pause: false, resume: false, stop: true }, + }); + vi.mocked(engineInvoke).mockResolvedValue({ + schema_version: "openadapt.control-overlay-coach/v1", + hint: null, + turn: "your_turn", + pause_reason: null, + target: null, + operator_response: "continue", + }); + + render(); + expect(await screen.findByText("Recording demonstration")).toBeTruthy(); + await waitFor(() => expect(onEngineEvent).toHaveBeenCalled()); + + engineHandlers.coach?.({ + turn: "auth", + pause_reason: "auth", + }); + + expect(await screen.findByText(AUTH_PAUSE_COPY)).toBeTruthy(); + await waitFor(() => + expect(setControlOverlayInteractive).toHaveBeenCalledWith(true), + ); + expect(screen.getByRole("button", { name: "Continue OpenAdapt" })).toBeTruthy(); + expect(setControlOverlayLayout).toHaveBeenCalledWith("paused"); +}); + +it("does not draw a ghost ring without an exact observation binding", async () => { + vi.mocked(engineTry).mockResolvedValue({ + recording: true, + paused: false, + controls: { pause: false, resume: false, stop: true }, + }); + + render(); + expect(await screen.findByText("Recording demonstration")).toBeTruthy(); + await waitFor(() => expect(onEngineEvent).toHaveBeenCalled()); + + engineHandlers.coach?.({ + hint: "Open the claim screen", + turn: "your_turn", + target: { + coordinate_space: "top_level_viewport_normalized", + rect: { x: 0.2, y: 0.2, width: 0.15, height: 0.1 }, + }, + }); + + expect(await screen.findByText("Open the claim screen")).toBeTruthy(); + expect(document.querySelector(".overlay-target-ring")).toBeNull(); + expect(document.querySelector(".control-overlay-host.stage")).toBeNull(); +}); + +it("draws a click-through ring only when the rect is bound to an observation", async () => { + vi.mocked(engineTry).mockResolvedValue({ + recording: true, + paused: false, + controls: { pause: false, resume: false, stop: true }, + }); + + render(); + expect(await screen.findByText("Recording demonstration")).toBeTruthy(); + await waitFor(() => expect(onEngineEvent).toHaveBeenCalled()); + + engineHandlers.coach?.({ + hint: "Open the claim screen", + turn: "your_turn", + target: { + coordinate_space: "top_level_viewport_normalized", + rect: { x: 0.2, y: 0.25, width: 0.15, height: 0.1 }, + binding: { + kind: "observation_hmac_sha256", + observation_hmac_sha256: "ab".repeat(32), + }, + }, + }); + + expect(await screen.findByText("Open the claim screen")).toBeTruthy(); + expect(screen.queryByRole("button")).toBeNull(); + const ring = document.querySelector(".overlay-target-ring"); + expect(ring).toBeTruthy(); + expect(document.querySelector(".control-overlay-host.stage")).toBeTruthy(); + await waitFor(() => + expect(setControlOverlayLayout).toHaveBeenCalledWith("stage"), + ); + await waitFor(() => + expect(setControlOverlayInteractive).toHaveBeenCalledWith(false), + ); +}); diff --git a/src/overlay/ControlOverlay.tsx b/src/overlay/ControlOverlay.tsx index ba83919..0897ea4 100644 --- a/src/overlay/ControlOverlay.tsx +++ b/src/overlay/ControlOverlay.tsx @@ -9,6 +9,7 @@ import { inTauri, onEngineEvent, setControlOverlayInteractive, + setControlOverlayLayout, setControlOverlayVisible, } from "../lib/engine"; import type { @@ -18,6 +19,13 @@ import type { RunStep, Workflow, } from "../lib/types"; +import { + AUTH_PAUSE_COPY, + EMPTY_COACH, + coachHoldsPause, + overlayLayoutFor, + type CoachOperatorResponse, +} from "./coach"; import { buildControlOverlayFrame } from "./contract"; import { EMPTY_OVERLAY_STATE, @@ -119,6 +127,9 @@ export function ControlOverlay() { onEngineEvent(EVT.RUNNER_STATE, (status) => send({ kind: "runner-state", status }), ), + onEngineEvent(EVT.COACH, (payload) => + send({ kind: "coach", payload }), + ), onEngineEvent(EVT.LOG_LINE, (payload) => { // Only the bounded step ordinal is accepted. Never project the log // line, action, target, evidence, or typed value into the overlay. @@ -137,6 +148,14 @@ export function ControlOverlay() { const interactive = overlayAllowsInteraction(state.phase); const expanded = overlayExpands(state.phase); + const coach = state.coach; + const layout = overlayLayoutFor( + state.visible, + interactive, + expanded, + coach, + ); + const stage = layout === "stage"; useEffect(() => { if (!state.visible || state.startedAtUnixMs === null || expanded) return; @@ -176,6 +195,12 @@ export function ControlOverlay() { return () => window.clearTimeout(timeout); }, [state.phase, state.visible]); + useEffect(() => { + void setControlOverlayLayout(layout).catch((error) => { + console.error("Control overlay layout failed", error); + }); + }, [layout]); + useEffect(() => { if (!inTauri()) return; // Always broadcast the presentation-safe projection. A future deterministic @@ -190,6 +215,12 @@ export function ControlOverlay() { void emit("overlay://frame", safeFrame).catch(() => {}); }, [state]); + useEffect(() => { + if (!inTauri()) return; + // Local operator channel. Never persist, compose, ingest, or seal this. + void emit("overlay://coach", coach ?? EMPTY_COACH).catch(() => {}); + }, [coach]); + const stepLabel = useMemo(() => { if (state.currentStep !== null && state.totalSteps !== null) { return `Step ${state.currentStep} of ${state.totalSteps}`; @@ -215,6 +246,10 @@ export function ControlOverlay() { setControlError(false); send({ kind: "control-requested", action }); try { + if (action === "resume" && coachHoldsPause(coach)) { + send({ kind: "coach", payload: { operator_response: "continue" } }); + await engineInvoke(CMD.SET_COACH, { operator_response: "continue" }); + } const result = await engineInvoke( action === "pause" ? CMD.PAUSE_RECORDING @@ -234,116 +269,204 @@ export function ControlOverlay() { } } + async function respond(response: CoachOperatorResponse) { + if (!interactive || busy) return; + setBusy(true); + setControlError(false); + send({ kind: "coach", payload: { operator_response: response } }); + try { + await engineInvoke(CMD.SET_COACH, { operator_response: response }); + } catch { + setControlError(true); + } finally { + setBusy(false); + } + } + const showResume = state.phase === "paused"; + const coachPaused = coachHoldsPause(coach); const pauseAvailable = showResume ? state.controls.resume : state.controls.pause; const controlHelp = pauseAvailable ? undefined : "This operation does not currently advertise lossless pause or resume support."; + const authPause = + coachPaused && + (coach?.turn === "auth" || + coach?.pause_reason === "auth" || + coach?.pause_reason === "secret_field"); + const feedbackPause = + coachPaused && + (coach?.turn === "feedback" || + coach?.pause_reason === "wrong_step" || + coach?.pause_reason === "skip" || + coach?.pause_reason === "done"); + const target = stage ? coach?.target : null; + const continueLabel = coachPaused + ? "Continue" + : showResume + ? state.pausePrompt + ? "Continue" + : "Resume" + : "Pause"; return ( -
-
-
-
-