From dd8fb75d54720f143fde590dd1504ab8e294b450 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 19 Jun 2026 11:23:41 +0800 Subject: [PATCH] Add reactive screen observer (appear/vanish/change -> callback) --- README.md | 8 + README/README_zh-CN.md | 8 + README/README_zh-TW.md | 8 + .../Eng/doc/new_features/v17_features_doc.rst | 57 +++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v17_features_doc.rst | 53 +++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 + .../gui/script_builder/command_schema.py | 40 ++++ .../utils/executor/action_executor.py | 71 +++++++ .../utils/mcp_server/tools/_factories.py | 65 +++++- .../utils/mcp_server/tools/_handlers.py | 58 ++++++ je_auto_control/utils/observer/__init__.py | 12 ++ je_auto_control/utils/observer/observer.py | 196 ++++++++++++++++++ .../unit_test/headless/test_observer_batch.py | 105 ++++++++++ 15 files changed, 689 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v17_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v17_features_doc.rst create mode 100644 je_auto_control/utils/observer/__init__.py create mode 100644 je_auto_control/utils/observer/observer.py create mode 100644 test/unit_test/headless/test_observer_batch.py diff --git a/README.md b/README.md index 494f6630..2b71ebf1 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ ## Table of Contents +- [What's new (2026-06-19) — Reactive Observer](#whats-new-2026-06-19--reactive-observer) - [What's new (2026-06-19) — WCAG 2.2 Audit](#whats-new-2026-06-19--wcag-22-audit) - [What's new (2026-06-19) — Memory & Determinism](#whats-new-2026-06-19--memory--determinism) - [What's new (2026-06-19) — Office I/O](#whats-new-2026-06-19--office-io) @@ -69,6 +70,13 @@ --- +## What's new (2026-06-19) — Reactive Observer + +A non-blocking screen observer (SikuliX `observe` model), full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v17_features_doc.rst`](docs/source/Eng/doc/new_features/v17_features_doc.rst). + +- **`ScreenObserver`** (`AC_observe_add` / `AC_observe_remove` / `AC_observe_list` / `AC_observe_poll` / `AC_observe_start` / `AC_observe_stop`, `ac_observe_*`): register watches that fire on **appear** / **vanish** / **change** of an image/text/pixel and run a callback or action list — react to dialogs/progress/status while the main flow continues. +- **Testable by design** — detection is an injectable `predicate`; transition logic is unit-tested via `poll_once()` with synthetic values. Built-in `image_predicate` / `text_predicate` / `pixel_predicate` wrap the existing locate/OCR/pixel helpers. + ## What's new (2026-06-19) — WCAG 2.2 Audit The accessibility audit gains a WCAG 2.2 / EN 301 549 success-criterion layer, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v16_features_doc.rst`](docs/source/Eng/doc/new_features/v16_features_doc.rst). diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index ed27bec2..f660ebd0 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -12,6 +12,7 @@ ## 目录 +- [本次更新 (2026-06-19) — 反应式观察器](#本次更新-2026-06-19--反应式观察器) - [本次更新 (2026-06-19) — WCAG 2.2 审计](#本次更新-2026-06-19--wcag-22-审计) - [本次更新 (2026-06-19) — 记忆与确定性](#本次更新-2026-06-19--记忆与确定性) - [本次更新 (2026-06-19) — Office 读写](#本次更新-2026-06-19--office-读写) @@ -68,6 +69,13 @@ --- +## 本次更新 (2026-06-19) — 反应式观察器 + +非阻塞的屏幕观察器(SikuliX `observe` 模型),走完整五层(facade、`AC_*`、MCP、Script Builder)。完整参考:[`docs/source/Zh/doc/new_features/v17_features_doc.rst`](../docs/source/Zh/doc/new_features/v17_features_doc.rst)。 + +- **`ScreenObserver`**(`AC_observe_add` / `AC_observe_remove` / `AC_observe_list` / `AC_observe_poll` / `AC_observe_start` / `AC_observe_stop`、`ac_observe_*`):注册监看,在图像/文本/像素的 **appear** / **vanish** / **change** 时触发回调或执行 action list——在主流程继续的同时对对话框/进度/状态做出反应。 +- **为可测试而设计**——检测是可注入的 `predicate`;转换逻辑用 `poll_once()` 以合成值做单元测试。内建 `image_predicate` / `text_predicate` / `pixel_predicate` 包装既有的 locate/OCR/pixel 辅助函数。 + ## 本次更新 (2026-06-19) — WCAG 2.2 审计 无障碍审计新增 WCAG 2.2 / EN 301 549 成功准则层,走完整五层(facade、`AC_*`、MCP、Script Builder)。完整参考:[`docs/source/Zh/doc/new_features/v16_features_doc.rst`](../docs/source/Zh/doc/new_features/v16_features_doc.rst)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 272bb2c3..8ff86117 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -12,6 +12,7 @@ ## 目錄 +- [本次更新 (2026-06-19) — 反應式觀察器](#本次更新-2026-06-19--反應式觀察器) - [本次更新 (2026-06-19) — WCAG 2.2 稽核](#本次更新-2026-06-19--wcag-22-稽核) - [本次更新 (2026-06-19) — 記憶與決定性](#本次更新-2026-06-19--記憶與決定性) - [本次更新 (2026-06-19) — Office 讀寫](#本次更新-2026-06-19--office-讀寫) @@ -68,6 +69,13 @@ --- +## 本次更新 (2026-06-19) — 反應式觀察器 + +非阻塞的螢幕觀察器(SikuliX `observe` 模型),走完整五層(facade、`AC_*`、MCP、Script Builder)。完整參考:[`docs/source/Zh/doc/new_features/v17_features_doc.rst`](../docs/source/Zh/doc/new_features/v17_features_doc.rst)。 + +- **`ScreenObserver`**(`AC_observe_add` / `AC_observe_remove` / `AC_observe_list` / `AC_observe_poll` / `AC_observe_start` / `AC_observe_stop`、`ac_observe_*`):註冊監看,在影像/文字/像素的 **appear** / **vanish** / **change** 時觸發回呼或執行 action list——在主流程繼續的同時對對話框/進度/狀態做出反應。 +- **為可測試而設計**——偵測是可注入的 `predicate`;轉換邏輯用 `poll_once()` 以合成值做單元測試。內建 `image_predicate` / `text_predicate` / `pixel_predicate` 包裝既有的 locate/OCR/pixel 輔助函式。 + ## 本次更新 (2026-06-19) — WCAG 2.2 稽核 無障礙稽核新增 WCAG 2.2 / EN 301 549 成功準則層,走完整五層(facade、`AC_*`、MCP、Script Builder)。完整參考:[`docs/source/Zh/doc/new_features/v16_features_doc.rst`](../docs/source/Zh/doc/new_features/v16_features_doc.rst)。 diff --git a/docs/source/Eng/doc/new_features/v17_features_doc.rst b/docs/source/Eng/doc/new_features/v17_features_doc.rst new file mode 100644 index 00000000..738e0acb --- /dev/null +++ b/docs/source/Eng/doc/new_features/v17_features_doc.rst @@ -0,0 +1,57 @@ +================================================== +New Features (2026-06-19) — Reactive Observer +================================================== + +A non-blocking **screen observer**: register watches on a region/predicate +and get a callback (or run an action list) when the watched thing +**appears**, **vanishes**, or **changes**. This is the complement to the +blocking ``wait_for_*`` helpers — a flow can react to dialogs, progress, or +status changes *while doing other work* (the SikuliX ``observe`` model). + +Pure standard library; wired through the full stack (facade, ``AC_*`` +executor commands, MCP tools, Script Builder). + +.. contents:: + :local: + :depth: 2 + + +Python API +========= + +:: + + from je_auto_control import ScreenObserver, image_predicate, EVENT_APPEAR + + obs = ScreenObserver(poll_interval_s=0.5) + obs.add("error-dialog", + image_predicate("error.png", threshold=0.9), + on_event=lambda event, value: dismiss(), + events=(EVENT_APPEAR,)) + obs.start() # background polling thread + ... + obs.stop() + +Detection is decoupled from the screen: a watch's ``predicate`` just +returns the current value (truthy = present), so transition logic is +unit-tested with synthetic values via ``poll_once()``. Built-in predicate +builders — :func:`image_predicate`, :func:`text_predicate`, +:func:`pixel_predicate` — wrap the existing locate / OCR / pixel helpers. + +Transitions: ``appear`` (absent -> present), ``vanish`` (present -> +absent), ``change`` (present, value differs). Subscribe to a subset via +``events=``. + + +Executor / MCP commands +====================== + +* ``AC_observe_add`` — watch ``kind`` (``image`` / ``text`` / ``pixel``) + for ``event`` and run ``actions`` when it fires (the watchdog pattern, + generalised to screen content). +* ``AC_observe_remove`` / ``AC_observe_list`` — manage watches. +* ``AC_observe_poll`` — evaluate every watch once and return fired events + (deterministic, thread-free — ideal in scripts/tests). +* ``AC_observe_start`` / ``AC_observe_stop`` — background poll thread. + +The matching ``ac_observe_*`` MCP tools expose the same surface. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index f06e6062..91ce935d 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -39,6 +39,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v14_features_doc doc/new_features/v15_features_doc doc/new_features/v16_features_doc + doc/new_features/v17_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v17_features_doc.rst b/docs/source/Zh/doc/new_features/v17_features_doc.rst new file mode 100644 index 00000000..b4417915 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v17_features_doc.rst @@ -0,0 +1,53 @@ +========================================== +新功能 (2026-06-19) — 反應式觀察器 +========================================== + +非阻塞的**螢幕觀察器**:在區域/條件上註冊監看,當被監看的目標**出現**、 +**消失**或**改變**時,觸發回呼(或執行一段 action list)。這是阻塞式 +``wait_for_*`` 的互補——流程可以在做其他事的同時對對話框、進度或狀態 +變化做出反應(SikuliX 的 ``observe`` 模型)。 + +純標準庫;走完整五層(facade、``AC_*`` 執行器指令、MCP 工具、Script +Builder)。 + +.. contents:: + :local: + :depth: 2 + + +Python API +========== + +:: + + from je_auto_control import ScreenObserver, image_predicate, EVENT_APPEAR + + obs = ScreenObserver(poll_interval_s=0.5) + obs.add("error-dialog", + image_predicate("error.png", threshold=0.9), + on_event=lambda event, value: dismiss(), + events=(EVENT_APPEAR,)) + obs.start() # 背景輪詢執行緒 + ... + obs.stop() + +偵測與螢幕解耦:監看的 ``predicate`` 只回傳當前值(truthy 代表存在), +因此轉換邏輯可用合成值透過 ``poll_once()`` 做單元測試。內建的條件建構器 +——:func:`image_predicate`、:func:`text_predicate`、 +:func:`pixel_predicate`——包裝既有的 locate / OCR / pixel 輔助函式。 + +轉換:``appear``(不存在→存在)、``vanish``(存在→不存在)、``change`` +(存在且值改變)。可用 ``events=`` 只訂閱其中一部分。 + + +執行器 / MCP 指令 +================= + +* ``AC_observe_add`` — 監看 ``kind``(``image`` / ``text`` / ``pixel``) + 的 ``event``,觸發時執行 ``actions``(watchdog 模式,推廣到螢幕內容)。 +* ``AC_observe_remove`` / ``AC_observe_list`` — 管理監看。 +* ``AC_observe_poll`` — 評估每個監看一次並回傳觸發事件(決定性、免執行緒 + ——適合腳本/測試)。 +* ``AC_observe_start`` / ``AC_observe_stop`` — 背景輪詢執行緒。 + +對應的 ``ac_observe_*`` MCP 工具提供相同介面。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 20318bf3..1d679bcc 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -39,6 +39,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v14_features_doc doc/new_features/v15_features_doc doc/new_features/v16_features_doc + doc/new_features/v17_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 9c88171b..9058ab34 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -144,6 +144,11 @@ from je_auto_control.utils.deterministic import ( DeterministicRun, seed_everything, ) +# Reactive screen observer (appear / vanish / change -> callback) +from je_auto_control.utils.observer import ( + ScreenObserver, WatchRule, default_observer, + image_predicate, pixel_predicate, text_predicate, +) # Background popup/interrupt watchdog (unattended automation) from je_auto_control.utils.watchdog import ( PopupWatchdog, WatchdogRule, default_popup_watchdog, @@ -559,6 +564,8 @@ def start_autocontrol_gui(*args, **kwargs): "read_presentation", "write_presentation", "AgentMemory", "Episode", "DeterministicRun", "seed_everything", + "ScreenObserver", "WatchRule", "default_observer", + "image_predicate", "pixel_predicate", "text_predicate", # MCP server "AuditLogger", "HttpMCPServer", "MCPContent", "MCPPrompt", "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 14f53fae..589d0e44 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -669,6 +669,46 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="WCAG 2.2 audit: SC-tagged findings + Target Size 2.5.8.", )) + _add_observer_specs(specs) + + +def _add_observer_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_observe_add", "Flow", "Observe: Add Watch", + fields=( + FieldSpec("name", FieldType.STRING), + FieldSpec("kind", FieldType.ENUM, + choices=("image", "text", "pixel"), default="image"), + FieldSpec("event", FieldType.ENUM, + choices=("appear", "vanish", "change"), + default="appear"), + FieldSpec("image", FieldType.FILE_PATH, optional=True), + FieldSpec("threshold", FieldType.FLOAT, optional=True, + default=0.8), + FieldSpec("text", FieldType.STRING, optional=True), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + description="Run 'actions' (JSON view) on appear/vanish/change of an " + "image/text/pixel.", + )) + specs.append(CommandSpec( + "AC_observe_remove", "Flow", "Observe: Remove Watch", + fields=(FieldSpec("name", FieldType.STRING),), + description="Remove a registered watch.", + )) + specs.append(CommandSpec( + "AC_observe_list", "Flow", "Observe: List Watches", + description="List registered watch names.")) + specs.append(CommandSpec( + "AC_observe_poll", "Flow", "Observe: Poll Once", + description="Evaluate all watches once; return fired events.")) + specs.append(CommandSpec( + "AC_observe_start", "Flow", "Observe: Start", + description="Start the background observer thread.")) + specs.append(CommandSpec( + "AC_observe_stop", "Flow", "Observe: Stop", + description="Stop the background observer thread.")) def _add_memory_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 26e2d46e..f0d63d02 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2572,6 +2572,71 @@ def _seed_everything(seed: int = 0) -> Dict[str, Any]: return {"seed": seed_everything(int(seed))} +def _observe_handler(actions: List[Any]) -> Callable[[str, Any], None]: + """Build an observer callback that runs an action list on each event.""" + def handler(_event: str, _value: Any) -> None: + if actions: + executor.execute_action(list(actions)) + return handler + + +def _observe_predicate(kind: str, params: Dict[str, Any]): + from je_auto_control.utils.observer import ( + image_predicate, pixel_predicate, text_predicate) + builders = { + "image": lambda: image_predicate(params.get("image", ""), + params.get("threshold", 0.8)), + "text": lambda: text_predicate(params.get("text", "")), + "pixel": lambda: pixel_predicate(int(params.get("x", 0)), + int(params.get("y", 0))), + } + if kind not in builders: + raise AutoControlActionException(f"unknown observe kind: {kind!r}") + return builders[kind]() + + +def _observe_add(name: str, kind: str = "image", event: str = "appear", + actions: Optional[List[Any]] = None, + **params: Any) -> Dict[str, Any]: + """Adapter: watch image/text/pixel; run ``actions`` on the event.""" + from je_auto_control.utils.observer import default_observer + default_observer.add(name, _observe_predicate(kind, params), + _observe_handler(actions or []), events=(event,)) + return {"name": name, "kind": kind, "event": event} + + +def _observe_remove(name: str) -> Dict[str, Any]: + """Adapter: remove a registered watch.""" + from je_auto_control.utils.observer import default_observer + return {"removed": default_observer.remove(name)} + + +def _observe_list() -> Dict[str, Any]: + """Adapter: list registered watch names.""" + from je_auto_control.utils.observer import default_observer + return {"names": default_observer.names()} + + +def _observe_poll() -> Dict[str, Any]: + """Adapter: evaluate all watches once; return fired events.""" + from je_auto_control.utils.observer import default_observer + return {"fired": default_observer.poll_once()} + + +def _observe_start() -> Dict[str, Any]: + """Adapter: start the background observer thread.""" + from je_auto_control.utils.observer import default_observer + default_observer.start() + return {"running": default_observer.running} + + +def _observe_stop() -> Dict[str, Any]: + """Adapter: stop the background observer thread.""" + from je_auto_control.utils.observer import default_observer + default_observer.stop() + return {"running": default_observer.running} + + class Executor: """ Executor @@ -2764,6 +2829,12 @@ def __init__(self): "AC_memory_forget": _memory_forget, "AC_memory_stats": _memory_stats, "AC_seed_everything": _seed_everything, + "AC_observe_add": _observe_add, + "AC_observe_remove": _observe_remove, + "AC_observe_list": _observe_list, + "AC_observe_poll": _observe_poll, + "AC_observe_start": _observe_start, + "AC_observe_stop": _observe_stop, "AC_a11y_record_start": _a11y_record_start, "AC_a11y_record_stop": _a11y_record_stop, "AC_a11y_record_events": _a11y_record_events, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index a8595606..a4d0618b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2071,6 +2071,69 @@ def determinism_tools() -> List[MCPTool]: ] +def observer_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_observe_add", + description=("Register a non-blocking watch that runs 'actions' " + "when an image/text/pixel appears, vanishes, or " + "changes. kind=image|text|pixel; event=appear|vanish|" + "change. Provide image+threshold, text, or x+y."), + input_schema=schema({ + "name": {"type": "string"}, + "kind": {"type": "string", + "enum": ["image", "text", "pixel"]}, + "event": {"type": "string", + "enum": ["appear", "vanish", "change"]}, + "actions": {"type": "array"}, + "image": {"type": "string"}, + "threshold": {"type": "number"}, + "text": {"type": "string"}, + "x": {"type": "integer"}, "y": {"type": "integer"}, + }, required=["name"]), + handler=h.observe_add, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_observe_remove", + description="Remove a registered watch by name; returns {removed}.", + input_schema=schema({"name": {"type": "string"}}, + required=["name"]), + handler=h.observe_remove, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_observe_list", + description="List registered watch names.", + input_schema=schema({}), + handler=h.observe_list, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_observe_poll", + description=("Evaluate all watches once and return fired events " + "({rule, event, time}); useful without the thread."), + input_schema=schema({}), + handler=h.observe_poll, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_observe_start", + description="Start the background observer poll thread.", + input_schema=schema({}), + handler=h.observe_start, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_observe_stop", + description="Stop the background observer poll thread.", + input_schema=schema({}), + handler=h.observe_stop, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def unattended_tools() -> List[MCPTool]: return [ MCPTool( @@ -3123,7 +3186,7 @@ def media_assert_tools() -> List[MCPTool]: synthetic_data_tools, mcp_registry_tools, test_selection_tools, element_repository_tools, flow_debugger_tools, skill_library_tools, guardrail_tools, a2a_tools, office_tools, - agent_memory_tools, determinism_tools, + agent_memory_tools, determinism_tools, observer_tools, screen_record_tools, process_and_shell_tools, remote_desktop_tools, gamepad_tools, usb_passthrough_tools, assertion_tools, data_source_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index c96972e9..a1e69d1c 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -996,6 +996,64 @@ def seed_everything(seed=0): return {"seed": _seed(int(seed))} +def _observe_predicate(kind, params): + from je_auto_control.utils.observer import ( + image_predicate, pixel_predicate, text_predicate) + builders = { + "image": lambda: image_predicate(params.get("image", ""), + params.get("threshold", 0.8)), + "text": lambda: text_predicate(params.get("text", "")), + "pixel": lambda: pixel_predicate(int(params.get("x", 0)), + int(params.get("y", 0))), + } + if kind not in builders: + raise ValueError(f"unknown observe kind: {kind!r}") + return builders[kind]() + + +def _observe_handler(actions): + from je_auto_control.utils.executor.action_executor import executor + + def handler(_event, _value): + if actions: + executor.execute_action(list(actions)) + return handler + + +def observe_add(name, kind="image", event="appear", actions=None, **params): + from je_auto_control.utils.observer import default_observer + default_observer.add(name, _observe_predicate(kind, params), + _observe_handler(actions or []), events=(event,)) + return {"name": name, "kind": kind, "event": event} + + +def observe_remove(name): + from je_auto_control.utils.observer import default_observer + return {"removed": default_observer.remove(name)} + + +def observe_list(): + from je_auto_control.utils.observer import default_observer + return {"names": default_observer.names()} + + +def observe_poll(): + from je_auto_control.utils.observer import default_observer + return {"fired": default_observer.poll_once()} + + +def observe_start(): + from je_auto_control.utils.observer import default_observer + default_observer.start() + return {"running": default_observer.running} + + +def observe_stop(): + from je_auto_control.utils.observer import default_observer + default_observer.stop() + return {"running": default_observer.running} + + def vlm_locate(description: str, screen_region: Optional[List[int]] = None, model: Optional[str] = None) -> Optional[List[int]]: diff --git a/je_auto_control/utils/observer/__init__.py b/je_auto_control/utils/observer/__init__.py new file mode 100644 index 00000000..0bd65825 --- /dev/null +++ b/je_auto_control/utils/observer/__init__.py @@ -0,0 +1,12 @@ +"""Reactive screen observer — fire on appear / vanish / change.""" +from je_auto_control.utils.observer.observer import ( + EVENT_APPEAR, EVENT_CHANGE, EVENT_VANISH, + ScreenObserver, WatchRule, default_observer, + image_predicate, pixel_predicate, text_predicate, +) + +__all__ = [ + "EVENT_APPEAR", "EVENT_CHANGE", "EVENT_VANISH", + "ScreenObserver", "WatchRule", "default_observer", + "image_predicate", "pixel_predicate", "text_predicate", +] diff --git a/je_auto_control/utils/observer/observer.py b/je_auto_control/utils/observer/observer.py new file mode 100644 index 00000000..bf392502 --- /dev/null +++ b/je_auto_control/utils/observer/observer.py @@ -0,0 +1,196 @@ +"""Reactive screen observer — fire a callback when something changes. + +The blocking ``wait_for_*`` helpers cover "pause until X appears". This is +the complementary *non-blocking* model (SikuliX's ``observe``): register +watches on a region/predicate and get a callback when the watched thing +**appears**, **vanishes**, or **changes** — so a flow can react to dialogs, +progress, or status changes while doing other work. + +A :class:`WatchRule` pairs a ``predicate`` (returns the current value / +presence) with an ``on_event`` callback. Detection is decoupled from the +screen: predicates are injectable, so transition logic is unit-tested with +synthetic values via :meth:`ScreenObserver.poll_once`; :meth:`start` adds an +optional background polling thread. Imports no ``PySide6`` — fully headless. +""" +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +EVENT_APPEAR = "appear" +EVENT_VANISH = "vanish" +EVENT_CHANGE = "change" +_ALL_EVENTS = (EVENT_APPEAR, EVENT_VANISH, EVENT_CHANGE) + +# Errors a predicate/handler may raise that must not kill the poll loop. +_RULE_ERRORS = (OSError, RuntimeError, ValueError, AttributeError, TypeError, + AutoControlException) +_UNSET = object() + + +@dataclass +class WatchRule: + """Pair a predicate with the callback to fire on its transitions.""" + name: str + predicate: Callable[[], Any] + on_event: Callable[[str, Any], None] + events: Sequence[str] = _ALL_EVENTS + last: Any = _UNSET + + +def _transition(last: Any, value: Any) -> Optional[str]: + """Classify the change from ``last`` to ``value`` as an event name.""" + now = bool(value) + if last is _UNSET: + return EVENT_APPEAR if now else None + was = bool(last) + if now and not was: + return EVENT_APPEAR + if was and not now: + return EVENT_VANISH + if now and was and value != last: + return EVENT_CHANGE + return None + + +class ScreenObserver: + """Poll registered watches and fire callbacks on appear/vanish/change.""" + + def __init__(self, poll_interval_s: float = 0.5) -> None: + self._poll = max(0.05, float(poll_interval_s)) + self._rules: List[WatchRule] = [] + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._stop = threading.Event() + self._events: List[Dict[str, Any]] = [] + + def add(self, name: str, predicate: Callable[[], Any], + on_event: Callable[[str, Any], None], *, + events: Optional[Sequence[str]] = None) -> WatchRule: + """Register a watch; ``events`` defaults to all three transitions.""" + rule = WatchRule(name=name, predicate=predicate, on_event=on_event, + events=tuple(events) if events else _ALL_EVENTS) + with self._lock: + self._rules.append(rule) + return rule + + def remove(self, name: str) -> bool: + """Remove every watch with ``name``; return whether any matched.""" + with self._lock: + before = len(self._rules) + self._rules = [r for r in self._rules if r.name != name] + return len(self._rules) < before + + def clear(self) -> None: + """Remove all watches.""" + with self._lock: + self._rules.clear() + + def names(self) -> List[str]: + """Return the registered watch names.""" + with self._lock: + return [rule.name for rule in self._rules] + + @property + def running(self) -> bool: + """Whether the background poll thread is alive.""" + return self._thread is not None and self._thread.is_alive() + + @property + def fired(self) -> List[Dict[str, Any]]: + """Log of fired events (``{rule, event, time}``).""" + with self._lock: + return list(self._events) + + def poll_once(self) -> List[Dict[str, Any]]: + """Evaluate every watch once; fire callbacks and return the events.""" + with self._lock: + rules = list(self._rules) + return [event for event in (self._evaluate(rule) for rule in rules) + if event is not None] + + def _evaluate(self, rule: WatchRule) -> Optional[Dict[str, Any]]: + try: + value = rule.predicate() + except _RULE_ERRORS as error: + autocontrol_logger.info( + "observer %r predicate error: %r", rule.name, error) + return None + event = _transition(rule.last, value) + rule.last = value + if event is None or event not in rule.events: + return None + self._fire(rule, event, value) + record = {"rule": rule.name, "event": event, "time": time.time()} + with self._lock: + self._events.append(record) + return record + + def _fire(self, rule: WatchRule, event: str, value: Any) -> None: + try: + rule.on_event(event, value) + except _RULE_ERRORS as error: + autocontrol_logger.info( + "observer %r handler error: %r", rule.name, error) + + def start(self) -> None: + """Start the background poll thread (idempotent).""" + if self.running: + return + self._stop.clear() + self._thread = threading.Thread( + target=self._loop, name="screen-observer", daemon=True) + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + """Signal the poll thread to stop and join it.""" + self._stop.set() + thread = self._thread + if thread is not None: + thread.join(timeout=float(timeout)) + self._thread = None + + def _loop(self) -> None: + while not self._stop.is_set(): + self.poll_once() + self._stop.wait(self._poll) + + +def image_predicate(image: str, threshold: float = 0.8) -> Callable[[], Any]: + """Predicate: the located image centre, or ``None`` when absent.""" + def predicate() -> Any: + from je_auto_control.wrapper.auto_control_image import ( + locate_image_center) + try: + return locate_image_center(image, detect_threshold=float(threshold)) + except _RULE_ERRORS: + return None + return predicate + + +def text_predicate(text: str) -> Callable[[], Any]: + """Predicate: the located text centre, or ``None`` when absent.""" + def predicate() -> Any: + from je_auto_control.utils.ocr.ocr_engine import locate_text_center + try: + return locate_text_center(text) + except _RULE_ERRORS: + return None + return predicate + + +def pixel_predicate(x: int, y: int) -> Callable[[], Any]: + """Predicate: the RGB tuple at ``(x, y)`` — use with the 'change' event.""" + def predicate() -> Any: + from je_auto_control.wrapper.auto_control_screen import get_pixel + try: + return tuple(get_pixel(int(x), int(y))) + except _RULE_ERRORS: + return None + return predicate + + +default_observer = ScreenObserver() diff --git a/test/unit_test/headless/test_observer_batch.py b/test/unit_test/headless/test_observer_batch.py new file mode 100644 index 00000000..7a9dca5c --- /dev/null +++ b/test/unit_test/headless/test_observer_batch.py @@ -0,0 +1,105 @@ +"""Headless tests for the reactive screen observer. Pure stdlib; detection +is injected via fake predicates so no real screen is required.""" +import je_auto_control as ac +from je_auto_control.utils.observer import ( + EVENT_APPEAR, EVENT_CHANGE, EVENT_VANISH, ScreenObserver) + + +class _Source: + """A mutable predicate source for driving transitions in tests.""" + + def __init__(self, value=None): + self.value = value + + def __call__(self): + return self.value + + +def test_appear_vanish_change_transitions(): + obs = ScreenObserver() + src = _Source(None) + seen = [] + obs.add("w", src, lambda event, value: seen.append(event)) + + assert obs.poll_once() == [] # absent -> no event + src.value = (10, 20) + assert obs.poll_once()[0]["event"] == EVENT_APPEAR + assert obs.poll_once() == [] # unchanged -> no event + src.value = (30, 40) + assert obs.poll_once()[0]["event"] == EVENT_CHANGE + src.value = None + assert obs.poll_once()[0]["event"] == EVENT_VANISH + assert seen == [EVENT_APPEAR, EVENT_CHANGE, EVENT_VANISH] + + +def test_event_filter_only_fires_selected(): + obs = ScreenObserver() + src = _Source(None) + fired = [] + obs.add("only-appear", src, lambda e, v: fired.append(e), + events=(EVENT_APPEAR,)) + src.value = "here" + obs.poll_once() + src.value = None + obs.poll_once() # vanish ignored (not subscribed) + assert fired == [EVENT_APPEAR] + + +def test_predicate_error_does_not_break_loop(): + obs = ScreenObserver() + + def boom(): + raise RuntimeError("predicate failed") + + obs.add("bad", boom, lambda e, v: None) + assert obs.poll_once() == [] # error swallowed, no crash + + +def test_remove_and_names_and_fired_log(): + obs = ScreenObserver() + obs.add("a", _Source("x"), lambda e, v: None) + obs.add("b", _Source(None), lambda e, v: None) + assert set(obs.names()) == {"a", "b"} + obs.poll_once() + assert obs.fired and obs.fired[-1]["rule"] == "a" + assert obs.remove("a") is True + assert obs.remove("a") is False + assert obs.names() == ["b"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_wiring(): + from je_auto_control.utils.observer import default_observer + default_observer.clear() + ac.execute_action([["AC_observe_add", { + "name": "img", "kind": "image", "event": "appear", + "image": "missing.png", "actions": []}]]) + listing = ac.execute_action([["AC_observe_list", {}]]) + assert any("img" in str(v) for v in listing.values()) + # poll is safe even though the image backend isn't available headless + polled = ac.execute_action([["AC_observe_poll", {}]]) + assert any("fired" in str(v) for v in polled.values()) + ac.execute_action([["AC_observe_remove", {"name": "img"}]]) + known = ac.executor.known_commands() + assert {"AC_observe_add", "AC_observe_remove", "AC_observe_list", + "AC_observe_poll", "AC_observe_start", "AC_observe_stop"} <= known + + +def test_mcp_and_builder_wiring(): + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_observe_add", "ac_observe_remove", "ac_observe_list", + "ac_observe_poll", "ac_observe_start", "ac_observe_stop"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + cmds = {s.command for s in _build_specs()} + assert {"AC_observe_add", "AC_observe_remove", "AC_observe_list", + "AC_observe_poll", "AC_observe_start", "AC_observe_stop"} <= cmds + + +def test_facade_exports(): + for attr in ("ScreenObserver", "WatchRule", "default_observer", + "image_predicate", "pixel_predicate", "text_predicate"): + assert hasattr(ac, attr) + assert attr in ac.__all__