From 30169e790f209a44cae173de22987e2c51ff4a9a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 19 Jun 2026 07:36:27 +0800 Subject: [PATCH 1/2] Add synthetic test data, MCP registry manifest, risk-based test selection --- README.md | 9 + README/README_zh-CN.md | 9 + README/README_zh-TW.md | 9 + .../Eng/doc/new_features/v11_features_doc.rst | 73 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v11_features_doc.rst | 72 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 11 + .../gui/script_builder/command_schema.py | 44 ++++ .../utils/executor/action_executor.py | 46 ++++ .../utils/mcp_registry/__init__.py | 6 + .../utils/mcp_registry/registry.py | 86 ++++++++ .../utils/mcp_server/tools/_factories.py | 69 ++++++ .../utils/mcp_server/tools/_handlers.py | 31 +++ je_auto_control/utils/test_data/__init__.py | 6 + je_auto_control/utils/test_data/test_data.py | 199 ++++++++++++++++++ je_auto_control/utils/test_select/__init__.py | 6 + .../utils/test_select/test_select.py | 115 ++++++++++ test/unit_test/headless/test_tooling_batch.py | 125 +++++++++++ 19 files changed, 918 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v11_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v11_features_doc.rst create mode 100644 je_auto_control/utils/mcp_registry/__init__.py create mode 100644 je_auto_control/utils/mcp_registry/registry.py create mode 100644 je_auto_control/utils/test_data/__init__.py create mode 100644 je_auto_control/utils/test_data/test_data.py create mode 100644 je_auto_control/utils/test_select/__init__.py create mode 100644 je_auto_control/utils/test_select/test_select.py create mode 100644 test/unit_test/headless/test_tooling_batch.py diff --git a/README.md b/README.md index d580298f..59c338b8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ ## Table of Contents +- [What's new (2026-06-19) — Test & Tooling Batch](#whats-new-2026-06-19--test--tooling-batch) - [What's new (2026-06-19) — Transactional Queue](#whats-new-2026-06-19--transactional-queue) - [What's new (2026-06-19) — Unattended Reliability](#whats-new-2026-06-19--unattended-reliability) - [What's new (2026-06-19) — Popup Watchdog](#whats-new-2026-06-19--popup-watchdog) @@ -63,6 +64,14 @@ --- +## What's new (2026-06-19) — Test & Tooling Batch + +Three pure-stdlib quality-of-life tools, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v11_features_doc.rst`](docs/source/Eng/doc/new_features/v11_features_doc.rst). + +- **Synthetic test data** — `generate_rows(schema, count, seed=...)` / `write_dataset` (`AC_generate_data`, `ac_generate_data`): deterministic fake rows (name/email/phone/int/choice/date…) to drive data-driven runs without real PII; no Faker. +- **MCP registry manifest** — `write_server_manifest("server.json", include_tools=True)` (`AC_mcp_manifest`, `ac_mcp_manifest`): publish a registry-valid `server.json` so MCP agents/IDEs can discover this server. +- **Risk-based test selection** — `rank_flows` / `select_flows` (`AC_rank_tests` / `AC_select_tests`): rank flows by recent failures, flakiness, staleness and never-run from run history; run the riskiest first or only the top-k. + ## What's new (2026-06-19) — Transactional Queue Turn AutoControl from "run a script" into "run a robot." A SQLite-backed work queue implements the production-RPA dispatcher/performer pattern: enqueue items, process one at a time with per-item status, dedup and retry, so a run of thousands is **resumable after a crash** and parallelizable. Pure stdlib, full stack. Full reference: [`docs/source/Eng/doc/new_features/v10_features_doc.rst`](docs/source/Eng/doc/new_features/v10_features_doc.rst). diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index a826ee6f..e2ad4710 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -12,6 +12,7 @@ ## 目录 +- [本次更新 (2026-06-19) — 测试与工具三件套](#本次更新-2026-06-19--测试与工具三件套) - [本次更新 (2026-06-19) — 事务式工作队列](#本次更新-2026-06-19--事务式工作队列) - [本次更新 (2026-06-19) — 无人值守可靠性](#本次更新-2026-06-19--无人值守可靠性) - [本次更新 (2026-06-19) — 弹窗看门狗](#本次更新-2026-06-19--弹窗看门狗) @@ -62,6 +63,14 @@ --- +## 本次更新 (2026-06-19) — 测试与工具三件套 + +三项纯标准库的生产力工具,走完整五层(facade、`AC_*`、MCP、Script Builder)。完整参考:[`docs/source/Zh/doc/new_features/v11_features_doc.rst`](../docs/source/Zh/doc/new_features/v11_features_doc.rst)。 + +- **合成测试数据** — `generate_rows(schema, count, seed=...)` / `write_dataset`(`AC_generate_data`、`ac_generate_data`):生成可重现的假数据行(name/email/phone/int/choice/date…),驱动数据驱动执行而不需真实 PII;无需 Faker。 +- **MCP registry 清单** — `write_server_manifest("server.json", include_tools=True)`(`AC_mcp_manifest`、`ac_mcp_manifest`):生成符合 registry 规范的 `server.json`,让 MCP agent/IDE 能发现此服务器。 +- **风险导向测试选择** — `rank_flows` / `select_flows`(`AC_rank_tests` / `AC_select_tests`):依最近失败、不稳定、陈旧与从未跑过,从 run history 排序流程;先跑最高风险或只跑前 k 个。 + ## 本次更新 (2026-06-19) — 事务式工作队列 把 AutoControl 从「跑脚本」升级成「跑机器人」。以 SQLite 为底的工作队列实作生产级 RPA dispatcher/performer:入列项目、一次处理一项、具每项状态/去重/重试,使上千项执行能**崩溃后续跑**且可并行化。纯标准库、走完整五层。完整参考:[`docs/source/Eng/doc/new_features/v10_features_doc.rst`](../docs/source/Eng/doc/new_features/v10_features_doc.rst)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 36ec2782..59724870 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -12,6 +12,7 @@ ## 目錄 +- [本次更新 (2026-06-19) — 測試與工具三件套](#本次更新-2026-06-19--測試與工具三件套) - [本次更新 (2026-06-19) — 交易式工作佇列](#本次更新-2026-06-19--交易式工作佇列) - [本次更新 (2026-06-19) — 無人值守可靠性](#本次更新-2026-06-19--無人值守可靠性) - [本次更新 (2026-06-19) — 彈窗看門狗](#本次更新-2026-06-19--彈窗看門狗) @@ -62,6 +63,14 @@ --- +## 本次更新 (2026-06-19) — 測試與工具三件套 + +三項純標準庫的生產力工具,走完整五層(facade、`AC_*`、MCP、Script Builder)。完整參考:[`docs/source/Zh/doc/new_features/v11_features_doc.rst`](../docs/source/Zh/doc/new_features/v11_features_doc.rst)。 + +- **合成測試資料** — `generate_rows(schema, count, seed=...)` / `write_dataset`(`AC_generate_data`、`ac_generate_data`):產生可重現的假資料列(name/email/phone/int/choice/date…),驅動資料驅動執行而不需真實 PII;不需 Faker。 +- **MCP registry 清單** — `write_server_manifest("server.json", include_tools=True)`(`AC_mcp_manifest`、`ac_mcp_manifest`):產生符合 registry 規範的 `server.json`,讓 MCP agent/IDE 能發現此伺服器。 +- **風險導向測試選擇** — `rank_flows` / `select_flows`(`AC_rank_tests` / `AC_select_tests`):依最近失敗、不穩定、陳舊與從未跑過,從 run history 排序流程;先跑最高風險或只跑前 k 個。 + ## 本次更新 (2026-06-19) — 交易式工作佇列 把 AutoControl 從「跑腳本」升級成「跑機器人」。以 SQLite 為底的工作佇列實作生產級 RPA dispatcher/performer:入列項目、一次處理一項、具每項狀態/去重/重試,使上千項執行能**當機後續跑**且可平行化。純標準庫、走完整五層。完整參考:[`docs/source/Zh/doc/new_features/v10_features_doc.rst`](../docs/source/Zh/doc/new_features/v10_features_doc.rst)。 diff --git a/docs/source/Eng/doc/new_features/v11_features_doc.rst b/docs/source/Eng/doc/new_features/v11_features_doc.rst new file mode 100644 index 00000000..892b475c --- /dev/null +++ b/docs/source/Eng/doc/new_features/v11_features_doc.rst @@ -0,0 +1,73 @@ +================================================== +New Features (2026-06-19) — Test & Tooling Batch +================================================== + +Three quality-of-life tools, all pure standard library and wired through +the full stack (facade, ``AC_*`` executor commands, MCP tools, Script +Builder): seeded synthetic test data, an MCP registry ``server.json`` +generator, and risk-based test selection. + +.. contents:: + :local: + :depth: 2 + + +Synthetic test data +=================== + +Generate deterministic fake rows from a tiny field schema — to drive +data-driven runs without shipping real PII. No Faker dependency; the same +``seed`` always yields the same rows:: + + from je_auto_control import generate_rows, write_dataset + + rows = generate_rows({ + "name": "name", + "email": {"type": "email", "domain": "acme.test"}, + "age": {"type": "int", "min": 18, "max": 65}, + "status": {"type": "choice", "choices": ["new", "vip"]}, + }, count=100, seed=7) + + write_dataset(rows, "people.csv") # or .json + +Supported field types: ``first_name``, ``last_name``, ``name``, +``username``, ``email``, ``phone``, ``city``, ``company``, ``word``, +``sentence``, ``uuid``, ``bool``, ``int`` (min/max), ``float`` +(min/max/ndigits), ``choice`` (choices), ``date`` (start/end). + +The ``AC_generate_data`` command writes a file (then feed it to +``AC_load_data``) or returns the rows inline. + + +MCP registry manifest +==================== + +Publish a ``server.json`` describing this AutoControl MCP server so +MCP-aware agents and IDEs can discover and install it. The manifest is +built from live package metadata, so it never drifts:: + + from je_auto_control import write_server_manifest + + write_server_manifest("server.json", include_tools=True) + +``include_tools`` embeds the live tool list under ``_meta`` (without +touching the registry-valid core fields). Also exposed as +``AC_mcp_manifest`` and the ``ac_mcp_manifest`` MCP tool. + + +Risk-based test selection +======================== + +Instead of always running the whole suite, rank flows by how *risky* they +are — recently failing, flaky, stale, or never-run — using the run-history +store, then run the riskiest first (or only the top-k):: + + from je_auto_control import select_flows, rank_flows + + ranked = rank_flows(["login", "checkout", "report"]) + risky = select_flows(["login", "checkout", "report"], k=2) + +The score is ``0.5*failure_rate + 0.2*last_failed + 0.2*flakiness + +0.1*staleness``; a never-run flow scores ``0.8`` (untested is risky). +Exposed as ``AC_rank_tests`` / ``AC_select_tests`` and the +``ac_rank_tests`` / ``ac_select_tests`` MCP tools. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index b0a6be85..99807e44 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -33,6 +33,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v8_features_doc doc/new_features/v9_features_doc doc/new_features/v10_features_doc + doc/new_features/v11_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v11_features_doc.rst b/docs/source/Zh/doc/new_features/v11_features_doc.rst new file mode 100644 index 00000000..0957adf8 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v11_features_doc.rst @@ -0,0 +1,72 @@ +============================================ +新功能 (2026-06-19) — 測試與工具三件套 +============================================ + +三項提升生產力的工具,皆為純標準庫,並走完整五層(facade、``AC_*`` +執行器指令、MCP 工具、Script Builder):有種子的合成測試資料、MCP +registry ``server.json`` 產生器,以及風險導向的測試選擇。 + +.. contents:: + :local: + :depth: 2 + + +合成測試資料 +============ + +依一個極小的欄位 schema 產生**可重現**的假資料列——用來驅動資料驅動的 +執行,而不必散布真實 PII。不需要 Faker;相同的 ``seed`` 永遠產生相同的 +資料列:: + + from je_auto_control import generate_rows, write_dataset + + rows = generate_rows({ + "name": "name", + "email": {"type": "email", "domain": "acme.test"}, + "age": {"type": "int", "min": 18, "max": 65}, + "status": {"type": "choice", "choices": ["new", "vip"]}, + }, count=100, seed=7) + + write_dataset(rows, "people.csv") # 或 .json + +支援的欄位型別:``first_name``、``last_name``、``name``、``username``、 +``email``、``phone``、``city``、``company``、``word``、``sentence``、 +``uuid``、``bool``、``int``(min/max)、``float``(min/max/ndigits)、 +``choice``(choices)、``date``(start/end)。 + +``AC_generate_data`` 指令會寫出檔案(再交給 ``AC_load_data``)或直接 +回傳資料列。 + + +MCP registry 清單 +================= + +產生描述此 AutoControl MCP 伺服器的 ``server.json``,讓支援 MCP 的 +agent 與 IDE 能發現並安裝它。清單由即時套件中繼資料建構,因此不會與 +實際能力脫節:: + + from je_auto_control import write_server_manifest + + write_server_manifest("server.json", include_tools=True) + +``include_tools`` 會把即時工具清單嵌入 ``_meta``(不更動 registry 規範 +的核心欄位)。同時提供 ``AC_mcp_manifest`` 與 ``ac_mcp_manifest`` MCP +工具。 + + +風險導向測試選擇 +================ + +與其每次都跑整套測試,不如依據流程的**風險**排序——最近失敗、不穩定 +(flaky)、太久沒跑、或從未跑過——用 run-history 紀錄計算,然後先跑最 +高風險的(或只跑前 k 個):: + + from je_auto_control import select_flows, rank_flows + + ranked = rank_flows(["login", "checkout", "report"]) + risky = select_flows(["login", "checkout", "report"], k=2) + +分數為 ``0.5*失敗率 + 0.2*上次失敗 + 0.2*不穩定度 + 0.1*陳舊度``; +從未跑過的流程得 ``0.8``(未測試即高風險)。提供 ``AC_rank_tests`` / +``AC_select_tests`` 以及 ``ac_rank_tests`` / ``ac_select_tests`` MCP +工具。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index d54859dd..d4be94af 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -33,6 +33,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v8_features_doc doc/new_features/v9_features_doc doc/new_features/v10_features_doc + doc/new_features/v11_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index df9067c3..76fe4f00 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -113,6 +113,14 @@ from je_auto_control.utils.work_queue import ( BusinessError, WorkItem, WorkQueue, ) +# Seeded synthetic test-data generation +from je_auto_control.utils.test_data import generate_rows, write_dataset +# Risk-based test selection from run history +from je_auto_control.utils.test_select import rank_flows, select_flows +# MCP registry server.json manifest +from je_auto_control.utils.mcp_registry import ( + build_server_manifest, write_server_manifest, +) # Background popup/interrupt watchdog (unattended automation) from je_auto_control.utils.watchdog import ( PopupWatchdog, WatchdogRule, default_popup_watchdog, @@ -514,6 +522,9 @@ def start_autocontrol_gui(*args, **kwargs): "handle_file_dialog", "FileDialogDriver", "ensure_interactive_session", "is_session_locked", "WorkQueue", "WorkItem", "BusinessError", + "generate_rows", "write_dataset", + "rank_flows", "select_flows", + "build_server_manifest", "write_server_manifest", # MCP server "AuditLogger", "HttpMCPServer", "MCPContent", "MCPPrompt", "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 82cc9ac6..514215f9 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -653,6 +653,50 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: description="Fail if the session is locked / non-interactive.", )) _add_work_queue_specs(specs) + _add_tooling_specs(specs) + + +def _add_tooling_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_generate_data", "Data", "Generate Synthetic Data", + fields=( + FieldSpec("count", FieldType.INT, optional=True, default=10), + FieldSpec("path", FieldType.FILE_PATH, optional=True), + FieldSpec("fmt", FieldType.ENUM, choices=("json", "csv"), + optional=True), + FieldSpec("seed", FieldType.INT, optional=True), + ), + description="Generate seeded fake rows from a 'schema' (JSON view); " + "writes a file when 'path' is set.", + )) + specs.append(CommandSpec( + "AC_mcp_manifest", "Tools", "MCP Registry Manifest", + fields=( + FieldSpec("path", FieldType.FILE_PATH, optional=True, + default="server.json"), + FieldSpec("include_tools", FieldType.BOOL, optional=True, + default=False), + ), + description="Write an MCP registry server.json for this server.", + )) + specs.append(CommandSpec( + "AC_rank_tests", "Testing", "Rank Tests by Risk", + fields=( + FieldSpec("history_path", FieldType.FILE_PATH, optional=True), + FieldSpec("window", FieldType.INT, optional=True, default=10), + ), + description="Score 'flows' (JSON view) by risk from run history.", + )) + specs.append(CommandSpec( + "AC_select_tests", "Testing", "Select Risky Tests", + fields=( + FieldSpec("k", FieldType.INT, optional=True), + FieldSpec("threshold", FieldType.FLOAT, optional=True), + FieldSpec("history_path", FieldType.FILE_PATH, optional=True), + FieldSpec("window", FieldType.INT, optional=True, default=10), + ), + description="Pick riskiest 'flows' (JSON view): top-k or threshold.", + )) def _add_work_queue_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 61e942e2..d9506479 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2346,6 +2346,48 @@ def _queue_stats(db: str, name: str = "default") -> Dict[str, int]: return _queue(db, name).stats() +def _generate_data(schema: Dict[str, Any], count: int = 10, + path: Optional[str] = None, fmt: Optional[str] = None, + seed: Optional[int] = None) -> Dict[str, Any]: + """Adapter: generate synthetic rows; write to ``path`` when given.""" + from je_auto_control.utils.test_data import generate_rows, write_dataset + rows = generate_rows(schema, int(count), seed=seed) + if path: + return {"path": write_dataset(rows, path, fmt), "count": len(rows)} + return {"rows": rows, "count": len(rows)} + + +def _mcp_manifest(path: Optional[str] = None, + include_tools: bool = False) -> Dict[str, Any]: + """Adapter: build (or write) the MCP registry server.json manifest.""" + from je_auto_control.utils.mcp_registry import ( + build_server_manifest, write_server_manifest) + if path: + return {"path": write_server_manifest( + path, include_tools=bool(include_tools))} + return {"manifest": build_server_manifest( + include_tools=bool(include_tools))} + + +def _rank_tests(flows: List[str], history_path: Optional[str] = None, + window: int = 10) -> Dict[str, Any]: + """Adapter: score flows by risk (riskiest first).""" + from je_auto_control.utils.test_select import rank_flows + return {"ranked": rank_flows(flows, history_path=history_path, + window=int(window))} + + +def _select_tests(flows: List[str], k: Optional[int] = None, + threshold: Optional[float] = None, + history_path: Optional[str] = None, + window: int = 10) -> Dict[str, Any]: + """Adapter: pick the riskiest flows to run (top-k / threshold).""" + from je_auto_control.utils.test_select import select_flows + return {"selected": select_flows( + flows, k=k, threshold=threshold, history_path=history_path, + window=int(window))} + + class Executor: """ Executor @@ -2509,6 +2551,10 @@ def __init__(self): "AC_queue_complete": _queue_complete, "AC_queue_fail": _queue_fail, "AC_queue_stats": _queue_stats, + "AC_generate_data": _generate_data, + "AC_mcp_manifest": _mcp_manifest, + "AC_rank_tests": _rank_tests, + "AC_select_tests": _select_tests, "AC_a11y_record_start": _a11y_record_start, "AC_a11y_record_stop": _a11y_record_stop, "AC_a11y_record_events": _a11y_record_events, diff --git a/je_auto_control/utils/mcp_registry/__init__.py b/je_auto_control/utils/mcp_registry/__init__.py new file mode 100644 index 00000000..639a516c --- /dev/null +++ b/je_auto_control/utils/mcp_registry/__init__.py @@ -0,0 +1,6 @@ +"""MCP registry server.json manifest generation (discoverability).""" +from je_auto_control.utils.mcp_registry.registry import ( + build_server_manifest, write_server_manifest, +) + +__all__ = ["build_server_manifest", "write_server_manifest"] diff --git a/je_auto_control/utils/mcp_registry/registry.py b/je_auto_control/utils/mcp_registry/registry.py new file mode 100644 index 00000000..d4bd935a --- /dev/null +++ b/je_auto_control/utils/mcp_registry/registry.py @@ -0,0 +1,86 @@ +"""Generate an MCP registry ``server.json`` manifest for AutoControl. + +The MCP registry (https://registry.modelcontextprotocol.io) lists servers +described by a ``server.json`` document. Publishing one makes the +AutoControl MCP server discoverable and installable by MCP-aware agents +and IDEs. This module builds that manifest from the live package metadata +and (optionally) the real tool registry, so the advertised capabilities +never drift from what the server actually exposes. + +Pure standard library; imports no ``PySide6``. +""" +import json +from importlib import metadata +from pathlib import Path +from typing import Any, Dict, List + +_SCHEMA_URL = ("https://static.modelcontextprotocol.io/schemas/" + "2025-09-29/server.schema.json") +_SERVER_NAME = "io.github.intergration-automation-testing/autocontrol" +_REPO_URL = "https://github.com/Intergration-Automation-Testing/AutoControl" +_PYPI_NAME = "je_auto_control" +_DEFAULT_VERSION = "0.0.189" +_DESCRIPTION = ( + "Cross-platform GUI automation: mouse/keyboard control, image and OCR " + "recognition, native-UI (accessibility) control, and action scripting — " + "exposed as MCP tools.") + + +def _package_version() -> str: + """Best-effort installed version, falling back to a pinned default.""" + try: + return metadata.version(_PYPI_NAME) + except metadata.PackageNotFoundError: + return _DEFAULT_VERSION + + +def _tool_names() -> List[str]: + """Sorted names of every tool the default MCP registry exposes.""" + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry) + return sorted(tool.name for tool in build_default_tool_registry()) + + +def build_server_manifest(*, name: str = _SERVER_NAME, + version: str = "", + description: str = _DESCRIPTION, + repository_url: str = _REPO_URL, + pypi_name: str = _PYPI_NAME, + include_tools: bool = False) -> Dict[str, Any]: + """Return an MCP registry ``server.json`` manifest as a dict. + + ``version`` defaults to the installed package version. With + ``include_tools`` the live tool list is embedded under ``_meta`` for + discovery without changing the registry-valid core fields. + """ + resolved = version or _package_version() + manifest: Dict[str, Any] = { + "$schema": _SCHEMA_URL, + "name": name, + "description": description, + "version": resolved, + "repository": {"url": repository_url, "source": "github"}, + "packages": [{ + "registryType": "pypi", + "identifier": pypi_name, + "version": resolved, + "transport": {"type": "stdio"}, + }], + } + if include_tools: + names = _tool_names() + manifest["_meta"] = {name: {"toolCount": len(names), "tools": names}} + return manifest + + +def write_server_manifest(path: str = "server.json", *, + include_tools: bool = False, + **kwargs: Any) -> str: + """Write a ``server.json`` manifest to ``path``; return the resolved path. + + Extra keyword arguments are forwarded to :func:`build_server_manifest`. + """ + manifest = build_server_manifest(include_tools=include_tools, **kwargs) + target = Path(path) + target.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return str(target.resolve()) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 447bf76d..3a0f0aa6 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1727,6 +1727,74 @@ def work_queue_tools() -> List[MCPTool]: ] +def synthetic_data_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_generate_data", + description=("Generate deterministic synthetic test rows from a " + "field schema (e.g. {name:'name', age:{type:'int', " + "min:18,max:65}}). Same 'seed' -> same rows. Writes " + "JSON/CSV when 'path' is given, else returns rows."), + input_schema=schema({ + "schema": {"type": "object"}, + "count": {"type": "integer"}, + "path": {"type": "string"}, + "fmt": {"type": "string", "enum": ["json", "csv"]}, + "seed": {"type": "integer"}, + }, required=["schema"]), + handler=h.generate_data, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def mcp_registry_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_mcp_manifest", + description=("Build an MCP registry server.json manifest for this " + "AutoControl server (discoverability). Writes to " + "'path' when given, else returns the manifest. " + "include_tools embeds the live tool list."), + input_schema=schema({ + "path": {"type": "string"}, + "include_tools": {"type": "boolean"}, + }), + handler=h.mcp_manifest, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def test_selection_tools() -> List[MCPTool]: + _flows = {"flows": {"type": "array", "items": {"type": "string"}}, + "history_path": {"type": "string"}, + "window": {"type": "integer"}} + return [ + MCPTool( + name="ac_rank_tests", + description=("Score flows by risk (recent failures, flakiness, " + "staleness, never-run) from run history; returns the " + "ranked list, riskiest first."), + input_schema=schema(dict(_flows), required=["flows"]), + handler=h.rank_tests, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_select_tests", + description=("Pick the riskiest flows to run: top-'k', or score " + ">= 'threshold', else all ordered by risk. Returns " + "the selected flow names."), + input_schema=schema({ + "k": {"type": "integer"}, + "threshold": {"type": "number"}, **_flows, + }, required=["flows"]), + handler=h.select_tests, + annotations=READ_ONLY, + ), + ] + + def unattended_tools() -> List[MCPTool]: return [ MCPTool( @@ -2757,6 +2825,7 @@ def media_assert_tools() -> List[MCPTool]: redaction_tools, android_widget_tools, ios_tools, webrunner_tools, scheduler_tools, trigger_tools, hotkey_tools, watchdog_tools, unattended_tools, work_queue_tools, + synthetic_data_tools, mcp_registry_tools, test_selection_tools, screen_record_tools, process_and_shell_tools, remote_desktop_tools, gamepad_tools, usb_passthrough_tools, assertion_tools, data_source_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 5d9ec22d..10f02af2 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -826,6 +826,37 @@ def queue_stats(db, name="default"): return _work_queue(db, name).stats() +def generate_data(schema, count=10, path=None, fmt=None, seed=None): + from je_auto_control.utils.test_data import generate_rows, write_dataset + rows = generate_rows(schema, int(count), seed=seed) + if path: + return {"path": write_dataset(rows, path, fmt), "count": len(rows)} + return {"rows": rows, "count": len(rows)} + + +def mcp_manifest(path=None, include_tools=False): + from je_auto_control.utils.mcp_registry import ( + build_server_manifest, write_server_manifest) + if path: + return {"path": write_server_manifest( + path, include_tools=bool(include_tools))} + return {"manifest": build_server_manifest( + include_tools=bool(include_tools))} + + +def rank_tests(flows, history_path=None, window=10): + from je_auto_control.utils.test_select import rank_flows + return {"ranked": rank_flows(flows, history_path=history_path, + window=int(window))} + + +def select_tests(flows, k=None, threshold=None, history_path=None, window=10): + from je_auto_control.utils.test_select import select_flows + return {"selected": select_flows( + flows, k=k, threshold=threshold, history_path=history_path, + window=int(window))} + + def vlm_locate(description: str, screen_region: Optional[List[int]] = None, model: Optional[str] = None) -> Optional[List[int]]: diff --git a/je_auto_control/utils/test_data/__init__.py b/je_auto_control/utils/test_data/__init__.py new file mode 100644 index 00000000..01e5fbe7 --- /dev/null +++ b/je_auto_control/utils/test_data/__init__.py @@ -0,0 +1,6 @@ +"""Seeded synthetic test-data generation (pure standard library).""" +from je_auto_control.utils.test_data.test_data import ( + generate_rows, write_dataset, +) + +__all__ = ["generate_rows", "write_dataset"] diff --git a/je_auto_control/utils/test_data/test_data.py b/je_auto_control/utils/test_data/test_data.py new file mode 100644 index 00000000..991bc6dd --- /dev/null +++ b/je_auto_control/utils/test_data/test_data.py @@ -0,0 +1,199 @@ +"""Seeded synthetic test-data generator (pure standard library). + +Generate deterministic fake rows from a tiny field schema, to drive +data-driven runs (``AC_load_data`` / :mod:`data_source`) without shipping +real PII. No third-party dependency (no Faker): a curated word/name pool +plus the stdlib :class:`random.Random` keep output reproducible for a +given ``seed`` so test runs stay stable. + +A schema maps each field name to a *spec* — either a bare type name or a +``{"type": ..., **params}`` mapping:: + + from je_auto_control import generate_rows + + rows = generate_rows({ + "name": "name", + "email": {"type": "email", "domain": "acme.test"}, + "age": {"type": "int", "min": 18, "max": 65}, + "status": {"type": "choice", "choices": ["new", "vip"]}, + }, count=100, seed=7) + +The same ``seed`` always yields the same rows. +""" +import csv +import json +import random +import uuid +from datetime import date, timedelta +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +_FIRST = ("Alex", "Sam", "Jordan", "Taylor", "Morgan", "Casey", "Riley", + "Jamie", "Avery", "Quinn", "Drew", "Cameron", "Skyler", "Reese", + "Rowan", "Sage") +_LAST = ("Chen", "Smith", "Lee", "Garcia", "Patel", "Kim", "Nguyen", "Brown", + "Muller", "Rossi", "Tanaka", "Silva", "Khan", "Novak", "Haddad", + "Owusu") +_CITY = ("Taipei", "Berlin", "Lisbon", "Osaka", "Nairobi", "Lima", "Oslo", + "Cairo", "Toronto", "Pune", "Bogota", "Hanoi") +_WORDS = ("alpha", "beta", "gamma", "delta", "orbit", "river", "stone", + "ember", "quartz", "willow", "harbor", "meadow", "cobalt", "ivory", + "cedar", "onyx") +_SUFFIX = ("Labs", "Group", "Systems", "Works", "Co") +_DOMAINS = ("example.com", "test.org", "sample.net") + + +def _g_first(rng: random.Random, _spec: Dict[str, Any]) -> str: + return rng.choice(_FIRST) + + +def _g_last(rng: random.Random, _spec: Dict[str, Any]) -> str: + return rng.choice(_LAST) + + +def _g_name(rng: random.Random, _spec: Dict[str, Any]) -> str: + return f"{rng.choice(_FIRST)} {rng.choice(_LAST)}" + + +def _g_word(rng: random.Random, _spec: Dict[str, Any]) -> str: + return rng.choice(_WORDS) + + +def _g_city(rng: random.Random, _spec: Dict[str, Any]) -> str: + return rng.choice(_CITY) + + +def _g_company(rng: random.Random, _spec: Dict[str, Any]) -> str: + return f"{rng.choice(_WORDS).capitalize()} {rng.choice(_SUFFIX)}" + + +def _g_username(rng: random.Random, _spec: Dict[str, Any]) -> str: + return f"{rng.choice(_FIRST).lower()}{rng.randint(1, 9999)}" + + +def _g_email(rng: random.Random, spec: Dict[str, Any]) -> str: + domain = spec.get("domain") or rng.choice(_DOMAINS) + return (f"{rng.choice(_FIRST).lower()}.{rng.choice(_LAST).lower()}" + f"{rng.randint(1, 999)}@{domain}") + + +def _g_phone(rng: random.Random, _spec: Dict[str, Any]) -> str: + return (f"+1-{rng.randint(200, 999)}-{rng.randint(200, 999)}-" + f"{rng.randint(1000, 9999)}") + + +def _g_uuid(rng: random.Random, _spec: Dict[str, Any]) -> str: + return str(uuid.UUID(int=rng.getrandbits(128), version=4)) + + +def _g_bool(rng: random.Random, spec: Dict[str, Any]) -> bool: + return rng.random() < float(spec.get("p", 0.5)) + + +def _g_int(rng: random.Random, spec: Dict[str, Any]) -> int: + return rng.randint(int(spec.get("min", 0)), int(spec.get("max", 100))) + + +def _g_float(rng: random.Random, spec: Dict[str, Any]) -> float: + value = rng.uniform(float(spec.get("min", 0.0)), + float(spec.get("max", 1.0))) + return round(value, int(spec.get("ndigits", 2))) + + +def _g_choice(rng: random.Random, spec: Dict[str, Any]) -> Any: + choices = list(spec.get("choices") or [""]) + return rng.choice(choices) + + +def _g_sentence(rng: random.Random, spec: Dict[str, Any]) -> str: + count = max(1, int(spec.get("words", 6))) + body = " ".join(rng.choice(_WORDS) for _ in range(count)) + return f"{body.capitalize()}." + + +def _parse_date(raw: Optional[str], fallback: date) -> date: + if not raw: + return fallback + return date.fromisoformat(str(raw)) + + +def _g_date(rng: random.Random, spec: Dict[str, Any]) -> str: + start = _parse_date(spec.get("start"), date(2000, 1, 1)) + end = _parse_date(spec.get("end"), date.today()) + span = max(0, (end - start).days) + return (start + timedelta(days=rng.randint(0, span))).isoformat() + + +_GENERATORS: Dict[str, Callable[[random.Random, Dict[str, Any]], Any]] = { + "first_name": _g_first, "last_name": _g_last, "name": _g_name, + "word": _g_word, "city": _g_city, "company": _g_company, + "username": _g_username, "email": _g_email, "phone": _g_phone, + "uuid": _g_uuid, "bool": _g_bool, "int": _g_int, "float": _g_float, + "choice": _g_choice, "sentence": _g_sentence, "date": _g_date, +} + + +def field_types() -> List[str]: + """Return the sorted list of supported field-type names.""" + return sorted(_GENERATORS) + + +def _normalize(field: str, spec: Any) -> Dict[str, Any]: + if isinstance(spec, str): + spec = {"type": spec} + if not isinstance(spec, dict) or "type" not in spec: + raise ValueError( + f"field {field!r}: spec must be a type name or {{'type': ...}}") + kind = spec["type"] + if kind not in _GENERATORS: + raise ValueError( + f"field {field!r}: unknown type {kind!r}; " + f"known: {sorted(_GENERATORS)}") + return spec + + +def _make_row(rng: random.Random, + specs: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + return {field: _GENERATORS[spec["type"]](rng, spec) + for field, spec in specs.items()} + + +def generate_rows(schema: Dict[str, Any], count: int, *, + seed: Optional[int] = None) -> List[Dict[str, Any]]: + """Return ``count`` deterministic rows matching ``schema``. + + ``schema`` maps each field name to a type name (e.g. ``"email"``) or a + ``{"type": ..., **params}`` mapping. The same ``seed`` always yields + identical rows. + """ + if not isinstance(schema, dict) or not schema: + raise ValueError("schema must be a non-empty {field: spec} mapping") + rng = random.Random(seed) # nosec B311 # reason: non-crypto test data + specs = {field: _normalize(field, spec) for field, spec in schema.items()} + return [_make_row(rng, specs) for _ in range(max(0, int(count)))] + + +def write_dataset(rows: List[Dict[str, Any]], path: str, + fmt: Optional[str] = None) -> str: + """Write ``rows`` to ``path`` as JSON or CSV; return the resolved path. + + ``fmt`` defaults to the file extension (``.json`` or ``.csv``). + """ + target = Path(path) + chosen = (fmt or target.suffix.lstrip(".") or "json").lower() + if chosen == "json": + target.write_text(json.dumps(rows, indent=2, ensure_ascii=False), + encoding="utf-8") + elif chosen == "csv": + _write_csv(rows, target) + else: + raise ValueError(f"unsupported format {chosen!r}; use 'json' or 'csv'") + return str(target.resolve()) + + +def _write_csv(rows: List[Dict[str, Any]], target: Path) -> None: + fields = list(rows[0].keys()) if rows else [] + with target.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) diff --git a/je_auto_control/utils/test_select/__init__.py b/je_auto_control/utils/test_select/__init__.py new file mode 100644 index 00000000..dd39fc2c --- /dev/null +++ b/je_auto_control/utils/test_select/__init__.py @@ -0,0 +1,6 @@ +"""Risk-based test selection from run history.""" +from je_auto_control.utils.test_select.test_select import ( + rank_flows, select_flows, +) + +__all__ = ["rank_flows", "select_flows"] diff --git a/je_auto_control/utils/test_select/test_select.py b/je_auto_control/utils/test_select/test_select.py new file mode 100644 index 00000000..f396d0cc --- /dev/null +++ b/je_auto_control/utils/test_select/test_select.py @@ -0,0 +1,115 @@ +"""Risk-based test selection from run history (pure standard library). + +Instead of always running the whole suite, rank flows by how *risky* they +are — recently failing, flaky, stale, or never-run — and run the riskiest +first (or only the top-k). Risk is derived entirely from the run-history +store (:mod:`je_auto_control.utils.run_history`); no third-party +dependency. + +Each flow is identified by its ``script_path`` in the history. The score +is in ``[0, 1]``:: + + score = 0.5 * failure_rate + + 0.2 * (last run failed) + + 0.2 * flakiness + + 0.1 * staleness + +A flow that has never run scores ``0.8`` — untested means risky. +""" +import time +from typing import Any, Dict, List, Optional + +_WEIGHT_FAILURE = 0.5 +_WEIGHT_LAST_FAILED = 0.2 +_WEIGHT_FLAKY = 0.2 +_WEIGHT_STALE = 0.1 +_UNTESTED_SCORE = 0.8 +_STALE_DAYS = 30.0 +_SECONDS_PER_DAY = 86400.0 +_FINISHED = ("ok", "error") + + +def _open_store(history_path: Optional[str]): + """Return ``(store, owned)``; ``owned`` stores must be closed by caller.""" + from je_auto_control.utils.run_history import ( + HistoryStore, default_history_store) + if history_path: + return HistoryStore(history_path), True + return default_history_store(), False + + +def _runs_by_flow(store: Any, limit: int) -> Dict[str, List[Any]]: + grouped: Dict[str, List[Any]] = {} + for rec in store.list_runs(limit=limit): + grouped.setdefault(rec.script_path, []).append(rec) + return grouped + + +def _flaky(chrono: List[str]) -> tuple: + """Return ``(flaky_rate, transition_count)`` over a status sequence.""" + if len(chrono) < 2: + return 0.0, 0 + transitions = sum(1 for a, b in zip(chrono, chrono[1:]) if a != b) + return transitions / (len(chrono) - 1), transitions + + +def _score_flow(records: List[Any], window: int, + now: float) -> Dict[str, Any]: + finished = [r for r in records if r.status in _FINISHED][:window] + if not finished: + return {"runs": 0, "failures": 0, "last_status": None, + "flaky": False, "score": _UNTESTED_SCORE} + total = len(finished) + failures = sum(1 for r in finished if r.status == "error") + last_failed = finished[0].status == "error" + flaky_rate, transitions = _flaky([r.status for r in reversed(finished)]) + age_days = max(0.0, (now - finished[0].started_at) / _SECONDS_PER_DAY) + score = (_WEIGHT_FAILURE * failures / total + + _WEIGHT_LAST_FAILED * (1.0 if last_failed else 0.0) + + _WEIGHT_FLAKY * flaky_rate + + _WEIGHT_STALE * min(1.0, age_days / _STALE_DAYS)) + return {"runs": total, "failures": failures, + "last_status": finished[0].status, "flaky": transitions >= 2, + "score": round(min(1.0, score), 4)} + + +def rank_flows(flows: List[str], *, history_path: Optional[str] = None, + window: int = 10) -> List[Dict[str, Any]]: + """Return ``flows`` scored and sorted by risk (riskiest first). + + Each entry is ``{flow, score, runs, failures, last_status, flaky}``. + A flow absent from history scores ``0.8`` (untested is treated as + risky). ``window`` caps how many recent runs per flow are considered. + """ + flows = list(flows) + store, owned = _open_store(history_path) + now = time.time() + try: + grouped = _runs_by_flow(store, max(100, window * max(1, len(flows)))) + finally: + if owned: + store.close() + ranked: List[Dict[str, Any]] = [] + for flow in flows: + info = _score_flow(grouped.get(flow, []), window, now) + info["flow"] = flow + ranked.append(info) + ranked.sort(key=lambda d: (-d["score"], d["flow"])) + return ranked + + +def select_flows(flows: List[str], *, k: Optional[int] = None, + threshold: Optional[float] = None, + history_path: Optional[str] = None, + window: int = 10) -> List[str]: + """Return flow names to run, riskiest first. + + With ``k`` keep the top-k; with ``threshold`` keep score >= threshold; + with neither, return every flow ordered by risk. + """ + ranked = rank_flows(flows, history_path=history_path, window=window) + if threshold is not None: + ranked = [r for r in ranked if r["score"] >= float(threshold)] + if k is not None: + ranked = ranked[: max(0, int(k))] + return [r["flow"] for r in ranked] diff --git a/test/unit_test/headless/test_tooling_batch.py b/test/unit_test/headless/test_tooling_batch.py new file mode 100644 index 00000000..84dd6137 --- /dev/null +++ b/test/unit_test/headless/test_tooling_batch.py @@ -0,0 +1,125 @@ +"""Headless tests for the tooling batch: synthetic data, MCP registry +manifest, and risk-based test selection. Pure stdlib; no Qt imports.""" +import json + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.test_data import generate_rows, write_dataset +from je_auto_control.utils.test_select import rank_flows, select_flows +from je_auto_control.utils.mcp_registry import ( + build_server_manifest, write_server_manifest) + + +# --- synthetic data ------------------------------------------------------- + +def test_generate_rows_is_deterministic_and_typed(): + schema = {"name": "name", "age": {"type": "int", "min": 18, "max": 65}, + "email": {"type": "email", "domain": "acme.test"}} + a = generate_rows(schema, 5, seed=7) + b = generate_rows(schema, 5, seed=7) + assert a == b # same seed -> same rows + assert len(a) == 5 + assert generate_rows(schema, 5, seed=8) != a + row = a[0] + assert 18 <= row["age"] <= 65 + assert row["email"].endswith("@acme.test") + assert " " in row["name"] + + +def test_generate_rows_rejects_unknown_type(): + with pytest.raises(ValueError): + generate_rows({"x": "not_a_type"}, 1) + + +def test_write_dataset_json_and_csv(tmp_path): + rows = generate_rows({"id": "uuid", "n": "int"}, 3, seed=1) + jpath = write_dataset(rows, str(tmp_path / "d.json")) + assert json.loads(open(jpath, encoding="utf-8").read()) == rows + cpath = write_dataset(rows, str(tmp_path / "d.csv")) + text = open(cpath, encoding="utf-8").read() + assert text.splitlines()[0] == "id,n" + + +# --- MCP registry manifest ------------------------------------------------ + +def test_build_server_manifest_core_fields(): + m = build_server_manifest() + assert m["name"] and m["version"] + assert m["packages"][0]["registryType"] == "pypi" + assert m["repository"]["source"] == "github" + assert "_meta" not in m + + +def test_manifest_include_tools_embeds_live_list(tmp_path): + m = build_server_manifest(include_tools=True) + meta = m["_meta"][m["name"]] + assert meta["toolCount"] == len(meta["tools"]) > 0 + assert any(t.startswith("ac_") for t in meta["tools"]) + path = write_server_manifest(str(tmp_path / "server.json")) + assert json.loads(open(path, encoding="utf-8").read())["name"] == m["name"] + + +# --- risk-based test selection ------------------------------------------- + +@pytest.fixture() +def history(tmp_path): + from je_auto_control.utils.run_history import HistoryStore + store = HistoryStore(str(tmp_path / "h.sqlite")) + for status in ("ok", "ok", "ok"): + rid = store.start_run("manual", "x", "flow_pass") + store.finish_run(rid, status) + rid = store.start_run("manual", "x", "flow_fail") + store.finish_run(rid, "error") + store.close() + return str(tmp_path / "h.sqlite") + + +def test_rank_orders_by_risk(history): + ranked = rank_flows(["flow_pass", "flow_fail", "flow_new"], + history_path=history) + order = [r["flow"] for r in ranked] + assert order[0] == "flow_new" # untested == riskiest (0.8) + assert order.index("flow_fail") < order.index("flow_pass") + by = {r["flow"]: r for r in ranked} + assert by["flow_new"]["runs"] == 0 + assert by["flow_fail"]["last_status"] == "error" + + +def test_select_top_k_and_threshold(history): + flows = ["flow_pass", "flow_fail", "flow_new"] + assert select_flows(flows, k=2, history_path=history) == \ + ["flow_new", "flow_fail"] + assert select_flows(flows, threshold=0.75, history_path=history) == \ + ["flow_new"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_wiring(tmp_path): + rec = ac.execute_action( + [["AC_generate_data", {"schema": {"n": "int"}, "count": 3, "seed": 1}]]) + assert any("'count': 3" in str(v) for v in rec.values()) + known = ac.executor.known_commands() + assert {"AC_generate_data", "AC_mcp_manifest", "AC_rank_tests", + "AC_select_tests"} <= known + + +def test_mcp_and_builder_wiring(): + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_generate_data", "ac_mcp_manifest", "ac_rank_tests", + "ac_select_tests"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + cmds = {s.command for s in _build_specs()} + assert {"AC_generate_data", "AC_mcp_manifest", "AC_rank_tests", + "AC_select_tests"} <= cmds + + +def test_facade_exports(): + for attr in ("generate_rows", "write_dataset", "rank_flows", + "select_flows", "build_server_manifest", + "write_server_manifest"): + assert hasattr(ac, attr) + assert attr in ac.__all__ From a7e6fe78a53c00325cb5b693738e32a13b117327 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 19 Jun 2026 07:45:04 +0800 Subject: [PATCH 2/2] Fix risk selection default store: use shared instance, not call it --- je_auto_control/utils/test_select/test_select.py | 4 +++- test/unit_test/headless/test_tooling_batch.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/je_auto_control/utils/test_select/test_select.py b/je_auto_control/utils/test_select/test_select.py index f396d0cc..a1eb5083 100644 --- a/je_auto_control/utils/test_select/test_select.py +++ b/je_auto_control/utils/test_select/test_select.py @@ -35,7 +35,9 @@ def _open_store(history_path: Optional[str]): HistoryStore, default_history_store) if history_path: return HistoryStore(history_path), True - return default_history_store(), False + # default_history_store is a shared instance, not a factory — use it + # directly and leave it open (the caller must not close it). + return default_history_store, False def _runs_by_flow(store: Any, limit: int) -> Dict[str, List[Any]]: diff --git a/test/unit_test/headless/test_tooling_batch.py b/test/unit_test/headless/test_tooling_batch.py index 84dd6137..18e8042b 100644 --- a/test/unit_test/headless/test_tooling_batch.py +++ b/test/unit_test/headless/test_tooling_batch.py @@ -86,6 +86,17 @@ def test_rank_orders_by_risk(history): assert by["flow_fail"]["last_status"] == "error" +def test_rank_uses_shared_default_store(monkeypatch): + # history_path=None must use the shared default_history_store *instance* + # (not call it). Guards against the "not callable" regression. + import je_auto_control.utils.run_history as rh + from je_auto_control.utils.run_history import HistoryStore + monkeypatch.setattr(rh, "default_history_store", HistoryStore(":memory:")) + ranked = rank_flows(["never_run"]) + assert ranked[0]["flow"] == "never_run" + assert ranked[0]["runs"] == 0 + + def test_select_top_k_and_threshold(history): flows = ["flow_pass", "flow_fail", "flow_new"] assert select_flows(flows, k=2, history_path=history) == \