From 6bd3ecf5dfc5d7016bc08aac61ac26e35a918d7c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:56:26 +0000 Subject: [PATCH 1/5] Add pinned per-site configurations, with Chromium launcher personas --- README.md | 1 + arms/codex-repl-pinned.toml | 16 +++++ benchmarks/retail-cart.toml | 4 +- configurators/pinned/configure | 50 ++++++++++++++++ configurators/pinned/sites.toml | 91 ++++++++++++++++++++++++++++ harness/environment.py | 103 ++++++++++++++++++++++++++++++++ tests/test_environment.py | 57 ++++++++++++++++++ tests/test_pinned.py | 30 ++++++++++ 8 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 arms/codex-repl-pinned.toml create mode 100755 configurators/pinned/configure create mode 100644 configurators/pinned/sites.toml create mode 100644 tests/test_pinned.py diff --git a/README.md b/README.md index 8f19fc2..3cb40e9 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot |---|---|---| | `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | `stealth: true` and no proxy, so each browser gets its own egress IP from the stealth proxy pool | | `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. | +| `codex-repl-pinned` | Same | A fixed per-site table (`configurators/pinned/sites.toml`): stealth, GPU, a new US ISP or residential proxy per trial, and on some sites a macOS or WebGL persona applied by restarting Chromium with `KERNEL_*` launcher variables | The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: diff --git a/arms/codex-repl-pinned.toml b/arms/codex-repl-pinned.toml new file mode 100644 index 0000000..ae46fea --- /dev/null +++ b/arms/codex-repl-pinned.toml @@ -0,0 +1,16 @@ +# Same as codex-repl, but each site's browser, proxy and persona come from the +# fixed table in configurators/pinned/sites.toml, with a new proxy per trial. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } +configurator = "pinned" diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml index 4d486a6..c980f3d 100644 --- a/benchmarks/retail-cart.toml +++ b/benchmarks/retail-cart.toml @@ -1,5 +1,5 @@ # Guest add-to-cart flows on bot-protected retail and industrial-supply sites, # driven through the browser REPL, with a stealth browser and with the config -# registry's per-site settings. +# registry's per-site settings, and with a fixed per-site table. suite = "retail-cart" -arms = ["codex-repl", "codex-repl-site-config"] +arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned"] diff --git a/configurators/pinned/configure b/configurators/pinned/configure new file mode 100755 index 0000000..e461df6 --- /dev/null +++ b/configurators/pinned/configure @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Choose browser settings from the fixed per-site table in sites.toml. + +The task's site comes from `metadata.site` in its task.toml. Proxies are US +ISP or US residential, created per trial. A `persona` of `macos` or `webgl` +restarts Chromium with the matching launcher variables; `webgl` rotates +through the listed WebGL personas per trial. + +stdin: {"task": {"config": str, "instruction": str}, "args": {}} +stdout: {"browser": {...}, "steering": str | null, "provenance": {...}} +""" + +from __future__ import annotations + +import json +import sys +import tomllib +from pathlib import Path + +TABLE = tomllib.loads((Path(__file__).parent / "sites.toml").read_text()) + + +def main() -> int: + task = json.load(sys.stdin)["task"] + site = tomllib.loads(task["config"]).get("metadata", {}).get("site") + entry = TABLE["sites"].get(site) + if entry is None: + raise SystemExit(f"No pinned configuration for site {site!r}") + browser = dict(entry["browser"]) + browser["proxy_create"] = {"type": entry["proxy"], "config": {"country": "US"}} + persona = entry.get("persona") + if persona == "macos": + browser["chromium_env"] = TABLE["macos_persona"] + elif persona == "webgl": + browser["chromium_env"] = TABLE["webgl_personas"] + elif persona is not None: + raise SystemExit(f"Unknown persona {persona!r} for {site}") + json.dump( + { + "browser": browser, + "steering": entry.get("steering"), + "provenance": {"site": site, "proxy": entry["proxy"], "persona": persona}, + }, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/configurators/pinned/sites.toml b/configurators/pinned/sites.toml new file mode 100644 index 0000000..eec8757 --- /dev/null +++ b/configurators/pinned/sites.toml @@ -0,0 +1,91 @@ +# Browser settings per site, measured by hand on 2026-10-29..31. Each entry is +# merged over the arm's config. `proxy_create` gives every trial its own proxy; +# `chromium_env` restarts Chromium with KERNEL_* launcher variables (a list is +# one set per trial, picked at random); `steering` is added to the instruction. + +[macos_persona] +KERNEL_MAC_PERSONA = "1" + +[[webgl_personas]] +KERNEL_WEBGL_VENDOR = "Google Inc. (NVIDIA Corporation)" +KERNEL_WEBGL_RENDERER = "ANGLE (NVIDIA Corporation, NVIDIA GeForce RTX 3060/PCIe/SSE2, OpenGL 4.5.0)" +KERNEL_CANVAS_AA_OFFSET = "-2" + +[[webgl_personas]] +KERNEL_WEBGL_VENDOR = "Google Inc. (AMD)" +KERNEL_WEBGL_RENDERER = "ANGLE (AMD, AMD Custom GPU 0405 (radeonsi vangogh LLVM 15.0.7), OpenGL 4.6 (Core Profile) Mesa 23.2.1)" +KERNEL_CANVAS_AA_OFFSET = "3" + +[sites."amazon.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."costco.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."dickssportinggoods.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."ebay.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" + +[sites."guitarcenter.com"] +# Measured passing only with Web Bot Auth signatures, which this project does not have. +browser = { stealth = true } +proxy = "isp" + +[sites."harborfreight.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."homedepot.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."lowes.com"] +browser = { stealth = false } +proxy = "isp" + +[sites."mouser.com"] +browser = { stealth = true } +proxy = "residential" +steering = "mouser.com may show a bot-check interstitial on a page load that clears by itself within about 12 seconds. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." + +[sites."newark.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."radwell.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."staples.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."target.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."walmart.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."wbmason.com"] +# No measured working configuration. The recommended interim is US residential, +# solving the challenge by hand, which a benchmark cannot do. +browser = { stealth = true } +proxy = "residential" +steering = "wbmason.com may show a bot-check interstitial that clears by itself. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." + +[sites."zoro.com"] +browser = { stealth = true, gpu = true } +proxy = "residential" +persona = "webgl" +steering = "zoro.com may show a bot-check interstitial that clears by itself. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." diff --git a/harness/environment.py b/harness/environment.py index 9a33116..937ee27 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -12,12 +12,21 @@ so `delete_run_proxies` can remove any a crashed trial left behind. A trial whose proxy cannot be created fails with `ProxyCreationError`, so reports can count those trials apart from other infrastructure failures. + +A `chromium_env` entry is a set of `KERNEL_*` launcher variables, or a list +of sets of which each trial picks one at random. Chromium is restarted with +them right after the browser is created, before the task container starts, +and the page is checked for the persona or WebGL renderer they ask for. A +trial where that fails fails with `ChromiumEnvError`. """ from __future__ import annotations +import asyncio import json import os +import random +import shlex import uuid from pathlib import Path from typing import Any @@ -27,6 +36,7 @@ CONFIG_FILE = "kernel-config.json" PROXY_CREATE = "proxy_create" +CHROMIUM_ENV = "chromium_env" RUN_ID_ENV = "BENCH_RUN_ID" @@ -34,6 +44,44 @@ class ProxyCreationError(RuntimeError): """The trial's proxy could not be created, so no browser was started.""" +class ChromiumEnvError(RuntimeError): + """Chromium could not be restarted with the trial's launcher variables.""" + + +SUPERVISOR_PROGRAM = """[program:chromium] +command=/usr/bin/env {env} /usr/local/bin/chromium-launcher +autostart=false +autorestart=true +startsecs=0 +stopsignal=KILL +stopasgroup=true +killasgroup=true +stopwaitsecs=1 +stdout_logfile=/var/log/supervisord/chromium +redirect_stderr=true +""" + +RESTART_CHROMIUM = """#!/bin/bash +set -euo pipefail +mv /tmp/bench-chromium.conf /etc/supervisor/conf.d/services/chromium.conf +supervisorctl reread +supervisorctl update +sleep 2 +supervisorctl start chromium +sleep 3 +supervisorctl status chromium +""" + +PAGE_IDENTITY = """return await page.evaluate(() => { + const gl = document.createElement("canvas").getContext("webgl"); + const info = gl && gl.getExtension("WEBGL_debug_renderer_info"); + return { + userAgent: navigator.userAgent, + renderer: info ? gl.getParameter(info.UNMASKED_RENDERER_WEBGL) : null, + }; +});""" + + def proxy_prefix(run_id: str) -> str: return f"bench-{run_id}-" @@ -54,10 +102,15 @@ class KernelTaskEnvironment(KernelEnvironment): def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: self._proxy_create: dict[str, Any] | None = None self._proxy_id: str | None = None + self._chromium_env: dict[str, str] | None = None config_path = Path(environment_dir) / CONFIG_FILE if config_path.is_file(): overrides = json.loads(config_path.read_text()) self._proxy_create = overrides.pop(PROXY_CREATE, None) + chromium_env = overrides.pop(CHROMIUM_ENV, None) + if isinstance(chromium_env, list): + chromium_env = random.choice(chromium_env) + self._chromium_env = chromium_env kwargs["browser_create_overrides"] = overrides super().__init__(environment_dir, *args, **kwargs) @@ -76,6 +129,56 @@ async def start(self, force_build: bool) -> None: self._browser_create_overrides["proxy_id"] = proxy.id await super().start(force_build) + async def _create_browser(self) -> Any: + response = await super()._create_browser() + if self._chromium_env: + try: + await self._restart_chromium(response.session_id) + except Exception as error: + raise ChromiumEnvError( + f"Could not restart Chromium with {sorted(self._chromium_env)}: {error}" + ) from error + return response + + async def _restart_chromium(self, session_id: str) -> None: + env = " ".join( + f"{key}={shlex.quote(str(value))}" for key, value in self._chromium_env.items() + ) + browsers = self._client.browsers + await browsers.fs.write_file( + session_id, SUPERVISOR_PROGRAM.format(env=env).encode(), path="/tmp/bench-chromium.conf" + ) + await browsers.fs.write_file( + session_id, RESTART_CHROMIUM.encode(), path="/tmp/bench-restart-chromium.sh" + ) + result = await browsers.process.exec( + session_id, + command="bash", + args=["/tmp/bench-restart-chromium.sh"], + as_root=True, + timeout_sec=60, + ) + if result.exit_code != 0: + raise RuntimeError(f"restart script exited {result.exit_code}") + identity = None + for _ in range(15): + check = await browsers.playwright.execute(session_id, code=PAGE_IDENTITY) + if check.success: + identity = check.result or {} + break + await asyncio.sleep(2) + if identity is None: + raise RuntimeError("Chromium did not come back after the restart") + if ( + self._chromium_env.get("KERNEL_MAC_PERSONA") + and "Macintosh" not in identity["userAgent"] + ): + raise RuntimeError(f"expected a macOS user agent, got {identity['userAgent']}") + renderer = self._chromium_env.get("KERNEL_WEBGL_RENDERER") + if renderer and identity["renderer"] != renderer: + raise RuntimeError(f"expected WebGL renderer {renderer}, got {identity['renderer']}") + self.logger.info(f"Chromium restarted with {sorted(self._chromium_env)}: {identity}") + async def stop(self, delete: bool) -> None: try: await super().stop(delete) diff --git a/tests/test_environment.py b/tests/test_environment.py index 92e564e..b7040f8 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -106,3 +106,60 @@ async def refuse(self, **body): with pytest.raises(environment.ProxyCreationError, match="isp proxy: 403"): asyncio.run(env.start(force_build=False)) assert calls == [] + + +class FakeBrowsers: + def __init__(self, calls, identity): + self.calls = calls + self.fs = type("Fs", (), {"write_file": self._write})() + self.process = type("Process", (), {"exec": self._exec})() + self.playwright = type("Playwright", (), {"execute": self._execute})() + self.identity = identity + + async def _write(self, session_id, contents, path): + self.calls.append(("write", path, contents.decode())) + + async def _exec(self, session_id, **kwargs): + self.calls.append(("exec", kwargs["args"], kwargs["as_root"])) + return type("Result", (), {"exit_code": 0})() + + async def _execute(self, session_id, code): + return type("Result", (), {"success": True, "result": self.identity})() + + +def _chromium_environment(tmp_path, monkeypatch, chromium_env, identity, calls): + async def create(self): + return type("Response", (), {"session_id": "s1"})() + + def init(self, environment_dir, **kwargs): + self._client = type("Client", (), {"browsers": FakeBrowsers(calls, identity)})() + self.logger = type("Logger", (), {"info": lambda self, message: None})() + + monkeypatch.setattr(KernelEnvironment, "__init__", init) + monkeypatch.setattr(KernelEnvironment, "_create_browser", create) + (tmp_path / "kernel-config.json").write_text(json.dumps({"chromium_env": chromium_env})) + return KernelTaskEnvironment(environment_dir=tmp_path) + + +def test_chromium_restarts_with_the_trials_launcher_variables(tmp_path, monkeypatch): + calls = [] + personas = [ + {"KERNEL_WEBGL_RENDERER": "ANGLE (NVIDIA)", "KERNEL_CANVAS_AA_OFFSET": "-2"}, + {"KERNEL_WEBGL_RENDERER": "ANGLE (NVIDIA)", "KERNEL_CANVAS_AA_OFFSET": "3"}, + ] + identity = {"userAgent": "X11; Linux", "renderer": "ANGLE (NVIDIA)"} + env = _chromium_environment(tmp_path, monkeypatch, personas, identity, calls) + + asyncio.run(env._create_browser()) + + program = next(contents for kind, path, *contents in calls if path.endswith(".conf")) + assert "KERNEL_WEBGL_RENDERER='ANGLE (NVIDIA)'" in program[0] + assert ("exec", ["/tmp/bench-restart-chromium.sh"], True) in calls + + +def test_a_persona_that_did_not_apply_fails_the_trial(tmp_path, monkeypatch): + identity = {"userAgent": "X11; Linux x86_64", "renderer": "ANGLE (Mesa)"} + env = _chromium_environment(tmp_path, monkeypatch, {"KERNEL_MAC_PERSONA": "1"}, identity, []) + + with pytest.raises(environment.ChromiumEnvError, match="macOS user agent"): + asyncio.run(env._create_browser()) diff --git a/tests/test_pinned.py b/tests/test_pinned.py new file mode 100644 index 0000000..3157f48 --- /dev/null +++ b/tests/test_pinned.py @@ -0,0 +1,30 @@ +import json +import subprocess + +from harness.definitions import ROOT + +CONFIGURE = ROOT / "configurators/pinned/configure" + + +def _configure(site): + task = {"task": {"config": f'[metadata]\nsite = "{site}"\n', "instruction": ""}, "args": {}} + result = subprocess.run( + [CONFIGURE], input=json.dumps(task), capture_output=True, text=True, check=True + ) + return json.loads(result.stdout) + + +def test_every_retail_cart_site_has_a_pinned_configuration(): + from harness.definitions import load_suite + + for task in load_suite("retail-cart").load_tasks(): + output = _configure(task.site) + assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + + +def test_personas_become_chromium_env(): + assert _configure("amazon.com")["browser"]["chromium_env"] == {"KERNEL_MAC_PERSONA": "1"} + zoro = _configure("zoro.com") + assert len(zoro["browser"]["chromium_env"]) == 2 + assert zoro["browser"]["proxy_create"]["type"] == "residential" + assert "30 seconds" in zoro["steering"] From 423c2a3ae4977cdb391c6d3d40248ab66f5a598a Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:10:36 +0000 Subject: [PATCH 2/5] Create GPU browsers without a memory request --- harness/environment.py | 7 +++++++ tests/test_environment.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/harness/environment.py b/harness/environment.py index 937ee27..cdf9ddd 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -129,6 +129,13 @@ async def start(self, force_build: bool) -> None: self._browser_create_overrides["proxy_id"] = proxy.id await super().start(force_build) + def _browser_create_kwargs(self) -> dict[str, Any]: + kwargs = super()._browser_create_kwargs() + # GPU browsers have a fixed memory size and reject a memory request. + if kwargs.get("gpu"): + kwargs.pop("memory", None) + return kwargs + async def _create_browser(self) -> Any: response = await super()._create_browser() if self._chromium_env: diff --git a/tests/test_environment.py b/tests/test_environment.py index b7040f8..a67145e 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -163,3 +163,24 @@ def test_a_persona_that_did_not_apply_fails_the_trial(tmp_path, monkeypatch): with pytest.raises(environment.ChromiumEnvError, match="macOS user agent"): asyncio.run(env._create_browser()) + + +def test_gpu_browsers_are_created_without_a_memory_request(tmp_path, monkeypatch): + monkeypatch.setattr( + KernelEnvironment, + "_browser_create_kwargs", + lambda self: {**self._browser_create_overrides, "memory": "8GiB"}, + ) + monkeypatch.setattr( + KernelEnvironment, + "__init__", + lambda self, environment_dir, **kwargs: setattr( + self, "_browser_create_overrides", kwargs["browser_create_overrides"] + ), + ) + (tmp_path / "kernel-config.json").write_text(json.dumps({"gpu": True})) + assert KernelTaskEnvironment(environment_dir=tmp_path)._browser_create_kwargs() == {"gpu": True} + (tmp_path / "kernel-config.json").write_text(json.dumps({"gpu": False})) + assert ( + KernelTaskEnvironment(environment_dir=tmp_path)._browser_create_kwargs()["memory"] == "8GiB" + ) From 7db0fbc5618c67b814b44b1527b39c8cce7462c5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:02:47 +0000 Subject: [PATCH 3/5] Ask for the outcome rather than the route, verify the cart, and add site guidance, VM setup and computer controls to the pinned arm --- arms/codex-repl-pinned.toml | 2 +- configurators/pinned/configure | 5 +- configurators/pinned/sites.toml | 42 ++++++++++--- harness/environment.py | 39 ++++++++---- suites/retail-cart/image/bench/score | 59 ++++++++++++++++--- suites/retail-cart/suite.py | 37 ++++++++---- suites/retail-cart/tasks.toml | 37 +++++++++++- tests/test_environment.py | 13 +++- tests/test_retail_cart.py | 6 ++ .../image/bin/kernel-mcp-verify | 8 ++- toolsets/kernel-mcp-local/toolset.py | 4 ++ 11 files changed, 205 insertions(+), 47 deletions(-) diff --git a/arms/codex-repl-pinned.toml b/arms/codex-repl-pinned.toml index ae46fea..b0fca7e 100644 --- a/arms/codex-repl-pinned.toml +++ b/arms/codex-repl-pinned.toml @@ -9,7 +9,7 @@ toolset = "kernel-mcp-local" disable_web_search = true [toolset_args] -toolsets = ["repl"] +toolsets = ["repl", "computer"] [browser] config = { stealth = true } diff --git a/configurators/pinned/configure b/configurators/pinned/configure index e461df6..16be541 100755 --- a/configurators/pinned/configure +++ b/configurators/pinned/configure @@ -4,7 +4,8 @@ The task's site comes from `metadata.site` in its task.toml. Proxies are US ISP or US residential, created per trial. A `persona` of `macos` or `webgl` restarts Chromium with the matching launcher variables; `webgl` rotates -through the listed WebGL personas per trial. +through the listed WebGL personas per trial. `vm_setup` commands run as root +in the browser's VM before the first page load. stdin: {"task": {"config": str, "instruction": str}, "args": {}} stdout: {"browser": {...}, "steering": str | null, "provenance": {...}} @@ -35,6 +36,8 @@ def main() -> int: browser["chromium_env"] = TABLE["webgl_personas"] elif persona is not None: raise SystemExit(f"Unknown persona {persona!r} for {site}") + if entry.get("vm_setup"): + browser["vm_setup"] = entry["vm_setup"] json.dump( { "browser": browser, diff --git a/configurators/pinned/sites.toml b/configurators/pinned/sites.toml index eec8757..1331411 100644 --- a/configurators/pinned/sites.toml +++ b/configurators/pinned/sites.toml @@ -1,7 +1,9 @@ # Browser settings per site, measured by hand on 2026-10-29..31. Each entry is -# merged over the arm's config. `proxy_create` gives every trial its own proxy; -# `chromium_env` restarts Chromium with KERNEL_* launcher variables (a list is -# one set per trial, picked at random); `steering` is added to the instruction. +# merged over the arm's config. `proxy` gives every trial its own proxy of that +# type; `persona` restarts Chromium with KERNEL_* launcher variables (`webgl` +# picks one set per trial at random); `vm_setup` commands run as root in the +# browser's VM before the first page load; `steering` is added to the +# instruction. [macos_persona] KERNEL_MAC_PERSONA = "1" @@ -38,6 +40,9 @@ proxy = "isp" # Measured passing only with Web Bot Auth signatures, which this project does not have. browser = { stealth = true } proxy = "isp" +steering = """ +guitarcenter.com denies its home page (/), /search and /cart to automated browsers with an "Access Denied" page, but search results at /search/ with a trailing slash, product pages and /cart/index load. Search by opening https://www.guitarcenter.com/search/?Ntt=, open the product, and click Add to Cart. Show the cart at https://www.guitarcenter.com/cart/index; if it does not list the item, report the add-to-cart confirmation the product page showed. +""" [sites."harborfreight.com"] browser = { stealth = true, gpu = true } @@ -55,11 +60,23 @@ proxy = "isp" [sites."mouser.com"] browser = { stealth = true } proxy = "residential" -steering = "mouser.com may show a bot-check interstitial on a page load that clears by itself within about 12 seconds. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." +steering = """ +mouser.com denies its search results, category and product pages (/c/..., /ProductDetail/...) to automated browsers, so do not press Enter in the search box. Find the part number instead: type the item into the search box and read the suggestions without submitting, or search the web (for example DuckDuckGo) for "mouser " and take the part number from the results. Then open https://www.mouser.com/en/Cart/, enter the manufacturer or Mouser part number and a quantity in its add-by-part-number row, and click Add. + +Pages may show a bot-check interstitial that clears by itself within about 12 seconds. Wait up to 30 seconds before treating a page as blocked. +""" [sites."newark.com"] browser = { stealth = true } proxy = "isp" +steering = """ +newark.com denies its search results, category and product pages (/search, /c/..., /dp/...) to automated browsers, so pressing Enter in the search box does nothing. The site's older store pages still work; use them: + +1. Type the item into the search box and read the suggestions without submitting, or open https://www.newark.com/webapp/wcs/stores/servlet/AjaxSearchLookAhead?searchTerm=&storeId=10194&catalogId=15003&langId=-1. Suggestions link to /dp/; note the order code of a matching product. +2. Open https://www.newark.com/webapp/wcs/stores/servlet/ProductDisplay?storeId=10194&catalogId=15003&langId=-1&partNumber=. +3. Clicking its Add to Cart link does nothing; navigate to the link's href instead. +4. Show the cart at https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1. The /cart page stays empty. +""" [sites."radwell.com"] browser = { stealth = true } @@ -78,14 +95,23 @@ browser = { stealth = true } proxy = "isp" [sites."wbmason.com"] -# No measured working configuration. The recommended interim is US residential, -# solving the challenge by hand, which a benchmark cannot do. +# The captcha-solver extension takes over the site's Cloudflare Turnstile check +# and never completes it, so its Turnstile interceptor is emptied before the +# first page load. browser = { stealth = true } proxy = "residential" -steering = "wbmason.com may show a bot-check interstitial that clears by itself. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." +vm_setup = ["printf '/* disabled */\\n' > /home/kernel/extensions/kernel/capmonster/interceptors/turnstileInterceptor.js"] +steering = """ +wbmason.com shows a "Verify Access to Continue" page with a Cloudflare Turnstile check before the site. Do not reload or navigate while it is showing: that turns it into a hard block. Wait up to 30 seconds for it to pass by itself; if a checkbox appears, click it once with computer_action. + +On this site use computer_action for clicks and typing: scripted clicks and typing in the search box do not take effect. +""" [sites."zoro.com"] browser = { stealth = true, gpu = true } proxy = "residential" persona = "webgl" -steering = "zoro.com may show a bot-check interstitial that clears by itself. Before treating a page as blocked, wait and recheck for real content for up to 30 seconds." +steering = """ +On zoro.com the first page load is often refused with a 403 or a bot-check page. That refusal sets a cookie, and loading the same URL once more usually shows the real site, so if the first load is blocked, load it again once. If the page says "Access is temporarily restricted", reloading will not help: report the task as blocked. +""" + diff --git a/harness/environment.py b/harness/environment.py index cdf9ddd..bf47afc 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -13,11 +13,13 @@ trial whose proxy cannot be created fails with `ProxyCreationError`, so reports can count those trials apart from other infrastructure failures. -A `chromium_env` entry is a set of `KERNEL_*` launcher variables, or a list -of sets of which each trial picks one at random. Chromium is restarted with -them right after the browser is created, before the task container starts, -and the page is checked for the persona or WebGL renderer they ask for. A -trial where that fails fails with `ChromiumEnvError`. +Two entries change the browser's VM right after it is created, before the +task container starts and before the first page load. `chromium_env` is a set +of `KERNEL_*` launcher variables, or a list of sets of which each trial picks +one at random: Chromium is restarted with them and the page is checked for +the persona or WebGL renderer they ask for. `vm_setup` is a list of shell +commands then run as root. A trial where either fails fails with +`VmSetupError`. """ from __future__ import annotations @@ -37,6 +39,7 @@ CONFIG_FILE = "kernel-config.json" PROXY_CREATE = "proxy_create" CHROMIUM_ENV = "chromium_env" +VM_SETUP = "vm_setup" RUN_ID_ENV = "BENCH_RUN_ID" @@ -44,8 +47,8 @@ class ProxyCreationError(RuntimeError): """The trial's proxy could not be created, so no browser was started.""" -class ChromiumEnvError(RuntimeError): - """Chromium could not be restarted with the trial's launcher variables.""" +class VmSetupError(RuntimeError): + """The browser's VM could not be set up for the trial before the first page load.""" SUPERVISOR_PROGRAM = """[program:chromium] @@ -103,6 +106,7 @@ def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: self._proxy_create: dict[str, Any] | None = None self._proxy_id: str | None = None self._chromium_env: dict[str, str] | None = None + self._vm_setup: list[str] = [] config_path = Path(environment_dir) / CONFIG_FILE if config_path.is_file(): overrides = json.loads(config_path.read_text()) @@ -111,6 +115,7 @@ def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: if isinstance(chromium_env, list): chromium_env = random.choice(chromium_env) self._chromium_env = chromium_env + self._vm_setup = overrides.pop(VM_SETUP, None) or [] kwargs["browser_create_overrides"] = overrides super().__init__(environment_dir, *args, **kwargs) @@ -138,15 +143,23 @@ def _browser_create_kwargs(self) -> dict[str, Any]: async def _create_browser(self) -> Any: response = await super()._create_browser() - if self._chromium_env: - try: + try: + if self._chromium_env: await self._restart_chromium(response.session_id) - except Exception as error: - raise ChromiumEnvError( - f"Could not restart Chromium with {sorted(self._chromium_env)}: {error}" - ) from error + for command in self._vm_setup: + await self._run_as_root(response.session_id, command) + except Exception as error: + raise VmSetupError(f"Could not set up the browser VM: {error}") from error return response + async def _run_as_root(self, session_id: str, command: str) -> None: + result = await self._client.browsers.process.exec( + session_id, command="bash", args=["-c", command], as_root=True, timeout_sec=60 + ) + if result.exit_code != 0: + raise RuntimeError(f"{command!r} exited {result.exit_code}") + self.logger.info(f"Ran in the browser VM: {command}") + async def _restart_chromium(self, session_id: str) -> None: env = " ".join( f"{key}={shlex.quote(str(value))}" for key, value in self._chromium_env.items() diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 679a7ce..185dc3f 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Turn the agent's finalize report into the task's rewards. +"""Turn the agent's finalize report into the task's rewards, and check the cart. `reward` is 1 when the agent reported success, and 0 for any other outcome or when it never called finalize. `finalized` records whether it called @@ -7,14 +7,17 @@ finalize, and `outcome_` which failure it reported, so bot blocks can be told apart from other website errors, Kernel errors and the agent's own failures. -The browser's final URL and title are saved next to the report, so a reported -success can be checked against the page the agent ended on. +The browser's final URL and title are saved, then the verifier checks the +cart itself: it opens the task's `cart_url`, or fetches its `cart_api` from +the page the agent ended on, and sets `cart_verified` when the text matches +`cart_match`. cart-check.json keeps what it saw. """ from __future__ import annotations import json import os +import re import shutil import urllib.request from pathlib import Path @@ -26,12 +29,31 @@ FAILURES = ("failed", "blocked", "website_error", "kernel_error") FINAL_PAGE = "return { url: page.url(), title: await page.title() };" +CART_PAGE = """ +const response = await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); +await page.waitForLoadState("networkidle", { timeout: 15000 }).catch(() => {}); +await page.waitForTimeout(3000); +return { + url: page.url(), + status: response ? response.status() : null, + title: await page.title(), + text: await page.evaluate(() => (document.body ? document.body.innerText : "")), +}; +""" + +CART_API = """ +return await page.evaluate(async (url) => { + const response = await fetch(url, { headers: { Accept: "application/json" }, credentials: "include" }); + return { url, status: response.status, text: await response.text() }; +}, %s); +""" -def final_page() -> dict: + +def execute(code: str, timeout_sec: int) -> dict: base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", - data=json.dumps({"code": FINAL_PAGE, "timeout_sec": 30}).encode(), + data=json.dumps({"code": code, "timeout_sec": timeout_sec}).encode(), headers={ "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", "Content-Type": "application/json", @@ -39,7 +61,7 @@ def final_page() -> dict: method="POST", ) try: - with urllib.request.urlopen(request, timeout=60) as response: + with urllib.request.urlopen(request, timeout=timeout_sec + 30) as response: result = json.load(response) except (OSError, ValueError, KeyError) as error: return {"error": str(error)} @@ -48,6 +70,23 @@ def final_page() -> dict: return result.get("result") or {} +def check_cart(task: dict) -> dict: + if task.get("cart_api"): + seen = execute(CART_API % json.dumps(task["cart_api"]), 60) + elif task.get("cart_url"): + seen = execute(CART_PAGE % json.dumps(task["cart_url"]), 90) + else: + return {"error": "task has no cart_url or cart_api"} + text = seen.pop("text", None) or "" + match = re.search(task["cart_match"], text, re.IGNORECASE) if task.get("cart_match") else None + seen["verified"] = match is not None + if match: + seen["excerpt"] = text[max(0, match.start() - 200) : match.end() + 200] + else: + seen["excerpt"] = text[:2000] + return seen + + def main() -> int: VERIFIER.mkdir(parents=True, exist_ok=True) try: @@ -56,13 +95,17 @@ def main() -> int: report = None if report is not None: shutil.copy2(REPORT, VERIFIER / "finalize.json") - if TASK.exists(): + task = json.loads(TASK.read_text()) if TASK.exists() else {} + if task: shutil.copy2(TASK, VERIFIER / "task.json") - (VERIFIER / "final-page.json").write_text(json.dumps(final_page(), indent=2)) + (VERIFIER / "final-page.json").write_text(json.dumps(execute(FINAL_PAGE, 30), indent=2)) + cart = check_cart(task) + (VERIFIER / "cart-check.json").write_text(json.dumps(cart, indent=2)) outcome = report.get("outcome") if report is not None else None rewards = { "reward": float(outcome == "success"), "finalized": float(report is not None), + "cart_verified": float(cart.get("verified") is True), **{f"outcome_{name}": float(outcome == name) for name in FAILURES}, } (VERIFIER / "reward.json").write_text(json.dumps(rewards, indent=2)) diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 7ddc17a..879c005 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -1,16 +1,18 @@ """Guest add-to-cart tasks on retail and industrial-supply sites with heavy bot protection. -Every task starts on a site's home page, opened before the agent starts, and -asks the agent to search for one item, open its product page, add it to the -cart without logging in, and report the cart. The flow crosses the pages -where bot protection usually acts: the first request, search, the product -page and the cart. +Every task starts on a site's home page, opened before the agent starts, +and asks the agent to put one item in the cart without logging in and report +the cart. The task fixes the outcome, not the route: on some sites the usual +search and product pages are denied to automated browsers, and per-site +guidance can send the agent another way. The agent ends each task by calling the `finalize` tool of the `bench` MCP server with its evidence, its reasoning and an outcome. The verifier turns that report into the reward and records the outcome, so a bot block is -counted apart from other failures. The browser's final URL and title are -saved for checking reported successes. +counted apart from other failures. The verifier then opens the site's cart +itself (`cart_url`, or `cart_api` fetched from the page) and records +`cart_verified` when it lists the item (`cart_match`, a case-insensitive +regular expression). """ from __future__ import annotations @@ -26,12 +28,12 @@ HERE = Path(__file__).parent IMAGE = HERE / "image" -INSTRUCTION = """Add {item} to the {site} cart, then open the cart. Search for it from the site's search box, as a shopper would, open its product page, and add one to the cart. Report the cart line for it with its quantity and price, and the cart subtotal. +INSTRUCTION = """Put one {item} in the {site} shopping cart as a guest, then show the cart. Report the cart line for it with its quantity and price, and the cart subtotal. ## Rules -- The browser is already open on {start_url}. Work on that site only. -- Do not log in, sign up, or enter personal or payment details. Continue as a guest. +- The browser is already open on {start_url}. Reach the item however works: the site's search, its menus and links, or a web search to identify the exact product or part number. The cart must be on {site}. +- Do not log in, sign up, or enter personal or payment details. - Never check out, pay, or place an order. Stop at the cart. - If the site shows a bot check, CAPTCHA or press-and-hold challenge, try to get past it as a person would. If you cannot, or the site denies access, report the task as blocked. @@ -47,6 +49,9 @@ class Task: site: str start_url: str item: str + cart_match: str + cart_url: str | None = None + cart_api: str | None = None timeout_sec: int = 900 @@ -75,7 +80,17 @@ def _write(task: Task, task_dir: Path) -> Path: shutil.rmtree(task_dir) environment = task_dir / "environment" shutil.copytree(IMAGE, environment) - (environment / "bench" / "task.json").write_text(json.dumps({"id": task.id, "site": task.site})) + (environment / "bench" / "task.json").write_text( + json.dumps( + { + "id": task.id, + "site": task.site, + "cart_url": task.cart_url, + "cart_api": task.cart_api, + "cart_match": task.cart_match, + } + ) + ) step = task_dir / "steps" / "run" (step / "tests").mkdir(parents=True) (step / "workdir").mkdir() diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml index a6e56c7..e9a98a5 100644 --- a/suites/retail-cart/tasks.toml +++ b/suites/retail-cart/tasks.toml @@ -1,99 +1,132 @@ # One guest add-to-cart task per site. Each starts on the site's home page, -# where most bot protection first sees the browser, and walks home page -> -# search -> product page -> cart, the path a shopping agent takes. +# where most bot protection first sees the browser. The verifier opens +# `cart_url` (or fetches `cart_api` from the page) and checks it lists the +# item with `cart_match`, a case-insensitive regular expression. [[tasks]] id = "amazon-aa-batteries" site = "amazon.com" start_url = "https://www.amazon.com/" item = "the Amazon Basics 48-pack of AA alkaline batteries" +cart_url = "https://www.amazon.com/gp/cart/view.html" +cart_match = 'Amazon Basics.{0,100}\bAA\b' [[tasks]] id = "walmart-paper-towels" site = "walmart.com" start_url = "https://www.walmart.com/" item = "a pack of Bounty Select-a-Size paper towels" +cart_url = "https://www.walmart.com/cart" +cart_match = 'Bounty' [[tasks]] id = "costco-microfiber-towels" site = "costco.com" start_url = "https://www.costco.com/" item = "the Kirkland Signature Ultra Plush microfiber towels, 36-count" +cart_url = "https://www.costco.com/CheckoutCartDisplayView" +cart_match = 'Ultra Plush' [[tasks]] id = "lowes-stanley-kit" site = "lowes.com" start_url = "https://www.lowes.com/" item = "the Stanley 27-pack 0.375-in metal kit" +cart_url = "https://www.lowes.com/cart" +cart_match = 'Stanley' [[tasks]] id = "staples-copy-paper" site = "staples.com" start_url = "https://www.staples.com/" item = "one 500-sheet ream of 8.5 x 11 in multipurpose copy paper" +cart_url = "https://www.staples.com/cc/sparq/cart" +cart_match = '(copy|multipurpose).{0,60}paper' [[tasks]] id = "target-paper-towels" site = "target.com" start_url = "https://www.target.com/" item = "a pack of Make-A-Size paper towels that can be shipped" +cart_url = "https://www.target.com/cart" +cart_match = 'paper towel' [[tasks]] id = "homedepot-desk" site = "homedepot.com" start_url = "https://www.homedepot.com/" item = "a computer desk under $150 that can be shipped" +cart_url = "https://www.homedepot.com/cart" +cart_match = '\bdesk' [[tasks]] id = "radwell-fuse" site = "radwell.com" start_url = "https://www.radwell.com/" item = "a 20 A, 32 V 3AG fuse" +cart_url = "https://www.radwell.com/Cart" +cart_match = 'fuse' [[tasks]] id = "ebay-usb-c-cable" site = "ebay.com" start_url = "https://www.ebay.com/" item = "a USB-C charging cable from a Buy It Now listing under $15" +cart_url = "https://cart.ebay.com/" +cart_match = 'usb.?c' [[tasks]] id = "harborfreight-utility-knife" site = "harborfreight.com" start_url = "https://www.harborfreight.com/" item = "a folding lock-back utility knife" +cart_url = "https://www.harborfreight.com/checkout/cart" +cart_match = 'utility knife|lock.?back' [[tasks]] id = "dickssportinggoods-water-bottle" site = "dickssportinggoods.com" start_url = "https://www.dickssportinggoods.com/" item = "a HydroJug 32 oz sport bottle that can be shipped" +cart_url = "https://www.dickssportinggoods.com/OrderItemDisplay" +cart_match = 'HydroJug' [[tasks]] id = "mouser-usb-c-receptacle" site = "mouser.com" start_url = "https://www.mouser.com/" item = "one GCT USB4105-GF-A USB-C receptacle" +cart_url = "https://www.mouser.com/en/Cart/" +cart_match = 'USB4105' [[tasks]] id = "newark-aa-batteries" site = "newark.com" start_url = "https://www.newark.com/" item = "a pack of AA alkaline batteries" +cart_url = "https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1" +cart_match = 'alkaline' [[tasks]] id = "zoro-nitrile-gloves" site = "zoro.com" start_url = "https://www.zoro.com/" item = "a box of disposable nitrile gloves, size large" +cart_url = "https://www.zoro.com/cart" +cart_match = 'nitrile' [[tasks]] id = "guitarcenter-picks" site = "guitarcenter.com" start_url = "https://www.guitarcenter.com/" item = "a 12-pack of Fender 351 celluloid medium guitar picks" +cart_api = "https://www.guitarcenter.com/rest/model/atg/commerce/ShoppingCartActor/cartRedesignNew?saveForLaterPage=1&SPA=true" +cart_match = '\b351\b' [[tasks]] id = "wbmason-sticky-notes" site = "wbmason.com" start_url = "https://www.wbmason.com/" item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" +cart_url = "https://www.wbmason.com/shopping-cart" +cart_match = 'Post-it|sticky note' diff --git a/tests/test_environment.py b/tests/test_environment.py index a67145e..2f282eb 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -161,7 +161,7 @@ def test_a_persona_that_did_not_apply_fails_the_trial(tmp_path, monkeypatch): identity = {"userAgent": "X11; Linux x86_64", "renderer": "ANGLE (Mesa)"} env = _chromium_environment(tmp_path, monkeypatch, {"KERNEL_MAC_PERSONA": "1"}, identity, []) - with pytest.raises(environment.ChromiumEnvError, match="macOS user agent"): + with pytest.raises(environment.VmSetupError, match="macOS user agent"): asyncio.run(env._create_browser()) @@ -184,3 +184,14 @@ def test_gpu_browsers_are_created_without_a_memory_request(tmp_path, monkeypatch assert ( KernelTaskEnvironment(environment_dir=tmp_path)._browser_create_kwargs()["memory"] == "8GiB" ) + + +def test_vm_setup_commands_run_as_root_after_the_browser_is_created(tmp_path, monkeypatch): + calls = [] + env = _chromium_environment(tmp_path, monkeypatch, None, {}, calls) + (tmp_path / "kernel-config.json").write_text(json.dumps({"vm_setup": ["echo hi", "true"]})) + env = KernelTaskEnvironment(environment_dir=tmp_path) + + asyncio.run(env._create_browser()) + + assert calls == [("exec", ["-c", "echo hi"], True), ("exec", ["-c", "true"], True)] diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index ffdd61c..c80885b 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -25,6 +25,7 @@ def test_generate_writes_a_harbor_task(tmp_path): assert [server["name"] for server in config["environment"]["mcp_servers"]] == ["bench"] instruction = (task_dir / "steps/run/instruction.md").read_text() assert "Kirkland Signature" in instruction + assert "Reach the item however works" in instruction assert "https://www.costco.com/" in instruction setup = (task_dir / "steps/run/workdir/setup.sh").read_text() assert 'open-start-url "https://www.costco.com/"' in setup @@ -71,6 +72,9 @@ def test_finalize_records_the_first_report_only(tmp_path): def test_score_counts_a_block_apart_from_other_failures(tmp_path): verifier = tmp_path / "verifier" (tmp_path / "finalize.json").write_text(json.dumps({"outcome": "blocked"})) + (tmp_path / "task.json").write_text( + json.dumps({"id": "t", "cart_url": "https://example.com/cart", "cart_match": "x"}) + ) source = ( (BENCH / "score") .read_text() @@ -90,9 +94,11 @@ def test_score_counts_a_block_apart_from_other_failures(tmp_path): assert rewards == { "reward": 0.0, "finalized": 1.0, + "cart_verified": 0.0, "outcome_failed": 0.0, "outcome_blocked": 1.0, "outcome_website_error": 0.0, "outcome_kernel_error": 0.0, } assert "error" in json.loads((verifier / "final-page.json").read_text()) + assert json.loads((verifier / "cart-check.json").read_text())["verified"] is False diff --git a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify index 1b202c0..945888f 100755 --- a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify +++ b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify @@ -3,7 +3,7 @@ The suite's verifier owns the task reward. This check records whether the agent called the Kernel MCP browser tools its arm enables -(`execute_playwright_code`, `browser_repl`), and whether every call used the +(`execute_playwright_code`, `browser_repl`, `computer_action`), and whether every call used the browser Harbor assigned to the trial, so a failed task can be told apart from a trial that never exercised the toolset. It reads the calls mcp-call-log recorded at the server, which every agent harness goes through, and does not @@ -22,7 +22,11 @@ from typing import Any LOGS = Path("/var/log/kernel-mcp") VERIFIER = Path("/logs/verifier") -TOOLS = {"playwright": "execute_playwright_code", "repl": "browser_repl"} +TOOLS = { + "playwright": "execute_playwright_code", + "repl": "browser_repl", + "computer": "computer_action", +} def read_calls(path: Path) -> list[dict[str, Any]]: diff --git a/toolsets/kernel-mcp-local/toolset.py b/toolsets/kernel-mcp-local/toolset.py index a492b2e..518dac8 100644 --- a/toolsets/kernel-mcp-local/toolset.py +++ b/toolsets/kernel-mcp-local/toolset.py @@ -35,6 +35,10 @@ "browser_repl", "- `browser_repl` runs JavaScript in a persistent REPL attached to your browser. If you have not used it before, start with `repl.help()`.", ), + "computer": ( + "computer_action", + "- `computer_action` moves the mouse, clicks, types, scrolls and takes screenshots at the operating-system level, as a person would. Use it when a site reacts to scripted clicks or typing, or when they do not take effect; take a screenshot first to find coordinates.", + ), } INSTRUCTION = """ From 26c94d6297db82ac812fc1a647b82902b3cfa1cb Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:10:09 +0000 Subject: [PATCH 4/5] Fix pinned configurator test for the zoro guidance --- tests/test_pinned.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pinned.py b/tests/test_pinned.py index 3157f48..9dfa2fd 100644 --- a/tests/test_pinned.py +++ b/tests/test_pinned.py @@ -27,4 +27,5 @@ def test_personas_become_chromium_env(): zoro = _configure("zoro.com") assert len(zoro["browser"]["chromium_env"]) == 2 assert zoro["browser"]["proxy_create"]["type"] == "residential" - assert "30 seconds" in zoro["steering"] + assert "load it again once" in zoro["steering"] + assert _configure("wbmason.com")["browser"]["vm_setup"] From 315e42d4266be640910672994732a938a156c239 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:48:32 +0000 Subject: [PATCH 5/5] Fix the lowes item, tell an unverifiable cart from an empty one, and use the hand-tested zoro config --- configurators/pinned/configure | 7 ++-- configurators/pinned/sites.toml | 24 +++++--------- suites/retail-cart/image/bench/score | 49 +++++++++++++++++++++++----- suites/retail-cart/tasks.toml | 6 ++-- tests/test_pinned.py | 2 +- tests/test_retail_cart.py | 20 +++++++++++- 6 files changed, 74 insertions(+), 34 deletions(-) diff --git a/configurators/pinned/configure b/configurators/pinned/configure index 16be541..b5ec6f8 100755 --- a/configurators/pinned/configure +++ b/configurators/pinned/configure @@ -2,9 +2,8 @@ """Choose browser settings from the fixed per-site table in sites.toml. The task's site comes from `metadata.site` in its task.toml. Proxies are US -ISP or US residential, created per trial. A `persona` of `macos` or `webgl` -restarts Chromium with the matching launcher variables; `webgl` rotates -through the listed WebGL personas per trial. `vm_setup` commands run as root +ISP or US residential, created per trial. A `persona` of `macos` restarts +Chromium with the macOS persona's launcher variables. `vm_setup` commands run as root in the browser's VM before the first page load. stdin: {"task": {"config": str, "instruction": str}, "args": {}} @@ -32,8 +31,6 @@ def main() -> int: persona = entry.get("persona") if persona == "macos": browser["chromium_env"] = TABLE["macos_persona"] - elif persona == "webgl": - browser["chromium_env"] = TABLE["webgl_personas"] elif persona is not None: raise SystemExit(f"Unknown persona {persona!r} for {site}") if entry.get("vm_setup"): diff --git a/configurators/pinned/sites.toml b/configurators/pinned/sites.toml index 1331411..f2b2469 100644 --- a/configurators/pinned/sites.toml +++ b/configurators/pinned/sites.toml @@ -1,23 +1,13 @@ -# Browser settings per site, measured by hand on 2026-10-29..31. Each entry is -# merged over the arm's config. `proxy` gives every trial its own proxy of that -# type; `persona` restarts Chromium with KERNEL_* launcher variables (`webgl` -# picks one set per trial at random); `vm_setup` commands run as root in the +# Browser settings per site, measured by hand on 2026-10-29 to 2026-11-01. +# Each entry is merged over the arm's config. `proxy` gives every trial its own +# proxy of that type; `persona = "macos"` restarts Chromium with the macOS +# persona's launcher variables; `vm_setup` commands run as root in the # browser's VM before the first page load; `steering` is added to the # instruction. [macos_persona] KERNEL_MAC_PERSONA = "1" -[[webgl_personas]] -KERNEL_WEBGL_VENDOR = "Google Inc. (NVIDIA Corporation)" -KERNEL_WEBGL_RENDERER = "ANGLE (NVIDIA Corporation, NVIDIA GeForce RTX 3060/PCIe/SSE2, OpenGL 4.5.0)" -KERNEL_CANVAS_AA_OFFSET = "-2" - -[[webgl_personas]] -KERNEL_WEBGL_VENDOR = "Google Inc. (AMD)" -KERNEL_WEBGL_RENDERER = "ANGLE (AMD, AMD Custom GPU 0405 (radeonsi vangogh LLVM 15.0.7), OpenGL 4.6 (Core Profile) Mesa 23.2.1)" -KERNEL_CANVAS_AA_OFFSET = "3" - [sites."amazon.com"] browser = { stealth = true, gpu = true } proxy = "isp" @@ -108,9 +98,11 @@ On this site use computer_action for clicks and typing: scripted clicks and typi """ [sites."zoro.com"] -browser = { stealth = true, gpu = true } +# A fresh residential IP per trial and one reload passed 5/5 by hand; GPU and +# the WebGL persona made no difference there and got a slider every time in +# the benchmark. +browser = { stealth = true } proxy = "residential" -persona = "webgl" steering = """ On zoro.com the first page load is often refused with a 403 or a bot-check page. That refusal sets a cookie, and loading the same URL once more usually shows the real site, so if the first load is blocked, load it again once. If the page says "Access is temporarily restricted", reloading will not help: report the task as blocked. """ diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 185dc3f..175d4a8 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -8,9 +8,12 @@ told apart from other website errors, Kernel errors and the agent's own failures. The browser's final URL and title are saved, then the verifier checks the -cart itself: it opens the task's `cart_url`, or fetches its `cart_api` from -the page the agent ended on, and sets `cart_verified` when the text matches -`cart_match`. cart-check.json keeps what it saw. +cart itself: it reads the page the agent ended on when that is already the +cart, otherwise opens the task's `cart_url`, or fetches its `cart_api` from +the page, and sets `cart_verified` when the text matches `cart_match`. When +the check could not see the cart, because loading it failed or showed a bot +check, it sets `cart_unverifiable` instead, so a failed check is not counted +as an empty cart. cart-check.json keeps what it saw. """ from __future__ import annotations @@ -19,6 +22,7 @@ import json import os import re import shutil +import urllib.parse import urllib.request from pathlib import Path @@ -29,6 +33,21 @@ FAILURES = ("failed", "blocked", "website_error", "kernel_error") FINAL_PAGE = "return { url: page.url(), title: await page.title() };" +# A bot check or access-denied page in place of the cart. +CHALLENGE = re.compile( + r"captcha|/checkpoint/|validateCaptcha|verify (yourself|access|you are)|access denied" + r"|access is temporarily restricted|press (and|&) hold|are you a robot", + re.IGNORECASE, +) + +CURRENT_PAGE = """ +return { + url: page.url(), + title: await page.title(), + text: await page.evaluate(() => (document.body ? document.body.innerText : "")), +}; +""" + CART_PAGE = """ const response = await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); await page.waitForLoadState("networkidle", { timeout: 15000 }).catch(() => {}); @@ -70,16 +89,28 @@ def execute(code: str, timeout_sec: int) -> dict: return result.get("result") or {} -def check_cart(task: dict) -> dict: +def same_page(url: str, other: str) -> bool: + a, b = urllib.parse.urlsplit(url), urllib.parse.urlsplit(other) + return (a.netloc, a.path.rstrip("/").lower()) == (b.netloc, b.path.rstrip("/").lower()) + + +def check_cart(task: dict, final_page: dict) -> dict: if task.get("cart_api"): seen = execute(CART_API % json.dumps(task["cart_api"]), 60) elif task.get("cart_url"): - seen = execute(CART_PAGE % json.dumps(task["cart_url"]), 90) + if same_page(final_page.get("url") or "", task["cart_url"]): + seen = execute(CURRENT_PAGE, 60) + else: + seen = execute(CART_PAGE % json.dumps(task["cart_url"]), 90) else: - return {"error": "task has no cart_url or cart_api"} + return {"verified": False, "unverifiable": True, "error": "task has no cart_url or cart_api"} text = seen.pop("text", None) or "" match = re.search(task["cart_match"], text, re.IGNORECASE) if task.get("cart_match") else None seen["verified"] = match is not None + challenge = CHALLENGE.search(f"{seen.get('url', '')} {seen.get('title', '')} {text[:3000]}") + seen["unverifiable"] = match is None and ("error" in seen or challenge is not None) + if challenge and match is None: + seen["challenge"] = challenge.group(0) if match: seen["excerpt"] = text[max(0, match.start() - 200) : match.end() + 200] else: @@ -98,14 +129,16 @@ def main() -> int: task = json.loads(TASK.read_text()) if TASK.exists() else {} if task: shutil.copy2(TASK, VERIFIER / "task.json") - (VERIFIER / "final-page.json").write_text(json.dumps(execute(FINAL_PAGE, 30), indent=2)) - cart = check_cart(task) + final_page = execute(FINAL_PAGE, 30) + (VERIFIER / "final-page.json").write_text(json.dumps(final_page, indent=2)) + cart = check_cart(task, final_page) (VERIFIER / "cart-check.json").write_text(json.dumps(cart, indent=2)) outcome = report.get("outcome") if report is not None else None rewards = { "reward": float(outcome == "success"), "finalized": float(report is not None), "cart_verified": float(cart.get("verified") is True), + "cart_unverifiable": float(cart.get("unverifiable") is True), **{f"outcome_{name}": float(outcome == name) for name in FAILURES}, } (VERIFIER / "reward.json").write_text(json.dumps(rewards, indent=2)) diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml index e9a98a5..233d945 100644 --- a/suites/retail-cart/tasks.toml +++ b/suites/retail-cart/tasks.toml @@ -28,12 +28,12 @@ cart_url = "https://www.costco.com/CheckoutCartDisplayView" cart_match = 'Ultra Plush' [[tasks]] -id = "lowes-stanley-kit" +id = "lowes-metal-kit" site = "lowes.com" start_url = "https://www.lowes.com/" -item = "the Stanley 27-pack 0.375-in metal kit" +item = "the 27-pack 0.375-in metal kit (sold under the Bostitch or Stanley brand)" cart_url = "https://www.lowes.com/cart" -cart_match = 'Stanley' +cart_match = '27.{0,4}Pack.{0,30}Metal Kit' [[tasks]] id = "staples-copy-paper" diff --git a/tests/test_pinned.py b/tests/test_pinned.py index 9dfa2fd..157ee6f 100644 --- a/tests/test_pinned.py +++ b/tests/test_pinned.py @@ -25,7 +25,7 @@ def test_every_retail_cart_site_has_a_pinned_configuration(): def test_personas_become_chromium_env(): assert _configure("amazon.com")["browser"]["chromium_env"] == {"KERNEL_MAC_PERSONA": "1"} zoro = _configure("zoro.com") - assert len(zoro["browser"]["chromium_env"]) == 2 + assert "chromium_env" not in zoro["browser"] assert zoro["browser"]["proxy_create"]["type"] == "residential" assert "load it again once" in zoro["steering"] assert _configure("wbmason.com")["browser"]["vm_setup"] diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index c80885b..962c9a5 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -95,10 +95,28 @@ def test_score_counts_a_block_apart_from_other_failures(tmp_path): "reward": 0.0, "finalized": 1.0, "cart_verified": 0.0, + "cart_unverifiable": 1.0, "outcome_failed": 0.0, "outcome_blocked": 1.0, "outcome_website_error": 0.0, "outcome_kernel_error": 0.0, } assert "error" in json.loads((verifier / "final-page.json").read_text()) - assert json.loads((verifier / "cart-check.json").read_text())["verified"] is False + check = json.loads((verifier / "cart-check.json").read_text()) + assert check["verified"] is False and check["unverifiable"] is True + + +def test_cart_check_tells_a_bot_check_from_an_empty_cart(monkeypatch): + source = (BENCH / "score").read_text().replace('if __name__ == "__main__":', "if False:") + score = {} + exec(compile(source, "score", "exec"), score) + task = {"cart_url": "https://shop.test/cart", "cart_match": "gloves"} + + def page(text, url="https://shop.test/cart"): + score["execute"] = lambda code, timeout: {"url": url, "title": "", "text": text} + return score["check_cart"](task, {"url": url}) + + assert page("Nitrile gloves, qty 1")["verified"] is True + assert page("Your cart is empty")["unverifiable"] is False + blocked = page("Please verify yourself to continue", "https://shop.test/splashui/captcha") + assert blocked["verified"] is False and blocked["unverifiable"] is True