Ask retail-cart for the outcome, verify the cart, and add the pinned per-site arm

README.md

@@ -127,6 +127,7 @@ Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot
 |---|---|---|
 | `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | `stealth: true` and no proxy, so each browser gets its own egress IP from the stealth proxy pool |
 | `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. |
+| `codex-repl-pinned` | Same | A fixed per-site table (`configurators/pinned/sites.toml`): stealth, GPU, a new US ISP or residential proxy per trial, and on some sites a macOS or WebGL persona applied by restarting Chromium with `KERNEL_*` launcher variables |
 
 The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_<name>` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site:
 

arms/codex-repl-pinned.toml

@@ -0,0 +1,16 @@
+# Same as codex-repl, but each site's browser, proxy and persona come from the
+# fixed table in configurators/pinned/sites.toml, with a new proxy per trial.
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "kernel-mcp-local"
+
+[agent_kwargs]
+disable_web_search = true
+
+[toolset_args]
+toolsets = ["repl", "computer"]
+
+[browser]
+config = { stealth = true }
+configurator = "pinned"

benchmarks/retail-cart.toml

@@ -1,5 +1,5 @@
 # Guest add-to-cart flows on bot-protected retail and industrial-supply sites,
 # driven through the browser REPL, with a stealth browser and with the config
-# registry's per-site settings.
+# registry's per-site settings, and with a fixed per-site table.
 suite = "retail-cart"
-arms = ["codex-repl", "codex-repl-site-config"]
+arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned"]

configurators/pinned/configure

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Choose browser settings from the fixed per-site table in sites.toml.
+
+The task's site comes from `metadata.site` in its task.toml. Proxies are US
+ISP or US residential, created per trial. A `persona` of `macos` restarts
+Chromium with the macOS persona's launcher variables. `vm_setup` commands run as root
+in the browser's VM before the first page load.
+
+stdin:  {"task": {"config": str, "instruction": str}, "args": {}}
+stdout: {"browser": {...}, "steering": str | null, "provenance": {...}}
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tomllib
+from pathlib import Path
+
+TABLE = tomllib.loads((Path(__file__).parent / "sites.toml").read_text())
+
+
+def main() -> int:
+    task = json.load(sys.stdin)["task"]
+    site = tomllib.loads(task["config"]).get("metadata", {}).get("site")
+    entry = TABLE["sites"].get(site)
+    if entry is None:
+        raise SystemExit(f"No pinned configuration for site {site!r}")
+    browser = dict(entry["browser"])
+    browser["proxy_create"] = {"type": entry["proxy"], "config": {"country": "US"}}
+    persona = entry.get("persona")
+    if persona == "macos":
+        browser["chromium_env"] = TABLE["macos_persona"]
+    elif persona is not None:
+        raise SystemExit(f"Unknown persona {persona!r} for {site}")
+    if entry.get("vm_setup"):
+        browser["vm_setup"] = entry["vm_setup"]
+    json.dump(
+        {
+            "browser": browser,
+            "steering": entry.get("steering"),
+            "provenance": {"site": site, "proxy": entry["proxy"], "persona": persona},
+        },
+        sys.stdout,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

configurators/pinned/sites.toml

@@ -0,0 +1,109 @@
+# Browser settings per site, measured by hand on 2026-10-29 to 2026-11-01.
+# Each entry is merged over the arm's config. `proxy` gives every trial its own
+# proxy of that type; `persona = "macos"` restarts Chromium with the macOS
+# persona's launcher variables; `vm_setup` commands run as root in the
+# browser's VM before the first page load; `steering` is added to the
+# instruction.
+
+[macos_persona]
+KERNEL_MAC_PERSONA = "1"
+
+[sites."amazon.com"]
+browser = { stealth = true, gpu = true }
+proxy = "isp"
+persona = "macos"
+
+[sites."costco.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."dickssportinggoods.com"]
+browser = { stealth = true, gpu = true }
+proxy = "isp"
+persona = "macos"
+
+[sites."ebay.com"]
+browser = { stealth = true, gpu = true }
+proxy = "isp"
+
+[sites."guitarcenter.com"]
+# Measured passing only with Web Bot Auth signatures, which this project does not have.
+browser = { stealth = true }
+proxy = "isp"
+steering = """
+guitarcenter.com denies its home page (/), /search and /cart to automated browsers with an "Access Denied" page, but search results at /search/ with a trailing slash, product pages and /cart/index load. Search by opening https://www.guitarcenter.com/search/?Ntt=<words joined with +>, open the product, and click Add to Cart. Show the cart at https://www.guitarcenter.com/cart/index; if it does not list the item, report the add-to-cart confirmation the product page showed.
+"""
+
+[sites."harborfreight.com"]
+browser = { stealth = true, gpu = true }
+proxy = "isp"
+persona = "macos"
+
+[sites."homedepot.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."lowes.com"]
+browser = { stealth = false }
+proxy = "isp"
+
+[sites."mouser.com"]
+browser = { stealth = true }
+proxy = "residential"
+steering = """
+mouser.com denies its search results, category and product pages (/c/..., /ProductDetail/...) to automated browsers, so do not press Enter in the search box. Find the part number instead: type the item into the search box and read the suggestions without submitting, or search the web (for example DuckDuckGo) for "mouser <item>" and take the part number from the results. Then open https://www.mouser.com/en/Cart/, enter the manufacturer or Mouser part number and a quantity in its add-by-part-number row, and click Add.
+
+Pages may show a bot-check interstitial that clears by itself within about 12 seconds. Wait up to 30 seconds before treating a page as blocked.
+"""
+
+[sites."newark.com"]
+browser = { stealth = true }
+proxy = "isp"
+steering = """
+newark.com denies its search results, category and product pages (/search, /c/..., /dp/...) to automated browsers, so pressing Enter in the search box does nothing. The site's older store pages still work; use them:
+
+1. Type the item into the search box and read the suggestions without submitting, or open https://www.newark.com/webapp/wcs/stores/servlet/AjaxSearchLookAhead?searchTerm=<words>&storeId=10194&catalogId=15003&langId=-1. Suggestions link to /dp/<order code>; note the order code of a matching product.
+2. Open https://www.newark.com/webapp/wcs/stores/servlet/ProductDisplay?storeId=10194&catalogId=15003&langId=-1&partNumber=<order code>.
+3. Clicking its Add to Cart link does nothing; navigate to the link's href instead.
+4. Show the cart at https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1. The /cart page stays empty.
+"""
+
+[sites."radwell.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."staples.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."target.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."walmart.com"]
+browser = { stealth = true }
+proxy = "isp"
+
+[sites."wbmason.com"]
+# The captcha-solver extension takes over the site's Cloudflare Turnstile check
+# and never completes it, so its Turnstile interceptor is emptied before the
+# first page load.
+browser = { stealth = true }
+proxy = "residential"
+vm_setup = ["printf '/* disabled */\\n' > /home/kernel/extensions/kernel/capmonster/interceptors/turnstileInterceptor.js"]
+steering = """
+wbmason.com shows a "Verify Access to Continue" page with a Cloudflare Turnstile check before the site. Do not reload or navigate while it is showing: that turns it into a hard block. Wait up to 30 seconds for it to pass by itself; if a checkbox appears, click it once with computer_action.
+
+On this site use computer_action for clicks and typing: scripted clicks and typing in the search box do not take effect.
+"""
+
+[sites."zoro.com"]
+# A fresh residential IP per trial and one reload passed 5/5 by hand; GPU and
+# the WebGL persona made no difference there and got a slider every time in
+# the benchmark.
+browser = { stealth = true }
+proxy = "residential"
+steering = """
+On zoro.com the first page load is often refused with a 403 or a bot-check page. That refusal sets a cookie, and loading the same URL once more usually shows the real site, so if the first load is blocked, load it again once. If the page says "Access is temporarily restricted", reloading will not help: report the task as blocked.
+"""
+

harness/environment.py

@@ -12,12 +12,23 @@
 so `delete_run_proxies` can remove any a crashed trial left behind. A
 trial whose proxy cannot be created fails with `ProxyCreationError`, so
 reports can count those trials apart from other infrastructure failures.
+
+Two entries change the browser's VM right after it is created, before the
+task container starts and before the first page load. `chromium_env` is a set
+of `KERNEL_*` launcher variables, or a list of sets of which each trial picks
+one at random: Chromium is restarted with them and the page is checked for
+the persona or WebGL renderer they ask for. `vm_setup` is a list of shell
+commands then run as root. A trial where either fails fails with
+`VmSetupError`.
 """
 
 from __future__ import annotations
 
+import asyncio
 import json
 import os
+import random
+import shlex
 import uuid
 from pathlib import Path
 from typing import Any
@@ -27,13 +38,53 @@
 
 CONFIG_FILE = "kernel-config.json"
 PROXY_CREATE = "proxy_create"
+CHROMIUM_ENV = "chromium_env"
+VM_SETUP = "vm_setup"
 RUN_ID_ENV = "BENCH_RUN_ID"
 
 
 class ProxyCreationError(RuntimeError):
     """The trial's proxy could not be created, so no browser was started."""
 
 
+class VmSetupError(RuntimeError):
+    """The browser's VM could not be set up for the trial before the first page load."""
+
+
+SUPERVISOR_PROGRAM = """[program:chromium]
+command=/usr/bin/env {env} /usr/local/bin/chromium-launcher
+autostart=false
+autorestart=true
+startsecs=0
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+stopwaitsecs=1
+stdout_logfile=/var/log/supervisord/chromium
+redirect_stderr=true
+"""
+
+RESTART_CHROMIUM = """#!/bin/bash
+set -euo pipefail
+mv /tmp/bench-chromium.conf /etc/supervisor/conf.d/services/chromium.conf
+supervisorctl reread
+supervisorctl update
+sleep 2
+supervisorctl start chromium
+sleep 3
+supervisorctl status chromium
+"""
+
+PAGE_IDENTITY = """return await page.evaluate(() => {
+  const gl = document.createElement("canvas").getContext("webgl");
+  const info = gl && gl.getExtension("WEBGL_debug_renderer_info");
+  return {
+    userAgent: navigator.userAgent,
+    renderer: info ? gl.getParameter(info.UNMASKED_RENDERER_WEBGL) : null,
+  };
+});"""
+
+
 def proxy_prefix(run_id: str) -> str:
     return f"bench-{run_id}-"
 
@@ -54,10 +105,17 @@ class KernelTaskEnvironment(KernelEnvironment):
     def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None:
         self._proxy_create: dict[str, Any] | None = None
         self._proxy_id: str | None = None
+        self._chromium_env: dict[str, str] | None = None
+        self._vm_setup: list[str] = []
         config_path = Path(environment_dir) / CONFIG_FILE
         if config_path.is_file():
             overrides = json.loads(config_path.read_text())
             self._proxy_create = overrides.pop(PROXY_CREATE, None)
+            chromium_env = overrides.pop(CHROMIUM_ENV, None)
+            if isinstance(chromium_env, list):
+                chromium_env = random.choice(chromium_env)
+            self._chromium_env = chromium_env
+            self._vm_setup = overrides.pop(VM_SETUP, None) or []
             kwargs["browser_create_overrides"] = overrides
         super().__init__(environment_dir, *args, **kwargs)
 
@@ -76,6 +134,71 @@ async def start(self, force_build: bool) -> None:
             self._browser_create_overrides["proxy_id"] = proxy.id
         await super().start(force_build)
 
+    def _browser_create_kwargs(self) -> dict[str, Any]:
+        kwargs = super()._browser_create_kwargs()
+        # GPU browsers have a fixed memory size and reject a memory request.
+        if kwargs.get("gpu"):
+            kwargs.pop("memory", None)
+        return kwargs
+
+    async def _create_browser(self) -> Any:
+        response = await super()._create_browser()
+        try:
+            if self._chromium_env:
+                await self._restart_chromium(response.session_id)
+            for command in self._vm_setup:
+                await self._run_as_root(response.session_id, command)
+        except Exception as error:
+            raise VmSetupError(f"Could not set up the browser VM: {error}") from error
+        return response
+
+    async def _run_as_root(self, session_id: str, command: str) -> None:
+        result = await self._client.browsers.process.exec(
+            session_id, command="bash", args=["-c", command], as_root=True, timeout_sec=60
+        )
+        if result.exit_code != 0:
+            raise RuntimeError(f"{command!r} exited {result.exit_code}")
+        self.logger.info(f"Ran in the browser VM: {command}")
+
+    async def _restart_chromium(self, session_id: str) -> None:
+        env = " ".join(
+            f"{key}={shlex.quote(str(value))}" for key, value in self._chromium_env.items()
+        )
+        browsers = self._client.browsers
+        await browsers.fs.write_file(
+            session_id, SUPERVISOR_PROGRAM.format(env=env).encode(), path="/tmp/bench-chromium.conf"
+        )
+        await browsers.fs.write_file(
+            session_id, RESTART_CHROMIUM.encode(), path="/tmp/bench-restart-chromium.sh"
+        )
+        result = await browsers.process.exec(
+            session_id,
+            command="bash",
+            args=["/tmp/bench-restart-chromium.sh"],
+            as_root=True,
+            timeout_sec=60,
+        )
+        if result.exit_code != 0:
+            raise RuntimeError(f"restart script exited {result.exit_code}")
+        identity = None
+        for _ in range(15):
+            check = await browsers.playwright.execute(session_id, code=PAGE_IDENTITY)
+            if check.success:
+                identity = check.result or {}
+                break
+            await asyncio.sleep(2)
+        if identity is None:
+            raise RuntimeError("Chromium did not come back after the restart")
+        if (
+            self._chromium_env.get("KERNEL_MAC_PERSONA")
+            and "Macintosh" not in identity["userAgent"]
+        ):
+            raise RuntimeError(f"expected a macOS user agent, got {identity['userAgent']}")
+        renderer = self._chromium_env.get("KERNEL_WEBGL_RENDERER")
+        if renderer and identity["renderer"] != renderer:
+            raise RuntimeError(f"expected WebGL renderer {renderer}, got {identity['renderer']}")
+        self.logger.info(f"Chromium restarted with {sorted(self._chromium_env)}: {identity}")
+
     async def stop(self, delete: bool) -> None:
         try:
             await super().stop(delete)