From 465fd2bc8c567eadb839c8b924713f6ee3290328 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:40:14 +0000 Subject: [PATCH 1/2] Add retail-cart benchmark: guest add-to-cart flows with the browser REPL --- .github/workflows/ci.yml | 2 + .github/workflows/retail-cart.yml | 45 +++++ README.md | 15 ++ arms/codex-repl-site-config.toml | 21 +++ arms/codex-repl.toml | 16 ++ benchmarks/retail-cart.toml | 5 + suites/retail-cart/image/Dockerfile | 13 ++ suites/retail-cart/image/bench/finalize-mcp | 157 ++++++++++++++++++ suites/retail-cart/image/bench/open-start-url | 55 ++++++ suites/retail-cart/image/bench/score | 73 ++++++++ suites/retail-cart/suite.py | 117 +++++++++++++ suites/retail-cart/tasks.toml | 99 +++++++++++ tests/test_retail_cart.py | 98 +++++++++++ 13 files changed, 716 insertions(+) create mode 100644 .github/workflows/retail-cart.yml create mode 100644 arms/codex-repl-site-config.toml create mode 100644 arms/codex-repl.toml create mode 100644 benchmarks/retail-cart.toml create mode 100644 suites/retail-cart/image/Dockerfile create mode 100755 suites/retail-cart/image/bench/finalize-mcp create mode 100755 suites/retail-cart/image/bench/open-start-url create mode 100644 suites/retail-cart/image/bench/score create mode 100644 suites/retail-cart/suite.py create mode 100644 suites/retail-cart/tasks.toml create mode 100644 tests/test_retail_cart.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12666a3..72ff36b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: - run: uv run pytest -q - name: Prepare every static arm for one task run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared" + - name: Prepare the static retail-cart arm + run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart" report: runs-on: ubuntu-latest diff --git a/.github/workflows/retail-cart.yml b/.github/workflows/retail-cart.yml new file mode 100644 index 0000000..83183ff --- /dev/null +++ b/.github/workflows/retail-cart.yml @@ -0,0 +1,45 @@ +name: Retail cart flows with the browser REPL + +on: + workflow_dispatch: + inputs: + arms: + description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart.toml) + required: false + default: "" + type: string + tasks: + description: Comma-separated task IDs, or all + required: false + default: all + type: string + attempts: + description: Trials per task and arm + required: false + default: "10" + type: string + concurrency: + description: Concurrent trials per arm + required: false + default: "16" + type: string + kernel_mcp_server_ref: + description: kernel/kernel-mcp-server git ref to build (empty builds main) + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + benchmark: + uses: ./.github/workflows/_run.yml + secrets: inherit + with: + benchmark: retail-cart + arms: ${{ inputs.arms }} + tasks: ${{ inputs.tasks }} + attempts: ${{ inputs.attempts }} + concurrency: ${{ inputs.concurrency }} + toolset_ref: ${{ inputs.kernel_mcp_server_ref }} diff --git a/README.md b/README.md index 3eb3e98..8f19fc2 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,21 @@ The Kernel MCP server on ClawBench. ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whether the agent called Kernel MCP's `execute_playwright_code` and used its assigned browser on every call, so a failed task can be told apart from a trial that never used the toolset. +### `retail-cart` + +Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to search for one item, open its product page, add it to the cart without logging in, and report the cart. + +| Arm | Agent | Browser | +|---|---|---| +| `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | `stealth: true` and no proxy, so each browser gets its own egress IP from the stealth proxy pool | +| `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. | + +The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: + +```bash +uv run bench run retail-cart --attempts 10 +``` + ## Extending ### Add an arm diff --git a/arms/codex-repl-site-config.toml b/arms/codex-repl-site-config.toml new file mode 100644 index 0000000..cf6d092 --- /dev/null +++ b/arms/codex-repl-site-config.toml @@ -0,0 +1,21 @@ +# Same as codex-repl, but each task's browser and proxy come from Kernel's +# config registry for the task's site, and the registry's guidance for that +# site is added to the instruction. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } +configurator = "site-config" + +[browser.configurator_args] +model = "openai/gpt-6-luna" +steering = true diff --git a/arms/codex-repl.toml b/arms/codex-repl.toml new file mode 100644 index 0000000..0817a68 --- /dev/null +++ b/arms/codex-repl.toml @@ -0,0 +1,16 @@ +# Codex driving the browser only through Kernel MCP's browser REPL, with a +# fixed stealth browser. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +# Codex's own web tool would let the agent answer without the browser. +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml new file mode 100644 index 0000000..4d486a6 --- /dev/null +++ b/benchmarks/retail-cart.toml @@ -0,0 +1,5 @@ +# Guest add-to-cart flows on bot-protected retail and industrial-supply sites, +# driven through the browser REPL, with a stealth browser and with the config +# registry's per-site settings. +suite = "retail-cart" +arms = ["codex-repl", "codex-repl-site-config"] diff --git a/suites/retail-cart/image/Dockerfile b/suites/retail-cart/image/Dockerfile new file mode 100644 index 0000000..4362d77 --- /dev/null +++ b/suites/retail-cart/image/Dockerfile @@ -0,0 +1,13 @@ +# Task image for retail-cart. The browser is the Kernel VM this container runs +# in, so the image only holds the finalize tool, the start-URL setup and the +# scorer. Toolsets append their own layers. +FROM node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY bench/ /opt/bench/ +RUN chmod +x /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score + +WORKDIR /app diff --git a/suites/retail-cart/image/bench/finalize-mcp b/suites/retail-cart/image/bench/finalize-mcp new file mode 100755 index 0000000..9339d7a --- /dev/null +++ b/suites/retail-cart/image/bench/finalize-mcp @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""stdio MCP server with one tool, `finalize`, the agent's report of the task. + +The first call writes the report to /logs/agent/finalize.json, where the +verifier reads it; later calls are refused, so the first report stands. +`evidence` and `reasoning` come before `outcome` in the schema so the agent +writes down what it saw and checks it before it gives a verdict. + +`outcome` separates the agent's own failures from failures it could not +control: the website's bot protection, other website errors, and Kernel's. +""" + +from __future__ import annotations + +import json +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +REPORT = Path("/logs/agent/finalize.json") + +OUTCOMES = { + "success": "every requirement of the task is met and the evidence shows it.", + "failed": ( + "the task is not done, including when what the task asks for does not exist or is " + "unavailable on the site." + ), + "blocked": ( + "the website's bot protection stopped you: an access-denied or 403 page, a CAPTCHA or " + "press-and-hold challenge you could not pass, or a rate-limit page." + ), + "website_error": ( + "the website kept you from finishing for another reason: it showed an error or failed " + "to load." + ), + "kernel_error": ( + "the Kernel browser or its tools kept you from finishing: the browser REPL or the " + "browser itself errored, hung, or crashed." + ), +} + +TOOL = { + "name": "finalize", + "description": ( + "Report the outcome of the task. Call this exactly once, when the task is done or you " + "are certain it cannot be done, and stop afterwards. Your report is the result of the " + "task." + ), + "inputSchema": { + "type": "object", + "properties": { + "evidence": { + "type": "string", + "description": ( + "What you observed in the browser that shows whether the task is done: the " + "values, text and URLs the task asks for, quoted as the page shows them, " + "including everything the task asks you to report." + ), + }, + "reasoning": { + "type": "string", + "description": ( + "Check the evidence against each requirement of the task in turn and say " + "whether it is met. Name anything you could not verify." + ), + }, + "outcome": { + "type": "string", + "enum": list(OUTCOMES), + "description": "How the task ended. " + + " ".join(f"{name}: {meaning}" for name, meaning in OUTCOMES.items()), + }, + }, + "required": ["evidence", "reasoning", "outcome"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": False, "idempotentHint": False}, +} + + +def finalize(arguments: dict[str, Any]) -> tuple[str, bool]: + if REPORT.exists(): + return "The task was already finalized. Stop now.", True + evidence, reasoning, outcome = ( + arguments.get(key) for key in ("evidence", "reasoning", "outcome") + ) + if not isinstance(evidence, str) or not evidence.strip(): + return "evidence must be a non-empty string.", True + if not isinstance(reasoning, str) or not reasoning.strip(): + return "reasoning must be a non-empty string.", True + if outcome not in OUTCOMES: + return f"outcome must be one of {', '.join(OUTCOMES)}.", True + REPORT.parent.mkdir(parents=True, exist_ok=True) + report = { + "outcome": outcome, + "evidence": evidence, + "reasoning": reasoning, + "finalized_at": datetime.now(UTC).isoformat(), + } + REPORT.write_text(json.dumps(report, indent=2)) + return "Recorded. The task is finalized; stop now.", False + + +def handle(message: dict[str, Any]) -> dict[str, Any] | None: + method = message.get("method") + if "id" not in message: + return None + if method == "initialize": + params = message.get("params") or {} + result: dict[str, Any] = { + "protocolVersion": params.get("protocolVersion", "2025-06-18"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": "bench", "version": "1.0.0"}, + } + elif method == "ping": + result = {} + elif method == "tools/list": + result = {"tools": [TOOL]} + elif method == "tools/call": + params = message.get("params") or {} + if params.get("name") != TOOL["name"]: + text, is_error = f"Unknown tool {params.get('name')!r}.", True + else: + text, is_error = finalize(params.get("arguments") or {}) + result = {"content": [{"type": "text", "text": text}], "isError": is_error} + else: + return { + "jsonrpc": "2.0", + "id": message["id"], + "error": {"code": -32601, "message": f"Method not found: {method}"}, + } + return {"jsonrpc": "2.0", "id": message["id"], "result": result} + + +def main() -> int: + for line in sys.stdin: + if not line.strip(): + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + response: dict[str, Any] | None = { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Parse error"}, + } + else: + response = handle(message) if isinstance(message, dict) else None + if response is not None: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/suites/retail-cart/image/bench/open-start-url b/suites/retail-cart/image/bench/open-start-url new file mode 100755 index 0000000..d6dd7f3 --- /dev/null +++ b/suites/retail-cart/image/bench/open-start-url @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Open the task's start URL in the trial's browser before the agent starts. + +Both arms start on the same loaded page, so neither spends its own time on +the first navigation. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +CODE = """ +await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); +await page.waitForLoadState("load", { timeout: 15000 }).catch(() => {}); +return page.url(); +""" + + +def main(url: str) -> int: + base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") + request = urllib.request.Request( + f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", + data=json.dumps({"code": CODE % json.dumps(url), "timeout_sec": 90}).encode(), + headers={ + "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", + "Content-Type": "application/json", + }, + method="POST", + ) + # The container's DNS can lag its start; retry until the API is reachable. + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + result = json.load(response) + break + except urllib.error.HTTPError: + raise + except urllib.error.URLError: + if attempt == 4: + raise + time.sleep(2 * (attempt + 1)) + if not result.get("success"): + print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) + return 1 + print(f"Opened {result.get('result')}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1])) diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score new file mode 100644 index 0000000..679a7ce --- /dev/null +++ b/suites/retail-cart/image/bench/score @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Turn the agent's finalize report into the task's rewards. + +`reward` is 1 when the agent reported success, and 0 for any other outcome or +when it never called finalize. `finalized` records whether it called +finalize, and `outcome_` which failure it reported, so bot blocks can be +told apart from other website errors, Kernel errors and the agent's own +failures. + +The browser's final URL and title are saved next to the report, so a reported +success can be checked against the page the agent ended on. +""" + +from __future__ import annotations + +import json +import os +import shutil +import urllib.request +from pathlib import Path + +REPORT = Path("/logs/agent/finalize.json") +TASK = Path("/opt/bench/task.json") +VERIFIER = Path("/logs/verifier") +FAILURES = ("failed", "blocked", "website_error", "kernel_error") + +FINAL_PAGE = "return { url: page.url(), title: await page.title() };" + + +def final_page() -> dict: + base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") + request = urllib.request.Request( + f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", + data=json.dumps({"code": FINAL_PAGE, "timeout_sec": 30}).encode(), + headers={ + "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + result = json.load(response) + except (OSError, ValueError, KeyError) as error: + return {"error": str(error)} + if not result.get("success"): + return {"error": result.get("error")} + return result.get("result") or {} + + +def main() -> int: + VERIFIER.mkdir(parents=True, exist_ok=True) + try: + report = json.loads(REPORT.read_text()) + except (OSError, json.JSONDecodeError): + report = None + if report is not None: + shutil.copy2(REPORT, VERIFIER / "finalize.json") + if TASK.exists(): + shutil.copy2(TASK, VERIFIER / "task.json") + (VERIFIER / "final-page.json").write_text(json.dumps(final_page(), indent=2)) + outcome = report.get("outcome") if report is not None else None + rewards = { + "reward": float(outcome == "success"), + "finalized": float(report is not None), + **{f"outcome_{name}": float(outcome == name) for name in FAILURES}, + } + (VERIFIER / "reward.json").write_text(json.dumps(rewards, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py new file mode 100644 index 0000000..7ddc17a --- /dev/null +++ b/suites/retail-cart/suite.py @@ -0,0 +1,117 @@ +"""Guest add-to-cart tasks on retail and industrial-supply sites with heavy bot protection. + +Every task starts on a site's home page, opened before the agent starts, and +asks the agent to search for one item, open its product page, add it to the +cart without logging in, and report the cart. The flow crosses the pages +where bot protection usually acts: the first request, search, the product +page and the cart. + +The agent ends each task by calling the `finalize` tool of the `bench` MCP +server with its evidence, its reasoning and an outcome. The verifier turns +that report into the reward and records the outcome, so a bot block is +counted apart from other failures. The browser's final URL and title are +saved for checking reported successes. +""" + +from __future__ import annotations + +import json +import shutil +import tomllib +from dataclasses import dataclass +from pathlib import Path + +import tomlkit + +HERE = Path(__file__).parent +IMAGE = HERE / "image" + +INSTRUCTION = """Add {item} to the {site} cart, then open the cart. Search for it from the site's search box, as a shopper would, open its product page, and add one to the cart. Report the cart line for it with its quantity and price, and the cart subtotal. + +## Rules + +- The browser is already open on {start_url}. Work on that site only. +- Do not log in, sign up, or enter personal or payment details. Continue as a guest. +- Never check out, pay, or place an order. Stop at the cart. +- If the site shows a bot check, CAPTCHA or press-and-hold challenge, try to get past it as a person would. If you cannot, or the site denies access, report the task as blocked. + +## Finishing + +When the task is done, or you are certain it cannot be done, call the `bench` MCP server's `finalize` tool exactly once, then stop. Your `finalize` report is the result of the task: put everything the task asks you to report in `evidence`. +""" + + +@dataclass(frozen=True) +class Task: + id: str + site: str + start_url: str + item: str + timeout_sec: int = 900 + + +def preflight() -> None: + """Nothing to check: tasks need only the Kernel browser Harbor creates.""" + + +def load_tasks() -> list[Task]: + return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] + + +def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: + tasks = load_tasks() + if task_ids: + known = {task.id: task for task in tasks} + missing = [task_id for task_id in task_ids if task_id not in known] + if missing: + raise SystemExit(f"Unknown tasks: {', '.join(missing)}") + tasks = [known[task_id] for task_id in task_ids] + out_dir.mkdir(parents=True, exist_ok=True) + return [_write(task, out_dir / task.id) for task in tasks] + + +def _write(task: Task, task_dir: Path) -> Path: + if task_dir.exists(): + shutil.rmtree(task_dir) + environment = task_dir / "environment" + shutil.copytree(IMAGE, environment) + (environment / "bench" / "task.json").write_text(json.dumps({"id": task.id, "site": task.site})) + step = task_dir / "steps" / "run" + (step / "tests").mkdir(parents=True) + (step / "workdir").mkdir() + + (step / "instruction.md").write_text( + INSTRUCTION.format(item=task.item, site=task.site, start_url=task.start_url) + ) + (step / "workdir" / "setup.sh").write_text( + f"#!/bin/bash\nset -euo pipefail\n/opt/bench/open-start-url {json.dumps(task.start_url)}\n" + ) + (step / "tests" / "test.sh").write_text("#!/bin/bash\nset -euo pipefail\n/opt/bench/score\n") + + config = tomlkit.document() + config["schema_version"] = "1.3" + config["source"] = "retail-cart" + config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} + config["metadata"] = {"site": task.site, "start_url": task.start_url} + config["environment"] = { + "build_timeout_sec": 1200.0, + "network_mode": "public", + "workdir": "/", + "env": {"KERNEL_API_KEY": "${KERNEL_API_KEY}", "KERNEL_BASE_URL": "${KERNEL_BASE_URL:-}"}, + } + servers = tomlkit.aot() + servers.append( + {"name": "bench", "transport": "stdio", "command": "/opt/bench/finalize-mcp", "args": []} + ) + config["environment"]["mcp_servers"] = servers + steps = tomlkit.aot() + steps.append( + { + "name": "run", + "agent": {"timeout_sec": float(task.timeout_sec)}, + "verifier": {"timeout_sec": 120.0}, + } + ) + config["steps"] = steps + (task_dir / "task.toml").write_text(tomlkit.dumps(config)) + return task_dir diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml new file mode 100644 index 0000000..a6e56c7 --- /dev/null +++ b/suites/retail-cart/tasks.toml @@ -0,0 +1,99 @@ +# One guest add-to-cart task per site. Each starts on the site's home page, +# where most bot protection first sees the browser, and walks home page -> +# search -> product page -> cart, the path a shopping agent takes. + +[[tasks]] +id = "amazon-aa-batteries" +site = "amazon.com" +start_url = "https://www.amazon.com/" +item = "the Amazon Basics 48-pack of AA alkaline batteries" + +[[tasks]] +id = "walmart-paper-towels" +site = "walmart.com" +start_url = "https://www.walmart.com/" +item = "a pack of Bounty Select-a-Size paper towels" + +[[tasks]] +id = "costco-microfiber-towels" +site = "costco.com" +start_url = "https://www.costco.com/" +item = "the Kirkland Signature Ultra Plush microfiber towels, 36-count" + +[[tasks]] +id = "lowes-stanley-kit" +site = "lowes.com" +start_url = "https://www.lowes.com/" +item = "the Stanley 27-pack 0.375-in metal kit" + +[[tasks]] +id = "staples-copy-paper" +site = "staples.com" +start_url = "https://www.staples.com/" +item = "one 500-sheet ream of 8.5 x 11 in multipurpose copy paper" + +[[tasks]] +id = "target-paper-towels" +site = "target.com" +start_url = "https://www.target.com/" +item = "a pack of Make-A-Size paper towels that can be shipped" + +[[tasks]] +id = "homedepot-desk" +site = "homedepot.com" +start_url = "https://www.homedepot.com/" +item = "a computer desk under $150 that can be shipped" + +[[tasks]] +id = "radwell-fuse" +site = "radwell.com" +start_url = "https://www.radwell.com/" +item = "a 20 A, 32 V 3AG fuse" + +[[tasks]] +id = "ebay-usb-c-cable" +site = "ebay.com" +start_url = "https://www.ebay.com/" +item = "a USB-C charging cable from a Buy It Now listing under $15" + +[[tasks]] +id = "harborfreight-utility-knife" +site = "harborfreight.com" +start_url = "https://www.harborfreight.com/" +item = "a folding lock-back utility knife" + +[[tasks]] +id = "dickssportinggoods-water-bottle" +site = "dickssportinggoods.com" +start_url = "https://www.dickssportinggoods.com/" +item = "a HydroJug 32 oz sport bottle that can be shipped" + +[[tasks]] +id = "mouser-usb-c-receptacle" +site = "mouser.com" +start_url = "https://www.mouser.com/" +item = "one GCT USB4105-GF-A USB-C receptacle" + +[[tasks]] +id = "newark-aa-batteries" +site = "newark.com" +start_url = "https://www.newark.com/" +item = "a pack of AA alkaline batteries" + +[[tasks]] +id = "zoro-nitrile-gloves" +site = "zoro.com" +start_url = "https://www.zoro.com/" +item = "a box of disposable nitrile gloves, size large" + +[[tasks]] +id = "guitarcenter-picks" +site = "guitarcenter.com" +start_url = "https://www.guitarcenter.com/" +item = "a 12-pack of Fender 351 celluloid medium guitar picks" + +[[tasks]] +id = "wbmason-sticky-notes" +site = "wbmason.com" +start_url = "https://www.wbmason.com/" +item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py new file mode 100644 index 0000000..ffdd61c --- /dev/null +++ b/tests/test_retail_cart.py @@ -0,0 +1,98 @@ +import json +import os +import subprocess +import tomllib + +import pytest + +from harness.definitions import ROOT, load_suite + +suite = load_suite("retail-cart") +BENCH = ROOT / "suites/retail-cart/image/bench" + + +def test_every_task_has_a_unique_id_and_an_https_start_url(): + tasks = suite.load_tasks() + assert len({task.id for task in tasks}) == len(tasks) + assert all(task.start_url.startswith("https://") for task in tasks) + + +def test_generate_writes_a_harbor_task(tmp_path): + [task_dir] = suite.generate(["costco-microfiber-towels"], tmp_path) + + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert config["metadata"]["site"] == "costco.com" + assert [server["name"] for server in config["environment"]["mcp_servers"]] == ["bench"] + instruction = (task_dir / "steps/run/instruction.md").read_text() + assert "Kirkland Signature" in instruction + assert "https://www.costco.com/" in instruction + setup = (task_dir / "steps/run/workdir/setup.sh").read_text() + assert 'open-start-url "https://www.costco.com/"' in setup + assert (task_dir / "environment/bench/finalize-mcp").is_file() + + +def test_unknown_task_is_refused(tmp_path): + with pytest.raises(SystemExit, match="nope"): + suite.generate(["nope"], tmp_path) + + +def _call(script, messages, tmp_path): + source = ( + (BENCH / script) + .read_text() + .replace('Path("/logs/agent/finalize.json")', f"Path({str(tmp_path / 'finalize.json')!r})") + ) + output = subprocess.run( + ["python3", "-c", source], + input="\n".join(json.dumps(message) for message in messages) + "\n", + capture_output=True, + text=True, + check=True, + ).stdout + return [json.loads(line) for line in output.splitlines()] + + +def test_finalize_records_the_first_report_only(tmp_path): + report = {"evidence": "Access Denied", "reasoning": "403 on home page", "outcome": "blocked"} + call = {"jsonrpc": "2.0", "method": "tools/call"} + responses = _call( + "finalize-mcp", + [ + {**call, "id": 1, "params": {"name": "finalize", "arguments": report}}, + {**call, "id": 2, "params": {"name": "finalize", "arguments": report}}, + ], + tmp_path, + ) + assert responses[0]["result"]["isError"] is False + assert responses[1]["result"]["isError"] is True + assert json.loads((tmp_path / "finalize.json").read_text())["outcome"] == "blocked" + + +def test_score_counts_a_block_apart_from_other_failures(tmp_path): + verifier = tmp_path / "verifier" + (tmp_path / "finalize.json").write_text(json.dumps({"outcome": "blocked"})) + source = ( + (BENCH / "score") + .read_text() + .replace('Path("/logs/agent/finalize.json")', f"Path({str(tmp_path / 'finalize.json')!r})") + .replace('Path("/logs/verifier")', f"Path({str(verifier)!r})") + .replace('Path("/opt/bench/task.json")', f"Path({str(tmp_path / 'task.json')!r})") + ) + env = { + **os.environ, + "KERNEL_SESSION_ID": "s1", + "KERNEL_API_KEY": "k", + "KERNEL_BASE_URL": "http://127.0.0.1:9", + } + subprocess.run(["python3", "-c", source], check=True, env=env) + + rewards = json.loads((verifier / "reward.json").read_text()) + assert rewards == { + "reward": 0.0, + "finalized": 1.0, + "outcome_failed": 0.0, + "outcome_blocked": 1.0, + "outcome_website_error": 0.0, + "outcome_kernel_error": 0.0, + } + assert "error" in json.loads((verifier / "final-page.json").read_text()) From d66817f99320e0bef26b83cfbd2cb223e3b00bfa Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:55:27 +0000 Subject: [PATCH 2/2] Ask for the outcome rather than the route, verify the cart, and add a pinned per-site arm --- README.md | 9 ++- arms/codex-repl-pinned.toml | 16 ++++ benchmarks/retail-cart.toml | 4 +- configurators/pinned/configure | 50 ++++++++++++ configurators/pinned/sites.toml | 109 +++++++++++++++++++++++++++ suites/retail-cart/image/bench/score | 92 ++++++++++++++++++++-- suites/retail-cart/suite.py | 37 ++++++--- suites/retail-cart/tasks.toml | 41 +++++++++- tests/test_pinned.py | 31 ++++++++ tests/test_retail_cart.py | 24 ++++++ 10 files changed, 386 insertions(+), 27 deletions(-) create mode 100644 arms/codex-repl-pinned.toml create mode 100755 configurators/pinned/configure create mode 100644 configurators/pinned/sites.toml create mode 100644 tests/test_pinned.py diff --git a/README.md b/README.md index 0132304..17942f6 100644 --- a/README.md +++ b/README.md @@ -121,14 +121,19 @@ ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whet ### `retail-cart` -Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to search for one item, open its product page, add it to the cart without logging in, and report the cart. +Guest add-to-cart tasks on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to put one item in the site's cart as a guest, by whatever route works, and show the cart. | Arm | Agent | Browser | |---|---|---| | `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | `stealth: true` and no proxy, so each browser gets its own egress IP from the stealth proxy pool | | `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. | +| `codex-repl-pinned` | Same, plus Kernel MCP's `computer_action` | A fixed per-site table (`configurators/pinned/sites.toml`): stealth, GPU, a new US ISP or residential proxy per trial, a macOS persona or VM setup on some sites, and per-site guidance | -The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: +The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. + +The verifier then checks the cart itself. It reads the page the agent ended on if that is the cart, otherwise opens the task's `cart_url` or fetches its `cart_api`, and sets `cart_verified` when the item is listed. When it could not see the cart, because loading it failed or showed a bot check, it sets `cart_unverifiable` instead. + +Run several attempts per task to get a pass rate per site: ```bash uv run bench run retail-cart --attempts 10 diff --git a/arms/codex-repl-pinned.toml b/arms/codex-repl-pinned.toml new file mode 100644 index 0000000..b0fca7e --- /dev/null +++ b/arms/codex-repl-pinned.toml @@ -0,0 +1,16 @@ +# Same as codex-repl, but each site's browser, proxy and persona come from the +# fixed table in configurators/pinned/sites.toml, with a new proxy per trial. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl", "computer"] + +[browser] +config = { stealth = true } +configurator = "pinned" diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml index 4d486a6..c980f3d 100644 --- a/benchmarks/retail-cart.toml +++ b/benchmarks/retail-cart.toml @@ -1,5 +1,5 @@ # Guest add-to-cart flows on bot-protected retail and industrial-supply sites, # driven through the browser REPL, with a stealth browser and with the config -# registry's per-site settings. +# registry's per-site settings, and with a fixed per-site table. suite = "retail-cart" -arms = ["codex-repl", "codex-repl-site-config"] +arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned"] diff --git a/configurators/pinned/configure b/configurators/pinned/configure new file mode 100755 index 0000000..b5ec6f8 --- /dev/null +++ b/configurators/pinned/configure @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Choose browser settings from the fixed per-site table in sites.toml. + +The task's site comes from `metadata.site` in its task.toml. Proxies are US +ISP or US residential, created per trial. A `persona` of `macos` restarts +Chromium with the macOS persona's launcher variables. `vm_setup` commands run as root +in the browser's VM before the first page load. + +stdin: {"task": {"config": str, "instruction": str}, "args": {}} +stdout: {"browser": {...}, "steering": str | null, "provenance": {...}} +""" + +from __future__ import annotations + +import json +import sys +import tomllib +from pathlib import Path + +TABLE = tomllib.loads((Path(__file__).parent / "sites.toml").read_text()) + + +def main() -> int: + task = json.load(sys.stdin)["task"] + site = tomllib.loads(task["config"]).get("metadata", {}).get("site") + entry = TABLE["sites"].get(site) + if entry is None: + raise SystemExit(f"No pinned configuration for site {site!r}") + browser = dict(entry["browser"]) + browser["proxy_create"] = {"type": entry["proxy"], "config": {"country": "US"}} + persona = entry.get("persona") + if persona == "macos": + browser["chromium_env"] = TABLE["macos_persona"] + elif persona is not None: + raise SystemExit(f"Unknown persona {persona!r} for {site}") + if entry.get("vm_setup"): + browser["vm_setup"] = entry["vm_setup"] + json.dump( + { + "browser": browser, + "steering": entry.get("steering"), + "provenance": {"site": site, "proxy": entry["proxy"], "persona": persona}, + }, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/configurators/pinned/sites.toml b/configurators/pinned/sites.toml new file mode 100644 index 0000000..f2b2469 --- /dev/null +++ b/configurators/pinned/sites.toml @@ -0,0 +1,109 @@ +# Browser settings per site, measured by hand on 2026-10-29 to 2026-11-01. +# Each entry is merged over the arm's config. `proxy` gives every trial its own +# proxy of that type; `persona = "macos"` restarts Chromium with the macOS +# persona's launcher variables; `vm_setup` commands run as root in the +# browser's VM before the first page load; `steering` is added to the +# instruction. + +[macos_persona] +KERNEL_MAC_PERSONA = "1" + +[sites."amazon.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."costco.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."dickssportinggoods.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."ebay.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" + +[sites."guitarcenter.com"] +# Measured passing only with Web Bot Auth signatures, which this project does not have. +browser = { stealth = true } +proxy = "isp" +steering = """ +guitarcenter.com denies its home page (/), /search and /cart to automated browsers with an "Access Denied" page, but search results at /search/ with a trailing slash, product pages and /cart/index load. Search by opening https://www.guitarcenter.com/search/?Ntt=, open the product, and click Add to Cart. Show the cart at https://www.guitarcenter.com/cart/index; if it does not list the item, report the add-to-cart confirmation the product page showed. +""" + +[sites."harborfreight.com"] +browser = { stealth = true, gpu = true } +proxy = "isp" +persona = "macos" + +[sites."homedepot.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."lowes.com"] +browser = { stealth = false } +proxy = "isp" + +[sites."mouser.com"] +browser = { stealth = true } +proxy = "residential" +steering = """ +mouser.com denies its search results, category and product pages (/c/..., /ProductDetail/...) to automated browsers, so do not press Enter in the search box. Find the part number instead: type the item into the search box and read the suggestions without submitting, or search the web (for example DuckDuckGo) for "mouser " and take the part number from the results. Then open https://www.mouser.com/en/Cart/, enter the manufacturer or Mouser part number and a quantity in its add-by-part-number row, and click Add. + +Pages may show a bot-check interstitial that clears by itself within about 12 seconds. Wait up to 30 seconds before treating a page as blocked. +""" + +[sites."newark.com"] +browser = { stealth = true } +proxy = "isp" +steering = """ +newark.com denies its search results, category and product pages (/search, /c/..., /dp/...) to automated browsers, so pressing Enter in the search box does nothing. The site's older store pages still work; use them: + +1. Type the item into the search box and read the suggestions without submitting, or open https://www.newark.com/webapp/wcs/stores/servlet/AjaxSearchLookAhead?searchTerm=&storeId=10194&catalogId=15003&langId=-1. Suggestions link to /dp/; note the order code of a matching product. +2. Open https://www.newark.com/webapp/wcs/stores/servlet/ProductDisplay?storeId=10194&catalogId=15003&langId=-1&partNumber=. +3. Clicking its Add to Cart link does nothing; navigate to the link's href instead. +4. Show the cart at https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1. The /cart page stays empty. +""" + +[sites."radwell.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."staples.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."target.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."walmart.com"] +browser = { stealth = true } +proxy = "isp" + +[sites."wbmason.com"] +# The captcha-solver extension takes over the site's Cloudflare Turnstile check +# and never completes it, so its Turnstile interceptor is emptied before the +# first page load. +browser = { stealth = true } +proxy = "residential" +vm_setup = ["printf '/* disabled */\\n' > /home/kernel/extensions/kernel/capmonster/interceptors/turnstileInterceptor.js"] +steering = """ +wbmason.com shows a "Verify Access to Continue" page with a Cloudflare Turnstile check before the site. Do not reload or navigate while it is showing: that turns it into a hard block. Wait up to 30 seconds for it to pass by itself; if a checkbox appears, click it once with computer_action. + +On this site use computer_action for clicks and typing: scripted clicks and typing in the search box do not take effect. +""" + +[sites."zoro.com"] +# A fresh residential IP per trial and one reload passed 5/5 by hand; GPU and +# the WebGL persona made no difference there and got a slider every time in +# the benchmark. +browser = { stealth = true } +proxy = "residential" +steering = """ +On zoro.com the first page load is often refused with a 403 or a bot-check page. That refusal sets a cookie, and loading the same URL once more usually shows the real site, so if the first load is blocked, load it again once. If the page says "Access is temporarily restricted", reloading will not help: report the task as blocked. +""" + diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 679a7ce..175d4a8 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Turn the agent's finalize report into the task's rewards. +"""Turn the agent's finalize report into the task's rewards, and check the cart. `reward` is 1 when the agent reported success, and 0 for any other outcome or when it never called finalize. `finalized` records whether it called @@ -7,15 +7,22 @@ finalize, and `outcome_` which failure it reported, so bot blocks can be told apart from other website errors, Kernel errors and the agent's own failures. -The browser's final URL and title are saved next to the report, so a reported -success can be checked against the page the agent ended on. +The browser's final URL and title are saved, then the verifier checks the +cart itself: it reads the page the agent ended on when that is already the +cart, otherwise opens the task's `cart_url`, or fetches its `cart_api` from +the page, and sets `cart_verified` when the text matches `cart_match`. When +the check could not see the cart, because loading it failed or showed a bot +check, it sets `cart_unverifiable` instead, so a failed check is not counted +as an empty cart. cart-check.json keeps what it saw. """ from __future__ import annotations import json import os +import re import shutil +import urllib.parse import urllib.request from pathlib import Path @@ -26,12 +33,46 @@ FAILURES = ("failed", "blocked", "website_error", "kernel_error") FINAL_PAGE = "return { url: page.url(), title: await page.title() };" +# A bot check or access-denied page in place of the cart. +CHALLENGE = re.compile( + r"captcha|/checkpoint/|validateCaptcha|verify (yourself|access|you are)|access denied" + r"|access is temporarily restricted|press (and|&) hold|are you a robot", + re.IGNORECASE, +) -def final_page() -> dict: +CURRENT_PAGE = """ +return { + url: page.url(), + title: await page.title(), + text: await page.evaluate(() => (document.body ? document.body.innerText : "")), +}; +""" + +CART_PAGE = """ +const response = await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); +await page.waitForLoadState("networkidle", { timeout: 15000 }).catch(() => {}); +await page.waitForTimeout(3000); +return { + url: page.url(), + status: response ? response.status() : null, + title: await page.title(), + text: await page.evaluate(() => (document.body ? document.body.innerText : "")), +}; +""" + +CART_API = """ +return await page.evaluate(async (url) => { + const response = await fetch(url, { headers: { Accept: "application/json" }, credentials: "include" }); + return { url, status: response.status, text: await response.text() }; +}, %s); +""" + + +def execute(code: str, timeout_sec: int) -> dict: base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", - data=json.dumps({"code": FINAL_PAGE, "timeout_sec": 30}).encode(), + data=json.dumps({"code": code, "timeout_sec": timeout_sec}).encode(), headers={ "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", "Content-Type": "application/json", @@ -39,7 +80,7 @@ def final_page() -> dict: method="POST", ) try: - with urllib.request.urlopen(request, timeout=60) as response: + with urllib.request.urlopen(request, timeout=timeout_sec + 30) as response: result = json.load(response) except (OSError, ValueError, KeyError) as error: return {"error": str(error)} @@ -48,6 +89,35 @@ def final_page() -> dict: return result.get("result") or {} +def same_page(url: str, other: str) -> bool: + a, b = urllib.parse.urlsplit(url), urllib.parse.urlsplit(other) + return (a.netloc, a.path.rstrip("/").lower()) == (b.netloc, b.path.rstrip("/").lower()) + + +def check_cart(task: dict, final_page: dict) -> dict: + if task.get("cart_api"): + seen = execute(CART_API % json.dumps(task["cart_api"]), 60) + elif task.get("cart_url"): + if same_page(final_page.get("url") or "", task["cart_url"]): + seen = execute(CURRENT_PAGE, 60) + else: + seen = execute(CART_PAGE % json.dumps(task["cart_url"]), 90) + else: + return {"verified": False, "unverifiable": True, "error": "task has no cart_url or cart_api"} + text = seen.pop("text", None) or "" + match = re.search(task["cart_match"], text, re.IGNORECASE) if task.get("cart_match") else None + seen["verified"] = match is not None + challenge = CHALLENGE.search(f"{seen.get('url', '')} {seen.get('title', '')} {text[:3000]}") + seen["unverifiable"] = match is None and ("error" in seen or challenge is not None) + if challenge and match is None: + seen["challenge"] = challenge.group(0) + if match: + seen["excerpt"] = text[max(0, match.start() - 200) : match.end() + 200] + else: + seen["excerpt"] = text[:2000] + return seen + + def main() -> int: VERIFIER.mkdir(parents=True, exist_ok=True) try: @@ -56,13 +126,19 @@ def main() -> int: report = None if report is not None: shutil.copy2(REPORT, VERIFIER / "finalize.json") - if TASK.exists(): + task = json.loads(TASK.read_text()) if TASK.exists() else {} + if task: shutil.copy2(TASK, VERIFIER / "task.json") - (VERIFIER / "final-page.json").write_text(json.dumps(final_page(), indent=2)) + final_page = execute(FINAL_PAGE, 30) + (VERIFIER / "final-page.json").write_text(json.dumps(final_page, indent=2)) + cart = check_cart(task, final_page) + (VERIFIER / "cart-check.json").write_text(json.dumps(cart, indent=2)) outcome = report.get("outcome") if report is not None else None rewards = { "reward": float(outcome == "success"), "finalized": float(report is not None), + "cart_verified": float(cart.get("verified") is True), + "cart_unverifiable": float(cart.get("unverifiable") is True), **{f"outcome_{name}": float(outcome == name) for name in FAILURES}, } (VERIFIER / "reward.json").write_text(json.dumps(rewards, indent=2)) diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 7ddc17a..879c005 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -1,16 +1,18 @@ """Guest add-to-cart tasks on retail and industrial-supply sites with heavy bot protection. -Every task starts on a site's home page, opened before the agent starts, and -asks the agent to search for one item, open its product page, add it to the -cart without logging in, and report the cart. The flow crosses the pages -where bot protection usually acts: the first request, search, the product -page and the cart. +Every task starts on a site's home page, opened before the agent starts, +and asks the agent to put one item in the cart without logging in and report +the cart. The task fixes the outcome, not the route: on some sites the usual +search and product pages are denied to automated browsers, and per-site +guidance can send the agent another way. The agent ends each task by calling the `finalize` tool of the `bench` MCP server with its evidence, its reasoning and an outcome. The verifier turns that report into the reward and records the outcome, so a bot block is -counted apart from other failures. The browser's final URL and title are -saved for checking reported successes. +counted apart from other failures. The verifier then opens the site's cart +itself (`cart_url`, or `cart_api` fetched from the page) and records +`cart_verified` when it lists the item (`cart_match`, a case-insensitive +regular expression). """ from __future__ import annotations @@ -26,12 +28,12 @@ HERE = Path(__file__).parent IMAGE = HERE / "image" -INSTRUCTION = """Add {item} to the {site} cart, then open the cart. Search for it from the site's search box, as a shopper would, open its product page, and add one to the cart. Report the cart line for it with its quantity and price, and the cart subtotal. +INSTRUCTION = """Put one {item} in the {site} shopping cart as a guest, then show the cart. Report the cart line for it with its quantity and price, and the cart subtotal. ## Rules -- The browser is already open on {start_url}. Work on that site only. -- Do not log in, sign up, or enter personal or payment details. Continue as a guest. +- The browser is already open on {start_url}. Reach the item however works: the site's search, its menus and links, or a web search to identify the exact product or part number. The cart must be on {site}. +- Do not log in, sign up, or enter personal or payment details. - Never check out, pay, or place an order. Stop at the cart. - If the site shows a bot check, CAPTCHA or press-and-hold challenge, try to get past it as a person would. If you cannot, or the site denies access, report the task as blocked. @@ -47,6 +49,9 @@ class Task: site: str start_url: str item: str + cart_match: str + cart_url: str | None = None + cart_api: str | None = None timeout_sec: int = 900 @@ -75,7 +80,17 @@ def _write(task: Task, task_dir: Path) -> Path: shutil.rmtree(task_dir) environment = task_dir / "environment" shutil.copytree(IMAGE, environment) - (environment / "bench" / "task.json").write_text(json.dumps({"id": task.id, "site": task.site})) + (environment / "bench" / "task.json").write_text( + json.dumps( + { + "id": task.id, + "site": task.site, + "cart_url": task.cart_url, + "cart_api": task.cart_api, + "cart_match": task.cart_match, + } + ) + ) step = task_dir / "steps" / "run" (step / "tests").mkdir(parents=True) (step / "workdir").mkdir() diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml index a6e56c7..233d945 100644 --- a/suites/retail-cart/tasks.toml +++ b/suites/retail-cart/tasks.toml @@ -1,99 +1,132 @@ # One guest add-to-cart task per site. Each starts on the site's home page, -# where most bot protection first sees the browser, and walks home page -> -# search -> product page -> cart, the path a shopping agent takes. +# where most bot protection first sees the browser. The verifier opens +# `cart_url` (or fetches `cart_api` from the page) and checks it lists the +# item with `cart_match`, a case-insensitive regular expression. [[tasks]] id = "amazon-aa-batteries" site = "amazon.com" start_url = "https://www.amazon.com/" item = "the Amazon Basics 48-pack of AA alkaline batteries" +cart_url = "https://www.amazon.com/gp/cart/view.html" +cart_match = 'Amazon Basics.{0,100}\bAA\b' [[tasks]] id = "walmart-paper-towels" site = "walmart.com" start_url = "https://www.walmart.com/" item = "a pack of Bounty Select-a-Size paper towels" +cart_url = "https://www.walmart.com/cart" +cart_match = 'Bounty' [[tasks]] id = "costco-microfiber-towels" site = "costco.com" start_url = "https://www.costco.com/" item = "the Kirkland Signature Ultra Plush microfiber towels, 36-count" +cart_url = "https://www.costco.com/CheckoutCartDisplayView" +cart_match = 'Ultra Plush' [[tasks]] -id = "lowes-stanley-kit" +id = "lowes-metal-kit" site = "lowes.com" start_url = "https://www.lowes.com/" -item = "the Stanley 27-pack 0.375-in metal kit" +item = "the 27-pack 0.375-in metal kit (sold under the Bostitch or Stanley brand)" +cart_url = "https://www.lowes.com/cart" +cart_match = '27.{0,4}Pack.{0,30}Metal Kit' [[tasks]] id = "staples-copy-paper" site = "staples.com" start_url = "https://www.staples.com/" item = "one 500-sheet ream of 8.5 x 11 in multipurpose copy paper" +cart_url = "https://www.staples.com/cc/sparq/cart" +cart_match = '(copy|multipurpose).{0,60}paper' [[tasks]] id = "target-paper-towels" site = "target.com" start_url = "https://www.target.com/" item = "a pack of Make-A-Size paper towels that can be shipped" +cart_url = "https://www.target.com/cart" +cart_match = 'paper towel' [[tasks]] id = "homedepot-desk" site = "homedepot.com" start_url = "https://www.homedepot.com/" item = "a computer desk under $150 that can be shipped" +cart_url = "https://www.homedepot.com/cart" +cart_match = '\bdesk' [[tasks]] id = "radwell-fuse" site = "radwell.com" start_url = "https://www.radwell.com/" item = "a 20 A, 32 V 3AG fuse" +cart_url = "https://www.radwell.com/Cart" +cart_match = 'fuse' [[tasks]] id = "ebay-usb-c-cable" site = "ebay.com" start_url = "https://www.ebay.com/" item = "a USB-C charging cable from a Buy It Now listing under $15" +cart_url = "https://cart.ebay.com/" +cart_match = 'usb.?c' [[tasks]] id = "harborfreight-utility-knife" site = "harborfreight.com" start_url = "https://www.harborfreight.com/" item = "a folding lock-back utility knife" +cart_url = "https://www.harborfreight.com/checkout/cart" +cart_match = 'utility knife|lock.?back' [[tasks]] id = "dickssportinggoods-water-bottle" site = "dickssportinggoods.com" start_url = "https://www.dickssportinggoods.com/" item = "a HydroJug 32 oz sport bottle that can be shipped" +cart_url = "https://www.dickssportinggoods.com/OrderItemDisplay" +cart_match = 'HydroJug' [[tasks]] id = "mouser-usb-c-receptacle" site = "mouser.com" start_url = "https://www.mouser.com/" item = "one GCT USB4105-GF-A USB-C receptacle" +cart_url = "https://www.mouser.com/en/Cart/" +cart_match = 'USB4105' [[tasks]] id = "newark-aa-batteries" site = "newark.com" start_url = "https://www.newark.com/" item = "a pack of AA alkaline batteries" +cart_url = "https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1" +cart_match = 'alkaline' [[tasks]] id = "zoro-nitrile-gloves" site = "zoro.com" start_url = "https://www.zoro.com/" item = "a box of disposable nitrile gloves, size large" +cart_url = "https://www.zoro.com/cart" +cart_match = 'nitrile' [[tasks]] id = "guitarcenter-picks" site = "guitarcenter.com" start_url = "https://www.guitarcenter.com/" item = "a 12-pack of Fender 351 celluloid medium guitar picks" +cart_api = "https://www.guitarcenter.com/rest/model/atg/commerce/ShoppingCartActor/cartRedesignNew?saveForLaterPage=1&SPA=true" +cart_match = '\b351\b' [[tasks]] id = "wbmason-sticky-notes" site = "wbmason.com" start_url = "https://www.wbmason.com/" item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" +cart_url = "https://www.wbmason.com/shopping-cart" +cart_match = 'Post-it|sticky note' diff --git a/tests/test_pinned.py b/tests/test_pinned.py new file mode 100644 index 0000000..157ee6f --- /dev/null +++ b/tests/test_pinned.py @@ -0,0 +1,31 @@ +import json +import subprocess + +from harness.definitions import ROOT + +CONFIGURE = ROOT / "configurators/pinned/configure" + + +def _configure(site): + task = {"task": {"config": f'[metadata]\nsite = "{site}"\n', "instruction": ""}, "args": {}} + result = subprocess.run( + [CONFIGURE], input=json.dumps(task), capture_output=True, text=True, check=True + ) + return json.loads(result.stdout) + + +def test_every_retail_cart_site_has_a_pinned_configuration(): + from harness.definitions import load_suite + + for task in load_suite("retail-cart").load_tasks(): + output = _configure(task.site) + assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + + +def test_personas_become_chromium_env(): + assert _configure("amazon.com")["browser"]["chromium_env"] == {"KERNEL_MAC_PERSONA": "1"} + zoro = _configure("zoro.com") + assert "chromium_env" not in zoro["browser"] + assert zoro["browser"]["proxy_create"]["type"] == "residential" + assert "load it again once" in zoro["steering"] + assert _configure("wbmason.com")["browser"]["vm_setup"] diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index ffdd61c..962c9a5 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -25,6 +25,7 @@ def test_generate_writes_a_harbor_task(tmp_path): assert [server["name"] for server in config["environment"]["mcp_servers"]] == ["bench"] instruction = (task_dir / "steps/run/instruction.md").read_text() assert "Kirkland Signature" in instruction + assert "Reach the item however works" in instruction assert "https://www.costco.com/" in instruction setup = (task_dir / "steps/run/workdir/setup.sh").read_text() assert 'open-start-url "https://www.costco.com/"' in setup @@ -71,6 +72,9 @@ def test_finalize_records_the_first_report_only(tmp_path): def test_score_counts_a_block_apart_from_other_failures(tmp_path): verifier = tmp_path / "verifier" (tmp_path / "finalize.json").write_text(json.dumps({"outcome": "blocked"})) + (tmp_path / "task.json").write_text( + json.dumps({"id": "t", "cart_url": "https://example.com/cart", "cart_match": "x"}) + ) source = ( (BENCH / "score") .read_text() @@ -90,9 +94,29 @@ def test_score_counts_a_block_apart_from_other_failures(tmp_path): assert rewards == { "reward": 0.0, "finalized": 1.0, + "cart_verified": 0.0, + "cart_unverifiable": 1.0, "outcome_failed": 0.0, "outcome_blocked": 1.0, "outcome_website_error": 0.0, "outcome_kernel_error": 0.0, } assert "error" in json.loads((verifier / "final-page.json").read_text()) + check = json.loads((verifier / "cart-check.json").read_text()) + assert check["verified"] is False and check["unverifiable"] is True + + +def test_cart_check_tells_a_bot_check_from_an_empty_cart(monkeypatch): + source = (BENCH / "score").read_text().replace('if __name__ == "__main__":', "if False:") + score = {} + exec(compile(source, "score", "exec"), score) + task = {"cart_url": "https://shop.test/cart", "cart_match": "gloves"} + + def page(text, url="https://shop.test/cart"): + score["execute"] = lambda code, timeout: {"url": url, "title": "", "text": text} + return score["check_cart"](task, {"url": url}) + + assert page("Nitrile gloves, qty 1")["verified"] is True + assert page("Your cart is empty")["unverifiable"] is False + blocked = page("Please verify yourself to continue", "https://shop.test/splashui/captcha") + assert blocked["verified"] is False and blocked["unverifiable"] is True