From c3a5d5dbe1c32ea0b2bfc9366b0408a7995756ca Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:37:07 +0000 Subject: [PATCH 1/4] bench: give agents pi's read tool, running them as dedicated OS users --- bench/README.md | 21 +++++++- bench/allowlist.txt | 1 + bench/lib/agent.mjs | 4 +- bench/lib/users.mjs | 102 +++++++++++++++++++++++++++++++++++++ bench/run.mjs | 44 +++++++++++++--- bench/setup-agent-users.sh | 32 ++++++++++++ bench/test/run.test.mjs | 15 ++++++ bench/test/stubs/getent | 5 ++ bench/test/stubs/sudo | 20 ++++++++ bench/test/users.test.mjs | 74 +++++++++++++++++++++++++++ 10 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 bench/lib/users.mjs create mode 100755 bench/setup-agent-users.sh create mode 100755 bench/test/stubs/getent create mode 100755 bench/test/stubs/sudo create mode 100644 bench/test/users.test.mjs diff --git a/bench/README.md b/bench/README.md index 586c0c39..b6576337 100644 --- a/bench/README.md +++ b/bench/README.md @@ -28,9 +28,10 @@ bench/ tasks//.json one intent per file; domain.json holds per-domain settings schemas/ JSON Schemas for answers prompts/system.md the system prompt both arms get - allowlist.txt the kernel MCP tools the agent gets, and no others + allowlist.txt the tools the agent gets, and no others: the kernel MCP tools and pi's read mcp.json the kernel MCP server; reads KERNEL_API_KEY .env.example keys run.mjs loads from bench/.env + setup-agent-users.sh creates the OS users agents run as run.mjs grade.mjs report.mjs lib/ test/ results// gitignored: run.json, rows.jsonl, transcripts ``` @@ -41,6 +42,22 @@ Needs the `kernel` and `pi` CLIs, Node 22.13+, git, `KERNEL_API_KEY` and the mod key (such as `OPENAI_API_KEY`), from the environment or `bench/.env`. Each agent gets its own `PI_CODING_AGENT_DIR`, so stored pi logins and MCP servers are not used. +pi cuts an MCP result over 20KB, such as a long WebMCP tool list, and saves the full text to a +file, so the agent gets pi's `read` tool to open it. Agents on Kernel browsers therefore run as +dedicated OS users, `xwebmcp-bench-`, one per slot, that cannot read this checkout or your home: +each agent starts in an empty directory under its user's home with its own `TMPDIR`, and gets only +`KERNEL_API_KEY` and the variables of its model's provider (`ANTHROPIC_*` for `anthropic/...`). +Create the users once per machine, as many as `--slots`; this is Linux only, and pi must be +installed outside your home: + +```sh +sudo bench/setup-agent-users.sh "$(id -un)" 4 +``` + +The script lets you run commands as those users through `sudo` without logging their environment, +and removes other users' access to this checkout and your home. The smoke gate stops the run if a +slot has no agent user, or if one can read either. + Run it as a Hypeship service under a hold that covers the whole run; holds cannot be renewed. This pilot is 105 trials and can take about 8 hours if every trial reaches its cap: @@ -108,6 +125,8 @@ Before any trial, stopping on the first failure: 1. the hold covers the remaining jobs if every trial reaches its cap after 180s of setup (the hold is checked again before each job, and no job starts that it cannot outlast); 2. `KERNEL_API_KEY` is set (Kernel browsers), and on resume pi is the same version; + with Kernel browsers, each slot has an agent user that can read neither this checkout nor your + home; 3. a zero-tool pi run answers and reports a cost above zero (pi prices an unknown model at 0, which would disable the spend cap); 4. a treatment setup on the first pending intent works end to end; diff --git a/bench/allowlist.txt b/bench/allowlist.txt index f054315a..e212e45b 100644 --- a/bench/allowlist.txt +++ b/bench/allowlist.txt @@ -2,3 +2,4 @@ mcp__kernel__browser_repl mcp__kernel__computer_action mcp__kernel__execute_playwright_code mcp__kernel__webmcp +read diff --git a/bench/lib/agent.mjs b/bench/lib/agent.mjs index 51264646..eb705e52 100644 --- a/bench/lib/agent.mjs +++ b/bench/lib/agent.mjs @@ -32,9 +32,9 @@ export function parseRecord(line) { // The clock runs from launch to agent_settled. pi has no turn or spend cap, so the runner // enforces both; a missing cost stops the agent, since the spend cap needs it. -export function runAgent({ args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, transcriptPath, stderrPath }) { +export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, transcriptPath, stderrPath }) { return new Promise((resolve, reject) => { - const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); runningAgents.add(child); const start = performance.now(); const elapsed = () => Math.round(performance.now() - start); diff --git a/bench/lib/users.mjs b/bench/lib/users.mjs new file mode 100644 index 00000000..842db52f --- /dev/null +++ b/bench/lib/users.mjs @@ -0,0 +1,102 @@ +// Agents with a built-in tool such as `read` run as dedicated OS users, xwebmcp-bench-, so +// they cannot read the bench's files: intents and their rules, results, other trials or keys. +// bench/setup-agent-users.sh creates the users and removes other users' access to the checkout +// and the bench user's home; the smoke gate checks that before any agent starts. + +import { execFileSync } from "node:child_process"; + +export const AGENT_USER = /^xwebmcp-bench-(\d+)$/; + +// Each user's agents get a fresh directory under its home, including TMPDIR, where pi saves the +// full text of a long MCP result, so nothing is left from the previous trial. +const LAUNCH = `set -eu +home=$1; shift +umask 077 +rm -rf "$home/trial" +mkdir -p "$home/trial/agent" "$home/trial/tmp" +if [ -n "\${BENCH_MCP_JSON-}" ]; then printf '%s\\n' "$BENCH_MCP_JSON" > "$home/trial/agent/mcp.json"; fi +export HOME="$home" PATH="$BENCH_PATH" PI_CODING_AGENT_DIR="$home/trial/agent" TMPDIR="$home/trial/tmp" +unset BENCH_MCP_JSON BENCH_PATH +cd "$home/trial" +exec pi "$@"`; + +export function findAgentUsers(passwd = execFileSync("getent", ["passwd"], { encoding: "utf8" })) { + return passwd.split("\n").map(line => line.split(":")).filter(fields => AGENT_USER.test(fields[0])) + .map(fields => ({ name: fields[0], home: fields[5] })) + .sort((a, b) => Number(AGENT_USER.exec(a.name)[1]) - Number(AGENT_USER.exec(b.name)[1])); +} + +// The model's provider is the part of --model before the slash; its variables, such as +// ANTHROPIC_API_KEY for anthropic/..., are the only keys the agent gets besides Kernel's. +export function agentUserEnv(model, mcpConfig, env = process.env) { + const prefix = `${model.split("/")[0].toUpperCase().replace(/-/g, "_")}_`; + const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || name.startsWith(prefix)); + return { + ...Object.fromEntries(keys), + PI_OFFLINE: "1", + PI_TELEMETRY: "0", + BENCH_PATH: env.PATH, + ...(mcpConfig ? { BENCH_MCP_JSON: JSON.stringify(mcpConfig) } : {}), + }; +} + +export function asAgentUser(user, piArgs, env) { + const preserved = Object.keys(env).join(","); + return { + command: "sudo", + args: ["-n", "-u", user.name, `--preserve-env=${preserved}`, "--", "/bin/sh", "-c", LAUNCH, "sh", user.home, ...piArgs], + env: { PATH: process.env.PATH, ...env }, + }; +} + +// sudo passes SIGTERM on to pi, but a SIGKILL stops only sudo. +export function killAgentUser(user) { + try { + execFileSync("sudo", ["-n", "-u", user.name, "pkill", "-KILL", "-u", user.name], { stdio: "ignore" }); + } catch { + // pkill exits 1 when no process matched. + } +} + +// The paths an agent user can read or enter. +export function readablePaths(user, paths) { + return paths.filter(path => { + try { + execFileSync("sudo", ["-n", "-u", user.name, "test", "-r", path, "-o", "-x", path], { stdio: "ignore" }); + return true; + } catch { + return false; + } + }); +} + +export function canSudo(user) { + try { + execFileSync("sudo", ["-n", "-u", user.name, "true"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +// One agent per user at a time. +export class AgentUserPool { + constructor(users) { + this.users = users; + this.free = [...users]; + this.waiting = []; + } + + async acquire() { + if (this.free.length) return this.free.shift(); + const { promise, resolve } = Promise.withResolvers(); + this.waiting.push(resolve); + return promise; + } + + release(user) { + const next = this.waiting.shift(); + if (next) next(user); + else this.free.push(user); + } +} diff --git a/bench/run.mjs b/bench/run.mjs index 88b09114..6d9de0a0 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -22,6 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs"; import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs"; import { checkRule, gradeAnswer } from "./grade.mjs"; +import { AgentUserPool, agentUserEnv, asAgentUser, canSudo, findAgentUsers, killAgentUser, readablePaths } from "./lib/users.mjs"; const benchDir = dirname(fileURLToPath(import.meta.url)); if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env")); @@ -281,6 +282,34 @@ function trialArgs(plan, prompt) { return piArgs({ prompt, systemPrompt: experiment.system_prompt, allow: plan.allowlist, model: plan.model, thinking: plan.thinking }); } +// Set by the smoke gate when the allowlist has a built-in tool, which then runs as an agent user. +let agentUsers = null; + +function checkAgentUsers(slots) { + const users = findAgentUsers(); + const setup = `run \`sudo bench/setup-agent-users.sh $(id -un) ${slots}\``; + if (users.length < slots) fail(`${experiment.files.allowlist} has a built-in tool, so each slot needs an agent user; found ${users.length} for ${slots} slot(s); ${setup}`); + const hidden = [resolve(benchDir, ".."), process.env.HOME].filter(Boolean); + for (const user of users.slice(0, slots)) { + if (!canSudo(user)) fail(`cannot run commands as ${user.name} with \`sudo -n\`; ${setup}`); + const readable = readablePaths(user, hidden); + if (readable.length) fail(`${user.name} can read ${readable.join(", ")}; ${setup}`); + } + agentUsers = new AgentUserPool(users.slice(0, slots)); +} + +// pi as the bench user, or as an agent user, which gets only the model's and Kernel's keys. +async function runPi({ agentDir, mcpConfig, model, ...options }) { + if (!agentUsers) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); + const user = await agentUsers.acquire(); + try { + return await runAgent({ ...options, ...asAgentUser(user, options.args, agentUserEnv(model, mcpConfig)) }); + } finally { + killAgentUser(user); + agentUsers.release(user); + } +} + const stderrTail = (file) => (existsSync(file) ? readFileSync(file, "utf8") : "").trim().split("\n").slice(-3).join(" "); function logSmokeSpend(cost_usd) { @@ -301,9 +330,11 @@ async function smokeGate(plan, probe, signedIn) { plan.kernel_cli ??= kernelCli; } - const plain = await runAgent({ + if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkAgentUsers(flags.slots); + + const plain = await runPi({ args: piArgs({ prompt: "Reply with the single word READY.", systemPrompt: "Answer briefly.", model: plan.model, thinking: plan.thinking }), - env: agentEnv(join(dir, "plain-agent"), null), + agentDir: join(dir, "plain-agent"), mcpConfig: null, model: plan.model, cwd: dir, capMs: 120_000, transcriptPath: join(dir, "plain.jsonl"), stderrPath: join(dir, "plain.stderr"), }); if (stopping) return; @@ -321,9 +352,9 @@ async function smokeGate(plan, probe, signedIn) { const setup = await setUpBrowser(plan, probe, probeArm, "smoke"); if (setup.stopped) return; if (setup.error) fail(`${probeArm} setup failed for ${probe.intent.id} after ${setup.attempts} browsers: ${setup.error}`); - const configured = await runAgent({ + const configured = await runPi({ args: trialArgs(plan, "Reply with the single word READY. Do not call any tools."), - env: agentEnv(join(dir, "configured-agent"), mcpFor(plan, setup.browser)), + agentDir: join(dir, "configured-agent"), mcpConfig: mcpFor(plan, setup.browser), model: plan.model, cwd: dir, capMs: 120_000, transcriptPath: join(dir, "configured.jsonl"), stderrPath: join(dir, "configured.stderr"), }); await releaseBrowser(setup.browser); @@ -415,6 +446,7 @@ async function shutdown(code, message) { stopping = true; (code ? console.error : log)(message); for (const child of runningAgents) child.kill("SIGKILL"); + for (const user of agentUsers?.users ?? []) killAgentUser(user); await Promise.allSettled([...creating]); await Promise.allSettled([...openBrowsers].map(id => browsers.deleteBrowser(id))); process.exit(code); @@ -533,9 +565,9 @@ async function runTrial(plan, intentPlan, arm, key) { const agentStartedAt = Date.now(); row.agent_started_at = new Date(agentStartedAt).toISOString(); - const agent = await runAgent({ + const agent = await runPi({ args: trialArgs(plan, trialPrompt(intent, plan.backend === "kernel" ? browser : null)), - env: agentEnv(join(dir, "agent"), mcpFor(plan, browser)), + agentDir: join(dir, "agent"), mcpConfig: mcpFor(plan, browser), model: plan.model, cwd: dir, capMs: intentPlan.cap_sec * 1000, maxTurns: intentPlan.max_turns, maxBudgetUsd: plan.max_budget_usd, transcriptPath: join(dir, "transcript.jsonl"), stderrPath: join(dir, "stderr.txt"), }); diff --git a/bench/setup-agent-users.sh b/bench/setup-agent-users.sh new file mode 100755 index 00000000..c874b316 --- /dev/null +++ b/bench/setup-agent-users.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Creates the OS users agents run as, xwebmcp-bench-1 to xwebmcp-bench-, lets +# run commands as them without a password, and removes other users' access to this checkout and +# to 's home, so agents cannot read intents, results or keys. Linux only. +# Usage: sudo bench/setup-agent-users.sh +set -eu +[ $# -eq 2 ] || { echo "usage: sudo $0 " >&2; exit 2; } +bench_user=$1 +count=$2 +bench_home=$(getent passwd "$bench_user" | cut -d: -f6) +[ -n "$bench_home" ] || { echo "no user $bench_user" >&2; exit 1; } +checkout=$(cd "$(dirname "$0")/.." && pwd) + +mkdir -p /var/lib/xwebmcp-bench +users= +for i in $(seq 1 "$count"); do + name=xwebmcp-bench-$i + home=/var/lib/xwebmcp-bench/$i + id -u "$name" >/dev/null 2>&1 || useradd --system --home-dir "$home" --create-home --shell /usr/sbin/nologin "$name" + chmod 700 "$home" + users=${users:+$users,}$name +done + +rule=$(mktemp) +# sudo would log the agents' environment, API keys included. +printf 'Defaults>%s !log_allowed\n%s ALL=(%s) NOPASSWD:SETENV: ALL\n' "$users" "$bench_user" "$users" > "$rule" +visudo -cqf "$rule" +install -m 440 "$rule" /etc/sudoers.d/xwebmcp-bench +rm -f "$rule" + +chmod o-rwx "$checkout" "$bench_home" +echo "agent users: $users; $checkout and $bench_home are closed to other users" diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs index 712e74a7..1ac07d9c 100644 --- a/bench/test/run.test.mjs +++ b/bench/test/run.test.mjs @@ -240,3 +240,18 @@ test("a task on a domain with no tools runs control alone", { timeout: 120_000 } assert.ok(!log().some(entry => entry[0] === "add")); assert.match(run("uncovered").stdout, /nothing to run/); }); + +test("a run with a built-in tool stops before any agent without an agent user per slot", { timeout: 120_000 }, () => { + const { run, log } = workspace(); + const result = run("no-agent-users", { STUB_AGENT_USERS: "1" }, ["--slots", "2"]); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /found 1 for 2 slot\(s\); run `sudo bench\/setup-agent-users.sh/); + assert.deepEqual(log().filter(entry => entry[0] === "create"), []); +}); + +test("a run stops before any agent when an agent user can read the checkout", { timeout: 120_000 }, () => { + const { run } = workspace(); + const result = run("readable", { STUB_AGENT_CAN_READ: "1" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /xwebmcp-bench-1 can read /); +}); diff --git a/bench/test/stubs/getent b/bench/test/stubs/getent new file mode 100755 index 00000000..d3ff39f7 --- /dev/null +++ b/bench/test/stubs/getent @@ -0,0 +1,5 @@ +#!/usr/bin/env node +// A stand-in for `getent passwd` in run.test.mjs: STUB_AGENT_USERS agent users (default 4), with +// homes under $HOME. +const count = Number(process.env.STUB_AGENT_USERS ?? 4); +for (let i = 1; i <= count; i++) console.log(`xwebmcp-bench-${i}:x:${2000 + i}:${2000 + i}::${process.env.HOME}/agent-users/${i}:/usr/sbin/nologin`); diff --git a/bench/test/stubs/sudo b/bench/test/stubs/sudo new file mode 100755 index 00000000..b7c23a75 --- /dev/null +++ b/bench/test/stubs/sudo @@ -0,0 +1,20 @@ +#!/usr/bin/env node +// A stand-in for sudo in run.test.mjs: runs the command as the current user with only the preserved +// variables and PATH, the way `sudo --preserve-env` does. Agent users read nothing they test for, +// unless STUB_AGENT_CAN_READ is set. +import { spawnSync } from "node:child_process"; + +const args = process.argv.slice(2); +let preserved = []; +while (args[0] !== "--" && args[0]?.startsWith("-")) { + const flag = args.shift(); + if (flag === "-u") args.shift(); + if (flag.startsWith("--preserve-env=")) preserved = flag.slice("--preserve-env=".length).split(","); +} +if (args[0] === "--") args.shift(); +const [command, ...rest] = args; +if (command === "test") process.exit(process.env.STUB_AGENT_CAN_READ ? 0 : 1); +if (command === "pkill") process.exit(1); +const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => name === "PATH" || preserved.includes(name))); +const result = spawnSync(command, rest, { stdio: "inherit", env }); +process.exit(result.status ?? 1); diff --git a/bench/test/users.test.mjs b/bench/test/users.test.mjs new file mode 100644 index 00000000..144cb139 --- /dev/null +++ b/bench/test/users.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { AgentUserPool, agentUserEnv, asAgentUser, findAgentUsers } from "../lib/users.mjs"; + +const stubs = join(dirname(fileURLToPath(import.meta.url)), "stubs"); + +test("agent users are read from passwd in number order", () => { + const passwd = [ + "agent:x:1001:1003::/home/agent:/bin/bash", + "xwebmcp-bench-10:x:2010:2010::/var/lib/xwebmcp-bench/10:/usr/sbin/nologin", + "xwebmcp-bench-2:x:2002:2002::/var/lib/xwebmcp-bench/2:/usr/sbin/nologin", + "xwebmcp-bench-x:x:2099:2099::/tmp:/usr/sbin/nologin", + ].join("\n"); + assert.deepEqual(findAgentUsers(passwd), [ + { name: "xwebmcp-bench-2", home: "/var/lib/xwebmcp-bench/2" }, + { name: "xwebmcp-bench-10", home: "/var/lib/xwebmcp-bench/10" }, + ]); +}); + +test("an agent user gets only Kernel's key and the model provider's variables", () => { + const env = agentUserEnv("anthropic/claude-haiku-5-5", { mcpServers: {} }, { + PATH: "/usr/bin", KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", OPENAI_API_KEY: "o", GH_TOKEN: "g", + }); + assert.deepEqual(env, { + KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", PI_OFFLINE: "1", PI_TELEMETRY: "0", BENCH_PATH: "/usr/bin", BENCH_MCP_JSON: '{"mcpServers":{}}', + }); +}); + +test("pi runs as the agent user in a fresh directory with its own TMPDIR and mcp.json", () => { + const dir = mkdtempSync(join(tmpdir(), "bench-users-")); + const bin = join(dir, "bin"); + mkdirSync(bin); + // A pi that reports where it runs and what it was given. + writeFileSync(join(bin, "pi"), `#!/bin/sh +printf '%s\\n' "$PWD" "$TMPDIR" "$PI_CODING_AGENT_DIR" "\${GH_TOKEN-unset}" "\${BENCH_MCP_JSON-unset}" "$*" +cat "$PI_CODING_AGENT_DIR/mcp.json" +ls "$TMPDIR" +`); + chmodSync(join(bin, "pi"), 0o755); + const home = join(dir, "home"); + mkdirSync(join(home, "trial/tmp"), { recursive: true }); + writeFileSync(join(home, "trial/tmp/pi-mcp-previous.txt"), "left by the previous trial"); + + const path = [bin, stubs, process.env.PATH].join(delimiter); + const env = agentUserEnv("openai/gpt-5.4", { mcpServers: { kernel: {} } }, { PATH: path, KERNEL_API_KEY: "k", OPENAI_API_KEY: "o", GH_TOKEN: "g" }); + const { command, args, env: sudoEnv } = asAgentUser({ name: "xwebmcp-bench-1", home }, ["-p", "--", "a prompt"], env); + const result = spawnSync(command, args, { encoding: "utf8", env: { ...sudoEnv, PATH: path, GH_TOKEN: "g" } }); + assert.equal(result.status, 0, result.stderr); + const [cwd, tmp, agentDir, ghToken, mcpJson, piArgs, mcp, ...leftovers] = result.stdout.trim().split("\n"); + assert.deepEqual([cwd, tmp, agentDir], [join(home, "trial"), join(home, "trial/tmp"), join(home, "trial/agent")]); + assert.deepEqual([ghToken, mcpJson, piArgs], ["unset", "unset", "-p -- a prompt"]); + assert.deepEqual(JSON.parse(mcp), { mcpServers: { kernel: {} } }); + assert.deepEqual(leftovers, []); + assert.ok(!existsSync(join(home, "trial/tmp/pi-mcp-previous.txt"))); + assert.equal(readFileSync(join(home, "trial/agent/mcp.json"), "utf8"), '{"mcpServers":{"kernel":{}}}\n'); +}); + +test("the pool hands each user to one agent at a time", async () => { + const pool = new AgentUserPool([{ name: "a" }, { name: "b" }]); + const first = await pool.acquire(); + const second = await pool.acquire(); + let third = null; + const waiting = pool.acquire().then(user => { third = user; }); + await Promise.resolve(); + assert.equal(third, null); + pool.release(first); + await waiting; + assert.deepEqual([first.name, second.name, third.name], ["a", "b", "a"]); +}); From c37a7ee175d7b78e51eec544ff9e75bee9b6d5a0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:51:35 +0000 Subject: [PATCH 2/4] bench: sandbox agents with bubblewrap instead of separate OS users --- bench/README.md | 30 +++++------ bench/lib/sandbox.mjs | 86 ++++++++++++++++++++++++++++++ bench/lib/users.mjs | 102 ------------------------------------ bench/run.mjs | 46 ++++++++-------- bench/setup-agent-users.sh | 32 ----------- bench/test/run.test.mjs | 14 ++--- bench/test/sandbox.test.mjs | 61 +++++++++++++++++++++ bench/test/stubs/bwrap | 21 ++++++++ bench/test/stubs/getent | 5 -- bench/test/stubs/sudo | 20 ------- bench/test/users.test.mjs | 74 -------------------------- 11 files changed, 212 insertions(+), 279 deletions(-) create mode 100644 bench/lib/sandbox.mjs delete mode 100644 bench/lib/users.mjs delete mode 100755 bench/setup-agent-users.sh create mode 100644 bench/test/sandbox.test.mjs create mode 100755 bench/test/stubs/bwrap delete mode 100755 bench/test/stubs/getent delete mode 100755 bench/test/stubs/sudo delete mode 100644 bench/test/users.test.mjs diff --git a/bench/README.md b/bench/README.md index b6576337..6215592e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -31,7 +31,6 @@ bench/ allowlist.txt the tools the agent gets, and no others: the kernel MCP tools and pi's read mcp.json the kernel MCP server; reads KERNEL_API_KEY .env.example keys run.mjs loads from bench/.env - setup-agent-users.sh creates the OS users agents run as run.mjs grade.mjs report.mjs lib/ test/ results// gitignored: run.json, rows.jsonl, transcripts ``` @@ -42,21 +41,19 @@ Needs the `kernel` and `pi` CLIs, Node 22.13+, git, `KERNEL_API_KEY` and the mod key (such as `OPENAI_API_KEY`), from the environment or `bench/.env`. Each agent gets its own `PI_CODING_AGENT_DIR`, so stored pi logins and MCP servers are not used. -pi cuts an MCP result over 20KB, such as a long WebMCP tool list, and saves the full text to a -file, so the agent gets pi's `read` tool to open it. Agents on Kernel browsers therefore run as -dedicated OS users, `xwebmcp-bench-`, one per slot, that cannot read this checkout or your home: -each agent starts in an empty directory under its user's home with its own `TMPDIR`, and gets only -`KERNEL_API_KEY` and the variables of its model's provider (`ANTHROPIC_*` for `anthropic/...`). -Create the users once per machine, as many as `--slots`; this is Linux only, and pi must be -installed outside your home: +### Sandbox -```sh -sudo bench/setup-agent-users.sh "$(id -un)" 4 -``` - -The script lets you run commands as those users through `sudo` without logging their environment, -and removes other users' access to this checkout and your home. The smoke gate stops the run if a -slot has no agent user, or if one can read either. +pi cuts an MCP result over 20KB, such as a long WebMCP tool list, and saves the full text to a +file, so the agent gets pi's `read` tool to open it. So that `read` cannot reach the bench's own +files, agents on Kernel browsers run in a [bubblewrap](https://github.com/containers/bubblewrap) +sandbox (`bwrap`, Linux only; `apt install bubblewrap`). It sees `/usr`, the `/etc` files DNS and +TLS need, its own processes and a fresh directory with its pi config and `TMPDIR`; not your home, +this checkout, other trials or the rest of `/tmp` and `/etc`. It shares the network. The agent gets +only `KERNEL_API_KEY` and the variables of its model's provider (`ANTHROPIC_*` for +`anthropic/...`). Node and pi installed outside `/usr`, such as under `~/.nvm`, are mounted +read-only from their own directories. Unprivileged user namespaces must be allowed, which some +distributions restrict with AppArmor; the smoke gate stops the run if `bwrap` cannot create a +sandbox or if one can see this checkout or your home. Run it as a Hypeship service under a hold that covers the whole run; holds cannot be renewed. This pilot is 105 trials and can take about 8 hours if every trial reaches its cap: @@ -125,8 +122,7 @@ Before any trial, stopping on the first failure: 1. the hold covers the remaining jobs if every trial reaches its cap after 180s of setup (the hold is checked again before each job, and no job starts that it cannot outlast); 2. `KERNEL_API_KEY` is set (Kernel browsers), and on resume pi is the same version; - with Kernel browsers, each slot has an agent user that can read neither this checkout nor your - home; + with Kernel browsers, a sandbox works and can see neither this checkout nor your home; 3. a zero-tool pi run answers and reports a cost above zero (pi prices an unknown model at 0, which would disable the spend cap); 4. a treatment setup on the first pending intent works end to end; diff --git a/bench/lib/sandbox.mjs b/bench/lib/sandbox.mjs new file mode 100644 index 00000000..c78d92dd --- /dev/null +++ b/bench/lib/sandbox.mjs @@ -0,0 +1,86 @@ +// Agents with a built-in tool such as `read` run in a bubblewrap sandbox, so they cannot read the +// bench's files: intents and their rules, results, other trials, the home directory or keys. The +// sandbox shows /usr, the /etc files DNS and TLS need, the agent's own processes, and a fresh +// directory with its pi config and TMPDIR, where pi saves the full text of a long MCP result. + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, join } from "node:path"; + +const ETC = [ + "/etc/resolv.conf", "/etc/hosts", "/etc/nsswitch.conf", "/etc/gai.conf", "/etc/ssl", "/etc/pki", + "/etc/ca-certificates", "/etc/passwd", "/etc/group", "/etc/localtime", "/etc/alternatives", +]; + +const under = (path, dir) => path === dir || path.startsWith(`${dir}/`); + +// Node's and pi's installation prefixes when they are outside /usr, such as ~/.nvm/versions/node/v22. +export function installMounts(env = process.env, execPath = process.execPath) { + const pi = (env.PATH ?? "").split(delimiter).map(dir => join(dir, "pi")).find(path => existsSync(path)); + const prefixes = [dirname(dirname(realpathSync(execPath)))]; + if (pi) { + const real = realpathSync(pi); + const modules = real.indexOf("/lib/node_modules/"); + prefixes.push(modules > 0 ? real.slice(0, modules) : dirname(real)); + } + return [...new Set(prefixes)].filter(prefix => !under(prefix, "/usr")); +} + +export function sandboxArgs(dir, mounts = []) { + return [ + "--unshare-all", "--share-net", "--die-with-parent", "--new-session", + "--ro-bind", "/usr", "/usr", + ...["/bin", "/sbin", "/lib", "/lib64"].flatMap(path => ["--ro-bind-try", path, path]), + ...ETC.flatMap(path => ["--ro-bind-try", path, path]), + ...mounts.flatMap(path => ["--ro-bind", path, path]), + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", + "--bind", dir, dir, "--chdir", dir, + ]; +} + +// A fresh directory for one agent, with its mcp.json, under the host's TMPDIR. +export function sandboxDir(mcpConfig) { + const dir = mkdtempSync(join(tmpdir(), "xwebmcp-agent-")); + mkdirSync(join(dir, "agent")); + mkdirSync(join(dir, "tmp")); + if (mcpConfig) writeFileSync(join(dir, "agent", "mcp.json"), JSON.stringify(mcpConfig, null, 2) + "\n"); + return dir; +} + +// The model's provider is the part of --model before the slash; its variables, such as +// ANTHROPIC_API_KEY for anthropic/..., are the only keys the agent gets besides Kernel's. +export function sandboxEnv(model, dir, env = process.env) { + const prefix = `${model.split("/")[0].toUpperCase().replace(/-/g, "_")}_`; + const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || name.startsWith(prefix)); + return { + ...Object.fromEntries(keys), + PATH: env.PATH, + HOME: dir, + TMPDIR: join(dir, "tmp"), + PI_CODING_AGENT_DIR: join(dir, "agent"), + PI_OFFLINE: "1", + PI_TELEMETRY: "0", + }; +} + +// The paths to check a sandbox cannot see: the bench, and each entry of the home directory except +// those that hold a mount, which bwrap creates as empty directories. +export function hiddenPaths(benchDir, home, mounts) { + const entries = home ? readdirSync(home).map(name => join(home, name)) : []; + return [benchDir, ...entries.filter(path => !mounts.some(mount => under(mount, path)))]; +} + +// Throws when bwrap cannot create a sandbox; returns the paths a sandbox can see. +export function visiblePaths(paths, mounts) { + const dir = sandboxDir(null); + try { + const probe = ["sh", "-c", 'for p; do [ -e "$p" ] && echo "$p"; done; true', "sh", ...paths]; + const output = execFileSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...probe], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + return output.split("\n").filter(Boolean); + } catch (error) { + throw new Error(`bwrap cannot create a sandbox: ${error.stderr?.toString().trim() || error.message}`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} diff --git a/bench/lib/users.mjs b/bench/lib/users.mjs deleted file mode 100644 index 842db52f..00000000 --- a/bench/lib/users.mjs +++ /dev/null @@ -1,102 +0,0 @@ -// Agents with a built-in tool such as `read` run as dedicated OS users, xwebmcp-bench-, so -// they cannot read the bench's files: intents and their rules, results, other trials or keys. -// bench/setup-agent-users.sh creates the users and removes other users' access to the checkout -// and the bench user's home; the smoke gate checks that before any agent starts. - -import { execFileSync } from "node:child_process"; - -export const AGENT_USER = /^xwebmcp-bench-(\d+)$/; - -// Each user's agents get a fresh directory under its home, including TMPDIR, where pi saves the -// full text of a long MCP result, so nothing is left from the previous trial. -const LAUNCH = `set -eu -home=$1; shift -umask 077 -rm -rf "$home/trial" -mkdir -p "$home/trial/agent" "$home/trial/tmp" -if [ -n "\${BENCH_MCP_JSON-}" ]; then printf '%s\\n' "$BENCH_MCP_JSON" > "$home/trial/agent/mcp.json"; fi -export HOME="$home" PATH="$BENCH_PATH" PI_CODING_AGENT_DIR="$home/trial/agent" TMPDIR="$home/trial/tmp" -unset BENCH_MCP_JSON BENCH_PATH -cd "$home/trial" -exec pi "$@"`; - -export function findAgentUsers(passwd = execFileSync("getent", ["passwd"], { encoding: "utf8" })) { - return passwd.split("\n").map(line => line.split(":")).filter(fields => AGENT_USER.test(fields[0])) - .map(fields => ({ name: fields[0], home: fields[5] })) - .sort((a, b) => Number(AGENT_USER.exec(a.name)[1]) - Number(AGENT_USER.exec(b.name)[1])); -} - -// The model's provider is the part of --model before the slash; its variables, such as -// ANTHROPIC_API_KEY for anthropic/..., are the only keys the agent gets besides Kernel's. -export function agentUserEnv(model, mcpConfig, env = process.env) { - const prefix = `${model.split("/")[0].toUpperCase().replace(/-/g, "_")}_`; - const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || name.startsWith(prefix)); - return { - ...Object.fromEntries(keys), - PI_OFFLINE: "1", - PI_TELEMETRY: "0", - BENCH_PATH: env.PATH, - ...(mcpConfig ? { BENCH_MCP_JSON: JSON.stringify(mcpConfig) } : {}), - }; -} - -export function asAgentUser(user, piArgs, env) { - const preserved = Object.keys(env).join(","); - return { - command: "sudo", - args: ["-n", "-u", user.name, `--preserve-env=${preserved}`, "--", "/bin/sh", "-c", LAUNCH, "sh", user.home, ...piArgs], - env: { PATH: process.env.PATH, ...env }, - }; -} - -// sudo passes SIGTERM on to pi, but a SIGKILL stops only sudo. -export function killAgentUser(user) { - try { - execFileSync("sudo", ["-n", "-u", user.name, "pkill", "-KILL", "-u", user.name], { stdio: "ignore" }); - } catch { - // pkill exits 1 when no process matched. - } -} - -// The paths an agent user can read or enter. -export function readablePaths(user, paths) { - return paths.filter(path => { - try { - execFileSync("sudo", ["-n", "-u", user.name, "test", "-r", path, "-o", "-x", path], { stdio: "ignore" }); - return true; - } catch { - return false; - } - }); -} - -export function canSudo(user) { - try { - execFileSync("sudo", ["-n", "-u", user.name, "true"], { stdio: "ignore" }); - return true; - } catch { - return false; - } -} - -// One agent per user at a time. -export class AgentUserPool { - constructor(users) { - this.users = users; - this.free = [...users]; - this.waiting = []; - } - - async acquire() { - if (this.free.length) return this.free.shift(); - const { promise, resolve } = Promise.withResolvers(); - this.waiting.push(resolve); - return promise; - } - - release(user) { - const next = this.waiting.shift(); - if (next) next(user); - else this.free.push(user); - } -} diff --git a/bench/run.mjs b/bench/run.mjs index 6d9de0a0..29a1251d 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -22,7 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs"; import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs"; import { checkRule, gradeAnswer } from "./grade.mjs"; -import { AgentUserPool, agentUserEnv, asAgentUser, canSudo, findAgentUsers, killAgentUser, readablePaths } from "./lib/users.mjs"; +import { hiddenPaths, installMounts, sandboxArgs, sandboxDir, sandboxEnv, visiblePaths } from "./lib/sandbox.mjs"; const benchDir = dirname(fileURLToPath(import.meta.url)); if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env")); @@ -282,31 +282,33 @@ function trialArgs(plan, prompt) { return piArgs({ prompt, systemPrompt: experiment.system_prompt, allow: plan.allowlist, model: plan.model, thinking: plan.thinking }); } -// Set by the smoke gate when the allowlist has a built-in tool, which then runs as an agent user. -let agentUsers = null; - -function checkAgentUsers(slots) { - const users = findAgentUsers(); - const setup = `run \`sudo bench/setup-agent-users.sh $(id -un) ${slots}\``; - if (users.length < slots) fail(`${experiment.files.allowlist} has a built-in tool, so each slot needs an agent user; found ${users.length} for ${slots} slot(s); ${setup}`); - const hidden = [resolve(benchDir, ".."), process.env.HOME].filter(Boolean); - for (const user of users.slice(0, slots)) { - if (!canSudo(user)) fail(`cannot run commands as ${user.name} with \`sudo -n\`; ${setup}`); - const readable = readablePaths(user, hidden); - if (readable.length) fail(`${user.name} can read ${readable.join(", ")}; ${setup}`); +// Set by the smoke gate when the allowlist has a built-in tool: agents then run in a sandbox, with +// these read-only mounts besides /usr. Open sandbox directories are removed on shutdown. +let sandboxMounts = null; +const sandboxDirs = new Set(); + +function checkSandbox() { + const mounts = installMounts(); + let visible; + try { + visible = visiblePaths(hiddenPaths(benchDir, process.env.HOME, mounts), mounts); + } catch (error) { + fail(`${experiment.files.allowlist} has a built-in tool, so agents run in a bubblewrap sandbox, and ${error.message}; see "Sandbox" in bench/README.md`); } - agentUsers = new AgentUserPool(users.slice(0, slots)); + if (visible.length) fail(`the agent sandbox can see ${visible.join(", ")}; pi or Node is installed in a directory that holds it, so install them in a directory of their own`); + sandboxMounts = mounts; } -// pi as the bench user, or as an agent user, which gets only the model's and Kernel's keys. +// pi as the bench user, or in a sandbox that gets only the model's and Kernel's keys. async function runPi({ agentDir, mcpConfig, model, ...options }) { - if (!agentUsers) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); - const user = await agentUsers.acquire(); + if (!sandboxMounts) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); + const dir = sandboxDir(mcpConfig); + sandboxDirs.add(dir); try { - return await runAgent({ ...options, ...asAgentUser(user, options.args, agentUserEnv(model, mcpConfig)) }); + return await runAgent({ ...options, command: "bwrap", args: [...sandboxArgs(dir, sandboxMounts), "--", "pi", ...options.args], env: sandboxEnv(model, dir) }); } finally { - killAgentUser(user); - agentUsers.release(user); + rmSync(dir, { recursive: true, force: true }); + sandboxDirs.delete(dir); } } @@ -330,7 +332,7 @@ async function smokeGate(plan, probe, signedIn) { plan.kernel_cli ??= kernelCli; } - if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkAgentUsers(flags.slots); + if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkSandbox(); const plain = await runPi({ args: piArgs({ prompt: "Reply with the single word READY.", systemPrompt: "Answer briefly.", model: plan.model, thinking: plan.thinking }), @@ -446,7 +448,7 @@ async function shutdown(code, message) { stopping = true; (code ? console.error : log)(message); for (const child of runningAgents) child.kill("SIGKILL"); - for (const user of agentUsers?.users ?? []) killAgentUser(user); + for (const dir of sandboxDirs) rmSync(dir, { recursive: true, force: true }); await Promise.allSettled([...creating]); await Promise.allSettled([...openBrowsers].map(id => browsers.deleteBrowser(id))); process.exit(code); diff --git a/bench/setup-agent-users.sh b/bench/setup-agent-users.sh deleted file mode 100755 index c874b316..00000000 --- a/bench/setup-agent-users.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Creates the OS users agents run as, xwebmcp-bench-1 to xwebmcp-bench-, lets -# run commands as them without a password, and removes other users' access to this checkout and -# to 's home, so agents cannot read intents, results or keys. Linux only. -# Usage: sudo bench/setup-agent-users.sh -set -eu -[ $# -eq 2 ] || { echo "usage: sudo $0 " >&2; exit 2; } -bench_user=$1 -count=$2 -bench_home=$(getent passwd "$bench_user" | cut -d: -f6) -[ -n "$bench_home" ] || { echo "no user $bench_user" >&2; exit 1; } -checkout=$(cd "$(dirname "$0")/.." && pwd) - -mkdir -p /var/lib/xwebmcp-bench -users= -for i in $(seq 1 "$count"); do - name=xwebmcp-bench-$i - home=/var/lib/xwebmcp-bench/$i - id -u "$name" >/dev/null 2>&1 || useradd --system --home-dir "$home" --create-home --shell /usr/sbin/nologin "$name" - chmod 700 "$home" - users=${users:+$users,}$name -done - -rule=$(mktemp) -# sudo would log the agents' environment, API keys included. -printf 'Defaults>%s !log_allowed\n%s ALL=(%s) NOPASSWD:SETENV: ALL\n' "$users" "$bench_user" "$users" > "$rule" -visudo -cqf "$rule" -install -m 440 "$rule" /etc/sudoers.d/xwebmcp-bench -rm -f "$rule" - -chmod o-rwx "$checkout" "$bench_home" -echo "agent users: $users; $checkout and $bench_home are closed to other users" diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs index 1ac07d9c..d27ed9b6 100644 --- a/bench/test/run.test.mjs +++ b/bench/test/run.test.mjs @@ -241,17 +241,17 @@ test("a task on a domain with no tools runs control alone", { timeout: 120_000 } assert.match(run("uncovered").stdout, /nothing to run/); }); -test("a run with a built-in tool stops before any agent without an agent user per slot", { timeout: 120_000 }, () => { +test("a run with a built-in tool stops before any agent when bwrap cannot create a sandbox", { timeout: 120_000 }, () => { const { run, log } = workspace(); - const result = run("no-agent-users", { STUB_AGENT_USERS: "1" }, ["--slots", "2"]); + const result = run("no-sandbox", { STUB_BWRAP_FAILS: "1" }); assert.notEqual(result.status, 0); - assert.match(result.stderr, /found 1 for 2 slot\(s\); run `sudo bench\/setup-agent-users.sh/); + assert.match(result.stderr, /bwrap cannot create a sandbox: bwrap: setting up uid map: Permission denied/); assert.deepEqual(log().filter(entry => entry[0] === "create"), []); }); -test("a run stops before any agent when an agent user can read the checkout", { timeout: 120_000 }, () => { - const { run } = workspace(); - const result = run("readable", { STUB_AGENT_CAN_READ: "1" }); +test("a run stops before any agent when the sandbox can see the bench", { timeout: 120_000 }, () => { + const { run, bench } = workspace(); + const result = run("leaky", { STUB_SANDBOX_LEAKS: "1" }); assert.notEqual(result.status, 0); - assert.match(result.stderr, /xwebmcp-bench-1 can read /); + assert.match(result.stderr, new RegExp(`the agent sandbox can see ${bench}`)); }); diff --git a/bench/test/sandbox.test.mjs b/bench/test/sandbox.test.mjs new file mode 100644 index 00000000..916b9d1d --- /dev/null +++ b/bench/test/sandbox.test.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { hiddenPaths, installMounts, sandboxArgs, sandboxDir, sandboxEnv } from "../lib/sandbox.mjs"; + +test("a sandboxed agent gets only Kernel's key, the model provider's variables and its own directories", () => { + const env = sandboxEnv("anthropic/claude-haiku-5-5", "/tmp/xwebmcp-agent-1", { + PATH: "/usr/bin", HOME: "/home/me", KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", OPENAI_API_KEY: "o", GH_TOKEN: "g", + }); + assert.deepEqual(env, { + KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", PATH: "/usr/bin", HOME: "/tmp/xwebmcp-agent-1", + TMPDIR: "/tmp/xwebmcp-agent-1/tmp", PI_CODING_AGENT_DIR: "/tmp/xwebmcp-agent-1/agent", PI_OFFLINE: "1", PI_TELEMETRY: "0", + }); +}); + +test("the sandbox binds its own directory over an empty /tmp and nothing from home", () => { + const args = sandboxArgs("/tmp/xwebmcp-agent-1", ["/home/me/.nvm/versions/node/v22"]); + const bound = []; + for (let i = 0; i < args.length; i++) if (args[i].startsWith("--") && args[i].includes("bind")) bound.push(args[i + 2]); + assert.ok(bound.includes("/usr") && bound.includes("/tmp/xwebmcp-agent-1")); + assert.deepEqual(bound.filter(path => path.startsWith("/home")), ["/home/me/.nvm/versions/node/v22"]); + assert.ok(!bound.includes("/etc")); + assert.ok(args.indexOf("--tmpfs") < args.indexOf("--bind")); +}); + +test("the paths checked hidden are the bench and the home entries that hold no mount", () => { + const home = mkdtempSync(join(tmpdir(), "bench-home-")); + for (const name of [".nvm", "xwebmcp", ".config"]) mkdirSync(join(home, name)); + const paths = hiddenPaths(join(home, "xwebmcp/bench"), home, [join(home, ".nvm/versions/node/v22")]); + assert.deepEqual(paths.sort(), [join(home, ".config"), join(home, "xwebmcp"), join(home, "xwebmcp/bench")]); +}); + +const bwrapWorks = spawnSync("bwrap", [...sandboxArgs(tmpdir()), "--", "true"]).status === 0; + +test("in a real sandbox the agent sees its directory, not other files or processes", { skip: !bwrapWorks && "bwrap cannot create a sandbox here" }, () => { + const other = mkdtempSync(join(tmpdir(), "bench-other-")); + writeFileSync(join(other, "secret.txt"), "secret"); + const dir = sandboxDir({ mcpServers: { kernel: {} } }); + try { + const script = `const fs = require("fs"); +fs.writeFileSync(process.env.TMPDIR + "/pi-mcp-1.txt", "full output"); +console.log(JSON.stringify({ + secret: fs.existsSync(${JSON.stringify(join(other, "secret.txt"))}), + etc: fs.existsSync("/etc/hostname"), + mcp: JSON.parse(fs.readFileSync(process.env.PI_CODING_AGENT_DIR + "/mcp.json", "utf8")), + processes: fs.readdirSync("/proc").filter(name => /^\\d+$/.test(name)).length, + cwd: process.cwd(), +}));`; + const result = spawnSync("bwrap", [...sandboxArgs(dir, installMounts()), "--", process.execPath, "-e", script], { encoding: "utf8", env: sandboxEnv("openai/gpt", dir, { PATH: process.env.PATH }) }); + assert.equal(result.status, 0, result.stderr); + const seen = JSON.parse(result.stdout); + assert.deepEqual({ ...seen, processes: seen.processes <= 3 }, { secret: false, etc: false, mcp: { mcpServers: { kernel: {} } }, processes: true, cwd: dir }); + assert.equal(readFileSync(join(dir, "tmp/pi-mcp-1.txt"), "utf8"), "full output"); + } finally { + rmSync(dir, { recursive: true, force: true }); + rmSync(other, { recursive: true, force: true }); + } +}); diff --git a/bench/test/stubs/bwrap b/bench/test/stubs/bwrap new file mode 100755 index 00000000..5fc064de --- /dev/null +++ b/bench/test/stubs/bwrap @@ -0,0 +1,21 @@ +#!/usr/bin/env node +// A stand-in for bwrap in run.test.mjs: runs the command in --chdir with the caller's environment, +// without mounts. The visibility probe sees nothing unless STUB_SANDBOX_LEAKS is set, and every +// sandbox fails when STUB_BWRAP_FAILS is set. +import { spawnSync } from "node:child_process"; + +if (process.env.STUB_BWRAP_FAILS) { + console.error("bwrap: setting up uid map: Permission denied"); + process.exit(1); +} +const args = process.argv.slice(2); +const dashes = args.indexOf("--"); +const options = args.slice(0, dashes); +const [command, ...rest] = args.slice(dashes + 1); +const chdir = options.includes("--chdir") ? options[options.indexOf("--chdir") + 1] : process.cwd(); +if (command === "sh" && rest[0] === "-c") { + if (process.env.STUB_SANDBOX_LEAKS) console.log(rest.slice(3).join("\n")); + process.exit(0); +} +const result = spawnSync(command, rest, { stdio: "inherit", cwd: chdir }); +process.exit(result.status ?? 1); diff --git a/bench/test/stubs/getent b/bench/test/stubs/getent deleted file mode 100755 index d3ff39f7..00000000 --- a/bench/test/stubs/getent +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env node -// A stand-in for `getent passwd` in run.test.mjs: STUB_AGENT_USERS agent users (default 4), with -// homes under $HOME. -const count = Number(process.env.STUB_AGENT_USERS ?? 4); -for (let i = 1; i <= count; i++) console.log(`xwebmcp-bench-${i}:x:${2000 + i}:${2000 + i}::${process.env.HOME}/agent-users/${i}:/usr/sbin/nologin`); diff --git a/bench/test/stubs/sudo b/bench/test/stubs/sudo deleted file mode 100755 index b7c23a75..00000000 --- a/bench/test/stubs/sudo +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env node -// A stand-in for sudo in run.test.mjs: runs the command as the current user with only the preserved -// variables and PATH, the way `sudo --preserve-env` does. Agent users read nothing they test for, -// unless STUB_AGENT_CAN_READ is set. -import { spawnSync } from "node:child_process"; - -const args = process.argv.slice(2); -let preserved = []; -while (args[0] !== "--" && args[0]?.startsWith("-")) { - const flag = args.shift(); - if (flag === "-u") args.shift(); - if (flag.startsWith("--preserve-env=")) preserved = flag.slice("--preserve-env=".length).split(","); -} -if (args[0] === "--") args.shift(); -const [command, ...rest] = args; -if (command === "test") process.exit(process.env.STUB_AGENT_CAN_READ ? 0 : 1); -if (command === "pkill") process.exit(1); -const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => name === "PATH" || preserved.includes(name))); -const result = spawnSync(command, rest, { stdio: "inherit", env }); -process.exit(result.status ?? 1); diff --git a/bench/test/users.test.mjs b/bench/test/users.test.mjs deleted file mode 100644 index 144cb139..00000000 --- a/bench/test/users.test.mjs +++ /dev/null @@ -1,74 +0,0 @@ -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { delimiter, dirname, join } from "node:path"; -import test from "node:test"; -import { fileURLToPath } from "node:url"; -import { AgentUserPool, agentUserEnv, asAgentUser, findAgentUsers } from "../lib/users.mjs"; - -const stubs = join(dirname(fileURLToPath(import.meta.url)), "stubs"); - -test("agent users are read from passwd in number order", () => { - const passwd = [ - "agent:x:1001:1003::/home/agent:/bin/bash", - "xwebmcp-bench-10:x:2010:2010::/var/lib/xwebmcp-bench/10:/usr/sbin/nologin", - "xwebmcp-bench-2:x:2002:2002::/var/lib/xwebmcp-bench/2:/usr/sbin/nologin", - "xwebmcp-bench-x:x:2099:2099::/tmp:/usr/sbin/nologin", - ].join("\n"); - assert.deepEqual(findAgentUsers(passwd), [ - { name: "xwebmcp-bench-2", home: "/var/lib/xwebmcp-bench/2" }, - { name: "xwebmcp-bench-10", home: "/var/lib/xwebmcp-bench/10" }, - ]); -}); - -test("an agent user gets only Kernel's key and the model provider's variables", () => { - const env = agentUserEnv("anthropic/claude-haiku-5-5", { mcpServers: {} }, { - PATH: "/usr/bin", KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", OPENAI_API_KEY: "o", GH_TOKEN: "g", - }); - assert.deepEqual(env, { - KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", PI_OFFLINE: "1", PI_TELEMETRY: "0", BENCH_PATH: "/usr/bin", BENCH_MCP_JSON: '{"mcpServers":{}}', - }); -}); - -test("pi runs as the agent user in a fresh directory with its own TMPDIR and mcp.json", () => { - const dir = mkdtempSync(join(tmpdir(), "bench-users-")); - const bin = join(dir, "bin"); - mkdirSync(bin); - // A pi that reports where it runs and what it was given. - writeFileSync(join(bin, "pi"), `#!/bin/sh -printf '%s\\n' "$PWD" "$TMPDIR" "$PI_CODING_AGENT_DIR" "\${GH_TOKEN-unset}" "\${BENCH_MCP_JSON-unset}" "$*" -cat "$PI_CODING_AGENT_DIR/mcp.json" -ls "$TMPDIR" -`); - chmodSync(join(bin, "pi"), 0o755); - const home = join(dir, "home"); - mkdirSync(join(home, "trial/tmp"), { recursive: true }); - writeFileSync(join(home, "trial/tmp/pi-mcp-previous.txt"), "left by the previous trial"); - - const path = [bin, stubs, process.env.PATH].join(delimiter); - const env = agentUserEnv("openai/gpt-5.4", { mcpServers: { kernel: {} } }, { PATH: path, KERNEL_API_KEY: "k", OPENAI_API_KEY: "o", GH_TOKEN: "g" }); - const { command, args, env: sudoEnv } = asAgentUser({ name: "xwebmcp-bench-1", home }, ["-p", "--", "a prompt"], env); - const result = spawnSync(command, args, { encoding: "utf8", env: { ...sudoEnv, PATH: path, GH_TOKEN: "g" } }); - assert.equal(result.status, 0, result.stderr); - const [cwd, tmp, agentDir, ghToken, mcpJson, piArgs, mcp, ...leftovers] = result.stdout.trim().split("\n"); - assert.deepEqual([cwd, tmp, agentDir], [join(home, "trial"), join(home, "trial/tmp"), join(home, "trial/agent")]); - assert.deepEqual([ghToken, mcpJson, piArgs], ["unset", "unset", "-p -- a prompt"]); - assert.deepEqual(JSON.parse(mcp), { mcpServers: { kernel: {} } }); - assert.deepEqual(leftovers, []); - assert.ok(!existsSync(join(home, "trial/tmp/pi-mcp-previous.txt"))); - assert.equal(readFileSync(join(home, "trial/agent/mcp.json"), "utf8"), '{"mcpServers":{"kernel":{}}}\n'); -}); - -test("the pool hands each user to one agent at a time", async () => { - const pool = new AgentUserPool([{ name: "a" }, { name: "b" }]); - const first = await pool.acquire(); - const second = await pool.acquire(); - let third = null; - const waiting = pool.acquire().then(user => { third = user; }); - await Promise.resolve(); - assert.equal(third, null); - pool.release(first); - await waiting; - assert.deepEqual([first.name, second.name, third.name], ["a", "b", "a"]); -}); From 1e390a36b6f72576f4a753b73e923790bcba2b9b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:07:14 +0000 Subject: [PATCH 3/4] bench: narrow sandbox mounts, map provider keys, redact keys from transcripts Mount only Node's binary and pi's package instead of their install prefixes, so a prefix such as ~/.local does not expose its other files. Pass the key variables of providers not named after them, such as GEMINI_API_KEY for google/. Create sandbox directories under /tmp, delete browsers before removing sandbox directories on shutdown, and redact the agent's keys from transcripts. --- bench/README.md | 5 ++-- bench/lib/agent.mjs | 7 +++-- bench/lib/sandbox.mjs | 57 ++++++++++++++++++++++++------------- bench/run.mjs | 23 ++++++++------- bench/test/agent.test.mjs | 7 +++++ bench/test/sandbox.test.mjs | 56 ++++++++++++++++++++++++++++++------ 6 files changed, 112 insertions(+), 43 deletions(-) diff --git a/bench/README.md b/bench/README.md index 6215592e..938dfbb0 100644 --- a/bench/README.md +++ b/bench/README.md @@ -50,8 +50,9 @@ sandbox (`bwrap`, Linux only; `apt install bubblewrap`). It sees `/usr`, the `/e TLS need, its own processes and a fresh directory with its pi config and `TMPDIR`; not your home, this checkout, other trials or the rest of `/tmp` and `/etc`. It shares the network. The agent gets only `KERNEL_API_KEY` and the variables of its model's provider (`ANTHROPIC_*` for -`anthropic/...`). Node and pi installed outside `/usr`, such as under `~/.nvm`, are mounted -read-only from their own directories. Unprivileged user namespaces must be allowed, which some +`anthropic/...`, `GEMINI_*` for `google/...`); their values are redacted from transcripts. If Node +or pi is installed outside `/usr`, such as under `~/.nvm` or `~/.local`, only Node's binary and +pi's package are mounted, read-only. Unprivileged user namespaces must be allowed, which some distributions restrict with AppArmor; the smoke gate stops the run if `bwrap` cannot create a sandbox or if one can see this checkout or your home. diff --git a/bench/lib/agent.mjs b/bench/lib/agent.mjs index eb705e52..89fa0a6a 100644 --- a/bench/lib/agent.mjs +++ b/bench/lib/agent.mjs @@ -31,8 +31,9 @@ export function parseRecord(line) { } // The clock runs from launch to agent_settled. pi has no turn or spend cap, so the runner -// enforces both; a missing cost stops the agent, since the spend cap needs it. -export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, transcriptPath, stderrPath }) { +// enforces both; a missing cost stops the agent, since the spend cap needs it. Each string in +// `redact` is replaced in the transcript. +export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, redact = [], transcriptPath, stderrPath }) { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); runningAgents.add(child); @@ -61,7 +62,7 @@ export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Inf const onRecord = (line) => { if (!line.trim()) return; - const event = parseRecord(line); + const event = parseRecord(redact.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), line)); event._t = elapsed(); events.push(event); transcript.write(JSON.stringify(event) + "\n"); diff --git a/bench/lib/sandbox.mjs b/bench/lib/sandbox.mjs index c78d92dd..aa0574b2 100644 --- a/bench/lib/sandbox.mjs +++ b/bench/lib/sandbox.mjs @@ -5,8 +5,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { delimiter, dirname, join } from "node:path"; +import { delimiter, join } from "node:path"; const ETC = [ "/etc/resolv.conf", "/etc/hosts", "/etc/nsswitch.conf", "/etc/gai.conf", "/etc/ssl", "/etc/pki", @@ -15,16 +14,15 @@ const ETC = [ const under = (path, dir) => path === dir || path.startsWith(`${dir}/`); -// Node's and pi's installation prefixes when they are outside /usr, such as ~/.nvm/versions/node/v22. -export function installMounts(env = process.env, execPath = process.execPath) { - const pi = (env.PATH ?? "").split(delimiter).map(dir => join(dir, "pi")).find(path => existsSync(path)); - const prefixes = [dirname(dirname(realpathSync(execPath)))]; - if (pi) { - const real = realpathSync(pi); - const modules = real.indexOf("/lib/node_modules/"); - prefixes.push(modules > 0 ? real.slice(0, modules) : dirname(real)); - } - return [...new Set(prefixes)].filter(prefix => !under(prefix, "/usr")); +const which = (name, env) => (env.PATH ?? "").split(delimiter).map(dir => join(dir, name)).find(path => existsSync(path)); + +// How a sandbox runs pi: with the host's node, mounting Node's binary and pi's package by themselves +// when they are outside /usr, so the rest of a prefix such as ~/.local or ~/.nvm stays hidden. +export function piLauncher(env = process.env, execPath = process.execPath) { + const node = realpathSync(which("node", env) ?? execPath); + const cli = realpathSync(which("pi", env)); + const pkg = cli.match(/^(.*\/node_modules\/(?:@[^/]+\/)?[^/]+)\//)?.[1] ?? cli; + return { mounts: [...new Set([node, pkg])].filter(path => !under(path, "/usr")), command: [node, cli] }; } export function sandboxArgs(dir, mounts = []) { @@ -33,26 +31,41 @@ export function sandboxArgs(dir, mounts = []) { "--ro-bind", "/usr", "/usr", ...["/bin", "/sbin", "/lib", "/lib64"].flatMap(path => ["--ro-bind-try", path, path]), ...ETC.flatMap(path => ["--ro-bind-try", path, path]), - ...mounts.flatMap(path => ["--ro-bind", path, path]), "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", + ...mounts.flatMap(path => ["--ro-bind", path, path]), "--bind", dir, dir, "--chdir", dir, ]; } -// A fresh directory for one agent, with its mcp.json, under the host's TMPDIR. +// A fresh directory for one agent, with its mcp.json. It is under /tmp, not the host's TMPDIR, +// which may be in the home directory and would then show the sandbox an entry of it. export function sandboxDir(mcpConfig) { - const dir = mkdtempSync(join(tmpdir(), "xwebmcp-agent-")); + const dir = mkdtempSync("/tmp/xwebmcp-agent-"); mkdirSync(join(dir, "agent")); mkdirSync(join(dir, "tmp")); if (mcpConfig) writeFileSync(join(dir, "agent", "mcp.json"), JSON.stringify(mcpConfig, null, 2) + "\n"); return dir; } -// The model's provider is the part of --model before the slash; its variables, such as -// ANTHROPIC_API_KEY for anthropic/..., are the only keys the agent gets besides Kernel's. +// pi's providers whose variables are not named after them, as in pi's getApiKeyEnvVars. +const PROVIDER_PREFIXES = { + "amazon-bedrock": ["AWS_"], "cloudflare-ai-gateway": ["CLOUDFLARE_"], "cloudflare-workers-ai": ["CLOUDFLARE_"], + "github-copilot": ["COPILOT_"], google: ["GEMINI_"], "google-vertex": ["GOOGLE_CLOUD_"], huggingface: ["HF_"], + "kimi-coding": ["KIMI_"], moonshotai: ["MOONSHOT_"], "moonshotai-cn": ["MOONSHOT_"], "opencode-go": ["OPENCODE_"], + "qwen-token-plan-individual": ["QWEN_TOKEN_PLAN_"], "vercel-ai-gateway": ["AI_GATEWAY_"], +}; + +// The model's provider is the part of --model before the slash. Its variables, such as +// ANTHROPIC_API_KEY for anthropic/... or GEMINI_API_KEY for google/..., are the only keys the agent +// gets besides Kernel's. +export function providerPrefixes(model) { + const provider = model.split("/")[0]; + return PROVIDER_PREFIXES[provider] ?? [`${provider.toUpperCase().replace(/-/g, "_")}_`]; +} + export function sandboxEnv(model, dir, env = process.env) { - const prefix = `${model.split("/")[0].toUpperCase().replace(/-/g, "_")}_`; - const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || name.startsWith(prefix)); + const prefixes = providerPrefixes(model); + const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || prefixes.some(prefix => name.startsWith(prefix))); return { ...Object.fromEntries(keys), PATH: env.PATH, @@ -64,6 +77,12 @@ export function sandboxEnv(model, dir, env = process.env) { }; } +// The agent can read its keys in /proc/self/environ, so the runner removes their values from its +// transcript. +export const secretValues = (env) => Object.entries(env) + .filter(([name, value]) => /KEY|TOKEN|SECRET/.test(name) && value.length >= 16) + .map(([, value]) => value); + // The paths to check a sandbox cannot see: the bench, and each entry of the home directory except // those that hold a mount, which bwrap creates as empty directories. export function hiddenPaths(benchDir, home, mounts) { diff --git a/bench/run.mjs b/bench/run.mjs index 29a1251d..8be09c7b 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -22,7 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs"; import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs"; import { checkRule, gradeAnswer } from "./grade.mjs"; -import { hiddenPaths, installMounts, sandboxArgs, sandboxDir, sandboxEnv, visiblePaths } from "./lib/sandbox.mjs"; +import { hiddenPaths, piLauncher, providerPrefixes, sandboxArgs, sandboxDir, sandboxEnv, secretValues, visiblePaths } from "./lib/sandbox.mjs"; const benchDir = dirname(fileURLToPath(import.meta.url)); if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env")); @@ -283,29 +283,31 @@ function trialArgs(plan, prompt) { } // Set by the smoke gate when the allowlist has a built-in tool: agents then run in a sandbox, with -// these read-only mounts besides /usr. Open sandbox directories are removed on shutdown. -let sandboxMounts = null; +// this pi command and its read-only mounts besides /usr. Open sandbox directories are removed on +// shutdown. +let sandbox = null; const sandboxDirs = new Set(); function checkSandbox() { - const mounts = installMounts(); + const launcher = piLauncher(); let visible; try { - visible = visiblePaths(hiddenPaths(benchDir, process.env.HOME, mounts), mounts); + visible = visiblePaths(hiddenPaths(benchDir, process.env.HOME, launcher.mounts), launcher.mounts); } catch (error) { fail(`${experiment.files.allowlist} has a built-in tool, so agents run in a bubblewrap sandbox, and ${error.message}; see "Sandbox" in bench/README.md`); } if (visible.length) fail(`the agent sandbox can see ${visible.join(", ")}; pi or Node is installed in a directory that holds it, so install them in a directory of their own`); - sandboxMounts = mounts; + sandbox = launcher; } // pi as the bench user, or in a sandbox that gets only the model's and Kernel's keys. async function runPi({ agentDir, mcpConfig, model, ...options }) { - if (!sandboxMounts) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); + if (!sandbox) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); const dir = sandboxDir(mcpConfig); sandboxDirs.add(dir); + const env = sandboxEnv(model, dir); try { - return await runAgent({ ...options, command: "bwrap", args: [...sandboxArgs(dir, sandboxMounts), "--", "pi", ...options.args], env: sandboxEnv(model, dir) }); + return await runAgent({ ...options, command: "bwrap", args: [...sandboxArgs(dir, sandbox.mounts), "--", ...sandbox.command, ...options.args], env, redact: secretValues(env) }); } finally { rmSync(dir, { recursive: true, force: true }); sandboxDirs.delete(dir); @@ -343,7 +345,7 @@ async function smokeGate(plan, probe, signedIn) { const { final, answer, cost_usd } = summarize(plain.events, []); logSmokeSpend(cost_usd); if (final?.stopReason !== "stop" || !/READY/.test(answer)) { - fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key.`); + fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key${sandbox ? `; in the sandbox the agent gets only KERNEL_API_KEY and ${providerPrefixes(plan.model).map(prefix => `${prefix}*`).join(", ")}` : ""}.`); } // pi prices a model missing from its table at 0, which would disable the spend cap. if (!(cost_usd > 0)) fail(`pi reported a cost of ${cost_usd} for ${plan.model}, so --max-budget-usd cannot apply`); @@ -448,9 +450,10 @@ async function shutdown(code, message) { stopping = true; (code ? console.error : log)(message); for (const child of runningAgents) child.kill("SIGKILL"); - for (const dir of sandboxDirs) rmSync(dir, { recursive: true, force: true }); await Promise.allSettled([...creating]); await Promise.allSettled([...openBrowsers].map(id => browsers.deleteBrowser(id))); + // After the browsers, so a directory a dying agent still writes to cannot stop their deletion. + for (const dir of sandboxDirs) rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); process.exit(code); } diff --git a/bench/test/agent.test.mjs b/bench/test/agent.test.mjs index cc9e2e01..e236fa66 100644 --- a/bench/test/agent.test.mjs +++ b/bench/test/agent.test.mjs @@ -188,6 +188,13 @@ test("the runner stops pi at the turn, spend and time caps, and when a cost is m assert.deepEqual([slow.limit, slow.durationMs], ["time cap", 300]); }); +test("the runner redacts the given strings from the transcript", async () => { + const key = "sk-ant-0123456789abcdef"; + const read = { type: "tool_execution_end", toolName: "read", result: { content: [{ type: "text", text: `ANTHROPIC_API_KEY=${key}\0PATH=/usr/bin` }] } }; + const { events } = await runEvents([read, { type: "agent_settled" }], { redact: [key] }); + assert.equal(events[0].result.content[0].text, "ANTHROPIC_API_KEY=[redacted]\0PATH=/usr/bin"); +}); + test("a failed invocation keeps its error, whether the call, the tool or the MCP server failed", () => { const invoke = (id) => call(id, "mcp__kernel__webmcp", { action: "invoke", tool_ref: "wmcp_a" }); const summary = summarize([ diff --git a/bench/test/sandbox.test.mjs b/bench/test/sandbox.test.mjs index 916b9d1d..0d0defb9 100644 --- a/bench/test/sandbox.test.mjs +++ b/bench/test/sandbox.test.mjs @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { delimiter, dirname, join } from "node:path"; import test from "node:test"; -import { hiddenPaths, installMounts, sandboxArgs, sandboxDir, sandboxEnv } from "../lib/sandbox.mjs"; +import { hiddenPaths, piLauncher, sandboxArgs, sandboxDir, sandboxEnv, secretValues } from "../lib/sandbox.mjs"; test("a sandboxed agent gets only Kernel's key, the model provider's variables and its own directories", () => { const env = sandboxEnv("anthropic/claude-haiku-5-5", "/tmp/xwebmcp-agent-1", { @@ -14,6 +14,40 @@ test("a sandboxed agent gets only Kernel's key, the model provider's variables a KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", PATH: "/usr/bin", HOME: "/tmp/xwebmcp-agent-1", TMPDIR: "/tmp/xwebmcp-agent-1/tmp", PI_CODING_AGENT_DIR: "/tmp/xwebmcp-agent-1/agent", PI_OFFLINE: "1", PI_TELEMETRY: "0", }); + // A provider whose key is not named after it. + const google = sandboxEnv("google/gemini-3-pro", "/tmp/xwebmcp-agent-1", { GEMINI_API_KEY: "g", GOOGLE_APPLICATION_CREDENTIALS: "/home/me/key.json" }); + assert.equal(google.GEMINI_API_KEY, "g"); + assert.ok(!("GOOGLE_APPLICATION_CREDENTIALS" in google)); +}); + +test("only long values of keys, tokens and secrets are redacted", () => { + const key = "sk-ant-0123456789abcdef"; + assert.deepEqual(secretValues({ ANTHROPIC_API_KEY: key, ANTHROPIC_BASE_URL: "https://api.anthropic.com/v1", HF_TOKEN: "short" }), [key]); +}); + +// A pi installed under a prefix that holds other files, as with an npm prefix of ~/.local. +function installPi(script) { + const prefix = mkdtempSync(join(tmpdir(), "bench-prefix-")); + const pkg = join(prefix, "lib/node_modules/@scope/pi"); + mkdirSync(join(pkg, "dist"), { recursive: true }); + writeFileSync(join(pkg, "dist/cli.js"), script); + mkdirSync(join(prefix, "bin")); + symlinkSync(join(pkg, "dist/cli.js"), join(prefix, "bin/pi")); + mkdirSync(join(prefix, "share")); + writeFileSync(join(prefix, "share/secret.txt"), "secret"); + return { prefix: realpathSync(prefix), env: { PATH: [join(prefix, "bin"), dirname(process.execPath)].join(delimiter) } }; +} + +test("the sandbox mounts pi's package and Node's binary, not their prefixes", () => { + const { prefix, env } = installPi(""); + try { + const node = realpathSync(process.execPath); + const { mounts, command } = piLauncher(env); + assert.deepEqual(mounts, [node, join(prefix, "lib/node_modules/@scope/pi")].filter(path => !path.startsWith("/usr/"))); + assert.deepEqual(command, [node, join(prefix, "lib/node_modules/@scope/pi/dist/cli.js")]); + } finally { + rmSync(prefix, { recursive: true, force: true }); + } }); test("the sandbox binds its own directory over an empty /tmp and nothing from home", () => { @@ -38,24 +72,28 @@ const bwrapWorks = spawnSync("bwrap", [...sandboxArgs(tmpdir()), "--", "true"]). test("in a real sandbox the agent sees its directory, not other files or processes", { skip: !bwrapWorks && "bwrap cannot create a sandbox here" }, () => { const other = mkdtempSync(join(tmpdir(), "bench-other-")); writeFileSync(join(other, "secret.txt"), "secret"); - const dir = sandboxDir({ mcpServers: { kernel: {} } }); - try { - const script = `const fs = require("fs"); + const { prefix, env } = installPi(`const fs = require("fs"); fs.writeFileSync(process.env.TMPDIR + "/pi-mcp-1.txt", "full output"); console.log(JSON.stringify({ secret: fs.existsSync(${JSON.stringify(join(other, "secret.txt"))}), etc: fs.existsSync("/etc/hostname"), + prefix: fs.existsSync(require("path").join(__dirname, "../../../../../share/secret.txt")), mcp: JSON.parse(fs.readFileSync(process.env.PI_CODING_AGENT_DIR + "/mcp.json", "utf8")), processes: fs.readdirSync("/proc").filter(name => /^\\d+$/.test(name)).length, cwd: process.cwd(), -}));`; - const result = spawnSync("bwrap", [...sandboxArgs(dir, installMounts()), "--", process.execPath, "-e", script], { encoding: "utf8", env: sandboxEnv("openai/gpt", dir, { PATH: process.env.PATH }) }); +}));`); + const dir = sandboxDir({ mcpServers: { kernel: {} } }); + try { + const { mounts, command } = piLauncher(env); + const result = spawnSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...command], { encoding: "utf8", env: sandboxEnv("openai/gpt", dir, env) }); assert.equal(result.status, 0, result.stderr); const seen = JSON.parse(result.stdout); - assert.deepEqual({ ...seen, processes: seen.processes <= 3 }, { secret: false, etc: false, mcp: { mcpServers: { kernel: {} } }, processes: true, cwd: dir }); + assert.deepEqual({ ...seen, processes: seen.processes <= 3 }, { secret: false, etc: false, prefix: false, mcp: { mcpServers: { kernel: {} } }, processes: true, cwd: dir }); + assert.match(dir, /^\/tmp\/xwebmcp-agent-/); assert.equal(readFileSync(join(dir, "tmp/pi-mcp-1.txt"), "utf8"), "full output"); } finally { rmSync(dir, { recursive: true, force: true }); rmSync(other, { recursive: true, force: true }); + rmSync(prefix, { recursive: true, force: true }); } }); From 695a569e61e59ebdcfa250515d97760e9f6dcbff Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:21:00 +0000 Subject: [PATCH 4/4] bench: keep split keys out of transcripts, check pi starts in the sandbox, pass proxy variables Drop pi's message_update events when redacting, since a key can be split across streamed deltas. Run pi --version in a sandbox in the smoke gate, so a standalone binary or a version manager's shim fails there with its error. Pass the proxy variables and NODE_EXTRA_CA_CERTS to the agent, and redact proxy URLs. --- bench/README.md | 18 +++++++++------ bench/lib/agent.mjs | 4 +++- bench/lib/sandbox.mjs | 44 ++++++++++++++++++++++++++++--------- bench/run.mjs | 13 ++++++++--- bench/test/agent.test.mjs | 10 +++++++-- bench/test/run.test.mjs | 10 +++++++++ bench/test/sandbox.test.mjs | 26 +++++++++++++++++++--- 7 files changed, 99 insertions(+), 26 deletions(-) diff --git a/bench/README.md b/bench/README.md index 938dfbb0..e35fb7f5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -49,12 +49,15 @@ files, agents on Kernel browsers run in a [bubblewrap](https://github.com/contai sandbox (`bwrap`, Linux only; `apt install bubblewrap`). It sees `/usr`, the `/etc` files DNS and TLS need, its own processes and a fresh directory with its pi config and `TMPDIR`; not your home, this checkout, other trials or the rest of `/tmp` and `/etc`. It shares the network. The agent gets -only `KERNEL_API_KEY` and the variables of its model's provider (`ANTHROPIC_*` for -`anthropic/...`, `GEMINI_*` for `google/...`); their values are redacted from transcripts. If Node -or pi is installed outside `/usr`, such as under `~/.nvm` or `~/.local`, only Node's binary and -pi's package are mounted, read-only. Unprivileged user namespaces must be allowed, which some -distributions restrict with AppArmor; the smoke gate stops the run if `bwrap` cannot create a -sandbox or if one can see this checkout or your home. +only `KERNEL_API_KEY`, the variables of its model's provider (`ANTHROPIC_*` for `anthropic/...`, +`GEMINI_*` for `google/...`), the proxy variables and `NODE_EXTRA_CA_CERTS`; the values of keys +and proxy URLs are redacted from transcripts, which then leave out pi's streamed `message_update` +events. If Node or pi is installed outside `/usr`, such as under `~/.nvm` or `~/.local`, only Node's +binary and pi's package are mounted, read-only, and pi runs as `node `, so it must be +an npm install rather than a standalone binary or a version manager's shim. Unprivileged user +namespaces must be allowed, which some distributions restrict with AppArmor; the smoke gate stops +the run if `bwrap` cannot create a sandbox, if one can see this checkout or your home, or if pi +does not start in one. Run it as a Hypeship service under a hold that covers the whole run; holds cannot be renewed. This pilot is 105 trials and can take about 8 hours if every trial reaches its cap: @@ -123,7 +126,8 @@ Before any trial, stopping on the first failure: 1. the hold covers the remaining jobs if every trial reaches its cap after 180s of setup (the hold is checked again before each job, and no job starts that it cannot outlast); 2. `KERNEL_API_KEY` is set (Kernel browsers), and on resume pi is the same version; - with Kernel browsers, a sandbox works and can see neither this checkout nor your home; + with Kernel browsers, a sandbox works, can see neither this checkout nor your home, and runs the + same pi; 3. a zero-tool pi run answers and reports a cost above zero (pi prices an unknown model at 0, which would disable the spend cap); 4. a treatment setup on the first pending intent works end to end; diff --git a/bench/lib/agent.mjs b/bench/lib/agent.mjs index 89fa0a6a..b099e750 100644 --- a/bench/lib/agent.mjs +++ b/bench/lib/agent.mjs @@ -32,7 +32,8 @@ export function parseRecord(line) { // The clock runs from launch to agent_settled. pi has no turn or spend cap, so the runner // enforces both; a missing cost stops the agent, since the spend cap needs it. Each string in -// `redact` is replaced in the transcript. +// `redact` is replaced in the transcript, which then leaves out message_update: its streamed deltas +// can split a string, and message_end holds the whole message. export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, redact = [], transcriptPath, stderrPath }) { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); @@ -63,6 +64,7 @@ export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Inf const onRecord = (line) => { if (!line.trim()) return; const event = parseRecord(redact.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), line)); + if (redact.length && event.type === "message_update") return; event._t = elapsed(); events.push(event); transcript.write(JSON.stringify(event) + "\n"); diff --git a/bench/lib/sandbox.mjs b/bench/lib/sandbox.mjs index aa0574b2..d301df09 100644 --- a/bench/lib/sandbox.mjs +++ b/bench/lib/sandbox.mjs @@ -63,9 +63,12 @@ export function providerPrefixes(model) { return PROVIDER_PREFIXES[provider] ?? [`${provider.toUpperCase().replace(/-/g, "_")}_`]; } +// pi's HTTP client reads the proxy variables, and Node the extra CA file an intercepting proxy needs. +const NETWORK = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy", "NODE_EXTRA_CA_CERTS"]; + export function sandboxEnv(model, dir, env = process.env) { const prefixes = providerPrefixes(model); - const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || prefixes.some(prefix => name.startsWith(prefix))); + const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || NETWORK.includes(name) || prefixes.some(prefix => name.startsWith(prefix))); return { ...Object.fromEntries(keys), PATH: env.PATH, @@ -78,9 +81,9 @@ export function sandboxEnv(model, dir, env = process.env) { } // The agent can read its keys in /proc/self/environ, so the runner removes their values from its -// transcript. +// transcript. A proxy URL can hold a password. export const secretValues = (env) => Object.entries(env) - .filter(([name, value]) => /KEY|TOKEN|SECRET/.test(name) && value.length >= 16) + .filter(([name, value]) => /KEY|TOKEN|SECRET|^https?_proxy$/i.test(name) && value.length >= 16) .map(([, value]) => value); // The paths to check a sandbox cannot see: the bench, and each entry of the home directory except @@ -90,16 +93,37 @@ export function hiddenPaths(benchDir, home, mounts) { return [benchDir, ...entries.filter(path => !mounts.some(mount => under(mount, path)))]; } -// Throws when bwrap cannot create a sandbox; returns the paths a sandbox can see. -export function visiblePaths(paths, mounts) { +function runSandboxed(command, mounts) { const dir = sandboxDir(null); try { - const probe = ["sh", "-c", 'for p; do [ -e "$p" ] && echo "$p"; done; true', "sh", ...paths]; - const output = execFileSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...probe], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); - return output.split("\n").filter(Boolean); - } catch (error) { - throw new Error(`bwrap cannot create a sandbox: ${error.stderr?.toString().trim() || error.message}`); + return execFileSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...command], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); } finally { rmSync(dir, { recursive: true, force: true }); } } + +// The error line of a failed command's stderr, such as Node's SyntaxError, or else its last line. +const stderrOf = (error) => { + const lines = (error.stderr?.toString() ?? "").trim().split("\n").filter(Boolean); + return lines.find(line => /^\w*Error\b/.test(line)) ?? lines.at(-1) ?? error.message; +}; + +// Throws when bwrap cannot create a sandbox; returns the paths a sandbox can see. +export function visiblePaths(paths, mounts) { + const probe = ["sh", "-c", 'for p; do [ -e "$p" ] && echo "$p"; done; true', "sh", ...paths]; + try { + return runSandboxed(probe, mounts).split("\n").filter(Boolean); + } catch (error) { + throw new Error(`bwrap cannot create a sandbox: ${stderrOf(error)}`); + } +} + +// pi's --version as the sandbox runs it. Throws when it does not start there, as when `pi` is a +// standalone binary or a version manager's shim rather than Node's script. +export function sandboxedPiVersion({ command, mounts }) { + try { + return runSandboxed([...command, "--version"], mounts).trim(); + } catch (error) { + throw new Error(`${command.join(" ")} --version failed in the sandbox: ${stderrOf(error)}`); + } +} diff --git a/bench/run.mjs b/bench/run.mjs index 8be09c7b..9c5d273a 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -22,7 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs"; import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs"; import { checkRule, gradeAnswer } from "./grade.mjs"; -import { hiddenPaths, piLauncher, providerPrefixes, sandboxArgs, sandboxDir, sandboxEnv, secretValues, visiblePaths } from "./lib/sandbox.mjs"; +import { hiddenPaths, piLauncher, providerPrefixes, sandboxArgs, sandboxDir, sandboxedPiVersion, sandboxEnv, secretValues, visiblePaths } from "./lib/sandbox.mjs"; const benchDir = dirname(fileURLToPath(import.meta.url)); if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env")); @@ -288,7 +288,7 @@ function trialArgs(plan, prompt) { let sandbox = null; const sandboxDirs = new Set(); -function checkSandbox() { +function checkSandbox(agent) { const launcher = piLauncher(); let visible; try { @@ -297,6 +297,13 @@ function checkSandbox() { fail(`${experiment.files.allowlist} has a built-in tool, so agents run in a bubblewrap sandbox, and ${error.message}; see "Sandbox" in bench/README.md`); } if (visible.length) fail(`the agent sandbox can see ${visible.join(", ")}; pi or Node is installed in a directory that holds it, so install them in a directory of their own`); + let version; + try { + version = `pi ${sandboxedPiVersion(launcher)}`; + } catch (error) { + fail(`the agent sandbox runs pi with Node, and ${error.message}; use an npm install of pi, with no version manager's shim for pi or Node on PATH`); + } + if (version !== agent) fail(`the agent sandbox runs ${version}, but \`pi --version\` is ${agent}`); sandbox = launcher; } @@ -334,7 +341,7 @@ async function smokeGate(plan, probe, signedIn) { plan.kernel_cli ??= kernelCli; } - if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkSandbox(); + if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkSandbox(agent); const plain = await runPi({ args: piArgs({ prompt: "Reply with the single word READY.", systemPrompt: "Answer briefly.", model: plan.model, thinking: plan.thinking }), diff --git a/bench/test/agent.test.mjs b/bench/test/agent.test.mjs index e236fa66..9229e733 100644 --- a/bench/test/agent.test.mjs +++ b/bench/test/agent.test.mjs @@ -188,11 +188,17 @@ test("the runner stops pi at the turn, spend and time caps, and when a cost is m assert.deepEqual([slow.limit, slow.durationMs], ["time cap", 300]); }); -test("the runner redacts the given strings from the transcript", async () => { +test("the runner redacts the given strings from the transcript, including one split across streamed deltas", async () => { const key = "sk-ant-0123456789abcdef"; const read = { type: "tool_execution_end", toolName: "read", result: { content: [{ type: "text", text: `ANTHROPIC_API_KEY=${key}\0PATH=/usr/bin` }] } }; - const { events } = await runEvents([read, { type: "agent_settled" }], { redact: [key] }); + const delta = (text) => ({ type: "message_update", assistantMessageEvent: { type: "text_delta", contentIndex: 0, delta: text } }); + const echo = { type: "message_end", message: { role: "assistant", content: [{ type: "text", text: `It is ${key}` }], usage: { cost: { total: 0.01 } } } }; + const { events } = await runEvents([read, delta(`It is ${key.slice(0, 10)}`), delta(key.slice(10)), echo, { type: "agent_settled" }], { redact: [key] }); assert.equal(events[0].result.content[0].text, "ANTHROPIC_API_KEY=[redacted]\0PATH=/usr/bin"); + assert.deepEqual(events.map(event => event.type), ["tool_execution_end", "message_end", "agent_settled"]); + assert.equal(events[1].message.content[0].text, "It is [redacted]"); + // Without strings to redact, the transcript keeps message_update. + assert.equal((await runEvents([delta("hi"), { type: "agent_settled" }])).events[0].type, "message_update"); }); test("a failed invocation keeps its error, whether the call, the tool or the MCP server failed", () => { diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs index d27ed9b6..de112d46 100644 --- a/bench/test/run.test.mjs +++ b/bench/test/run.test.mjs @@ -255,3 +255,13 @@ test("a run stops before any agent when the sandbox can see the bench", { timeou assert.notEqual(result.status, 0); assert.match(result.stderr, new RegExp(`the agent sandbox can see ${bench}`)); }); + +test("a run stops before any browser when pi is a shim the sandbox cannot run with Node", { timeout: 120_000 }, () => { + const { run, log } = workspace(); + const shim = mkdtempSync(join(tmpdir(), "bench-shim-")); + writeFileSync(join(shim, "pi"), `#!/bin/sh\nexec ${process.execPath} ${join(benchDir, "test/stubs/pi")} "$@"\n`, { mode: 0o755 }); + const result = run("shimmed", { PATH: [shim, join(benchDir, "test/stubs"), dirname(process.execPath), process.env.PATH].join(delimiter) }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /the agent sandbox runs pi with Node, and .* --version failed in the sandbox/); + assert.deepEqual(log().filter(entry => entry[0] === "create"), []); +}); diff --git a/bench/test/sandbox.test.mjs b/bench/test/sandbox.test.mjs index 0d0defb9..d95488ea 100644 --- a/bench/test/sandbox.test.mjs +++ b/bench/test/sandbox.test.mjs @@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync import { tmpdir } from "node:os"; import { delimiter, dirname, join } from "node:path"; import test from "node:test"; -import { hiddenPaths, piLauncher, sandboxArgs, sandboxDir, sandboxEnv, secretValues } from "../lib/sandbox.mjs"; +import { hiddenPaths, piLauncher, sandboxArgs, sandboxDir, sandboxedPiVersion, sandboxEnv, secretValues } from "../lib/sandbox.mjs"; test("a sandboxed agent gets only Kernel's key, the model provider's variables and its own directories", () => { const env = sandboxEnv("anthropic/claude-haiku-5-5", "/tmp/xwebmcp-agent-1", { @@ -18,11 +18,17 @@ test("a sandboxed agent gets only Kernel's key, the model provider's variables a const google = sandboxEnv("google/gemini-3-pro", "/tmp/xwebmcp-agent-1", { GEMINI_API_KEY: "g", GOOGLE_APPLICATION_CREDENTIALS: "/home/me/key.json" }); assert.equal(google.GEMINI_API_KEY, "g"); assert.ok(!("GOOGLE_APPLICATION_CREDENTIALS" in google)); + // The proxy variables pi's HTTP client reads. + const proxied = sandboxEnv("openai/gpt", "/tmp/xwebmcp-agent-1", { HTTPS_PROXY: "http://proxy:3128", no_proxy: "localhost", NODE_EXTRA_CA_CERTS: "/etc/ssl/proxy.pem" }); + assert.deepEqual([proxied.HTTPS_PROXY, proxied.no_proxy, proxied.NODE_EXTRA_CA_CERTS], ["http://proxy:3128", "localhost", "/etc/ssl/proxy.pem"]); }); -test("only long values of keys, tokens and secrets are redacted", () => { +test("only long values of keys, tokens, secrets and proxy URLs are redacted", () => { const key = "sk-ant-0123456789abcdef"; - assert.deepEqual(secretValues({ ANTHROPIC_API_KEY: key, ANTHROPIC_BASE_URL: "https://api.anthropic.com/v1", HF_TOKEN: "short" }), [key]); + const proxy = "http://user:pass@proxy:3128"; + assert.deepEqual(secretValues({ + ANTHROPIC_API_KEY: key, ANTHROPIC_BASE_URL: "https://api.anthropic.com/v1", HF_TOKEN: "short", https_proxy: proxy, NO_PROXY: "localhost,127.0.0.1", + }), [key, proxy]); }); // A pi installed under a prefix that holds other files, as with an npm prefix of ~/.local. @@ -97,3 +103,17 @@ console.log(JSON.stringify({ rmSync(prefix, { recursive: true, force: true }); } }); + +test("a sandbox reports pi's version, and fails when pi is a shim rather than Node's script", { skip: !bwrapWorks && "bwrap cannot create a sandbox here" }, () => { + const { prefix, env } = installPi(`if (process.argv[2] === "--version") console.log("1.2.3");`); + const shim = mkdtempSync(join(tmpdir(), "bench-shim-")); + writeFileSync(join(shim, "pi"), `#!/bin/sh\nexec ${join(prefix, "bin/pi")} "$@"\n`, { mode: 0o755 }); + try { + assert.equal(sandboxedPiVersion(piLauncher(env)), "1.2.3"); + const shimmed = piLauncher({ PATH: [shim, env.PATH].join(delimiter) }); + assert.throws(() => sandboxedPiVersion(shimmed), /--version failed in the sandbox: /); + } finally { + rmSync(prefix, { recursive: true, force: true }); + rmSync(shim, { recursive: true, force: true }); + } +});