bench: give agents pi's read tool, in a bubblewrap sandbox

bench/README.md

@@ -28,7 +28,7 @@ bench/
   tasks/<domain>/<intent>.json    one intent per file; domain.json holds per-domain settings
   schemas/                        JSON Schemas for answers
   prompts/system.md               the system prompt both arms get
-  allowlist.txt                   the kernel MCP tools the agent gets, and no others
+  allowlist.txt                   the tools the agent gets, and no others: the kernel MCP tools and pi's read
   mcp.json                        the kernel MCP server; reads KERNEL_API_KEY
   .env.example                    keys run.mjs loads from bench/.env
   run.mjs  grade.mjs  report.mjs  lib/  test/
@@ -41,6 +41,24 @@ Needs the `kernel` and `pi` CLIs, Node 22.13+, git, `KERNEL_API_KEY` and the mod
 key (such as `OPENAI_API_KEY`), from the environment or `bench/.env`. Each agent gets its own
 `PI_CODING_AGENT_DIR`, so stored pi logins and MCP servers are not used.
 
+### Sandbox
+
+pi cuts an MCP result over 20KB, such as a long WebMCP tool list, and saves the full text to a
+file, so the agent gets pi's `read` tool to open it. So that `read` cannot reach the bench's own
+files, agents on Kernel browsers run in a [bubblewrap](https://github.com/containers/bubblewrap)
+sandbox (`bwrap`, Linux only; `apt install bubblewrap`). It sees `/usr`, the `/etc` files DNS and
+TLS need, its own processes and a fresh directory with its pi config and `TMPDIR`; not your home,
+this checkout, other trials or the rest of `/tmp` and `/etc`. It shares the network. The agent gets
+only `KERNEL_API_KEY`, the variables of its model's provider (`ANTHROPIC_*` for `anthropic/...`,
+`GEMINI_*` for `google/...`), the proxy variables and `NODE_EXTRA_CA_CERTS`; the values of keys
+and proxy URLs are redacted from transcripts, which then leave out pi's streamed `message_update`
+events. If Node or pi is installed outside `/usr`, such as under `~/.nvm` or `~/.local`, only Node's
+binary and pi's package are mounted, read-only, and pi runs as `node <its cli.js>`, so it must be
+an npm install rather than a standalone binary or a version manager's shim. Unprivileged user
+namespaces must be allowed, which some distributions restrict with AppArmor; the smoke gate stops
+the run if `bwrap` cannot create a sandbox, if one can see this checkout or your home, or if pi
+does not start in one.
+
 Run it as a Hypeship service under a hold that covers the whole run; holds cannot be renewed.
 This pilot is 105 trials and can take about 8 hours if every trial reaches its cap:
 
@@ -108,6 +126,8 @@ Before any trial, stopping on the first failure:
 1. the hold covers the remaining jobs if every trial reaches its cap after 180s of setup (the
    hold is checked again before each job, and no job starts that it cannot outlast);
 2. `KERNEL_API_KEY` is set (Kernel browsers), and on resume pi is the same version;
+   with Kernel browsers, a sandbox works, can see neither this checkout nor your home, and runs the
+   same pi;
 3. a zero-tool pi run answers and reports a cost above zero (pi prices an unknown model at 0,
    which would disable the spend cap);
 4. a treatment setup on the first pending intent works end to end;

bench/allowlist.txt

@@ -2,3 +2,4 @@ mcp__kernel__browser_repl
 mcp__kernel__computer_action
 mcp__kernel__execute_playwright_code
 mcp__kernel__webmcp
+read

bench/lib/agent.mjs

@@ -31,10 +31,12 @@ export function parseRecord(line) {
 }
 
 // The clock runs from launch to agent_settled. pi has no turn or spend cap, so the runner
-// enforces both; a missing cost stops the agent, since the spend cap needs it.
-export function runAgent({ args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, transcriptPath, stderrPath }) {
+// enforces both; a missing cost stops the agent, since the spend cap needs it. Each string in
+// `redact` is replaced in the transcript, which then leaves out message_update: its streamed deltas
+// can split a string, and message_end holds the whole message.
+export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, redact = [], transcriptPath, stderrPath }) {
   return new Promise((resolve, reject) => {
-    const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] });
+    const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] });
     runningAgents.add(child);
     const start = performance.now();
     const elapsed = () => Math.round(performance.now() - start);
@@ -61,7 +63,8 @@ export function runAgent({ args, env, cwd, capMs, maxTurns = Infinity, maxBudget
 
     const onRecord = (line) => {
       if (!line.trim()) return;
-      const event = parseRecord(line);
+      const event = parseRecord(redact.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), line));
+      if (redact.length && event.type === "message_update") return;
       event._t = elapsed();
       events.push(event);
       transcript.write(JSON.stringify(event) + "\n");

bench/lib/sandbox.mjs

@@ -0,0 +1,129 @@
+// Agents with a built-in tool such as `read` run in a bubblewrap sandbox, so they cannot read the
+// bench's files: intents and their rules, results, other trials, the home directory or keys. The
+// sandbox shows /usr, the /etc files DNS and TLS need, the agent's own processes, and a fresh
+// directory with its pi config and TMPDIR, where pi saves the full text of a long MCP result.
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { delimiter, join } from "node:path";
+
+const ETC = [
+  "/etc/resolv.conf", "/etc/hosts", "/etc/nsswitch.conf", "/etc/gai.conf", "/etc/ssl", "/etc/pki",
+  "/etc/ca-certificates", "/etc/passwd", "/etc/group", "/etc/localtime", "/etc/alternatives",
+];
+
+const under = (path, dir) => path === dir || path.startsWith(`${dir}/`);
+
+const which = (name, env) => (env.PATH ?? "").split(delimiter).map(dir => join(dir, name)).find(path => existsSync(path));
+
+// How a sandbox runs pi: with the host's node, mounting Node's binary and pi's package by themselves
+// when they are outside /usr, so the rest of a prefix such as ~/.local or ~/.nvm stays hidden.
+export function piLauncher(env = process.env, execPath = process.execPath) {
+  const node = realpathSync(which("node", env) ?? execPath);
+  const cli = realpathSync(which("pi", env));
+  const pkg = cli.match(/^(.*\/node_modules\/(?:@[^/]+\/)?[^/]+)\//)?.[1] ?? cli;
+  return { mounts: [...new Set([node, pkg])].filter(path => !under(path, "/usr")), command: [node, cli] };
+}
+
+export function sandboxArgs(dir, mounts = []) {
+  return [
+    "--unshare-all", "--share-net", "--die-with-parent", "--new-session",
+    "--ro-bind", "/usr", "/usr",
+    ...["/bin", "/sbin", "/lib", "/lib64"].flatMap(path => ["--ro-bind-try", path, path]),
+    ...ETC.flatMap(path => ["--ro-bind-try", path, path]),
+    "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp",
+    ...mounts.flatMap(path => ["--ro-bind", path, path]),
+    "--bind", dir, dir, "--chdir", dir,
+  ];
+}
+
+// A fresh directory for one agent, with its mcp.json. It is under /tmp, not the host's TMPDIR,
+// which may be in the home directory and would then show the sandbox an entry of it.
+export function sandboxDir(mcpConfig) {
+  const dir = mkdtempSync("/tmp/xwebmcp-agent-");
+  mkdirSync(join(dir, "agent"));
+  mkdirSync(join(dir, "tmp"));
+  if (mcpConfig) writeFileSync(join(dir, "agent", "mcp.json"), JSON.stringify(mcpConfig, null, 2) + "\n");
+  return dir;
+}
+
+// pi's providers whose variables are not named after them, as in pi's getApiKeyEnvVars.
+const PROVIDER_PREFIXES = {
+  "amazon-bedrock": ["AWS_"], "cloudflare-ai-gateway": ["CLOUDFLARE_"], "cloudflare-workers-ai": ["CLOUDFLARE_"],
+  "github-copilot": ["COPILOT_"], google: ["GEMINI_"], "google-vertex": ["GOOGLE_CLOUD_"], huggingface: ["HF_"],
+  "kimi-coding": ["KIMI_"], moonshotai: ["MOONSHOT_"], "moonshotai-cn": ["MOONSHOT_"], "opencode-go": ["OPENCODE_"],
+  "qwen-token-plan-individual": ["QWEN_TOKEN_PLAN_"], "vercel-ai-gateway": ["AI_GATEWAY_"],
+};
+
+// The model's provider is the part of --model before the slash. Its variables, such as
+// ANTHROPIC_API_KEY for anthropic/... or GEMINI_API_KEY for google/..., are the only keys the agent
+// gets besides Kernel's.
+export function providerPrefixes(model) {
+  const provider = model.split("/")[0];
+  return PROVIDER_PREFIXES[provider] ?? [`${provider.toUpperCase().replace(/-/g, "_")}_`];
+}
+
+// pi's HTTP client reads the proxy variables, and Node the extra CA file an intercepting proxy needs.
+const NETWORK = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy", "NODE_EXTRA_CA_CERTS"];
+
+export function sandboxEnv(model, dir, env = process.env) {
+  const prefixes = providerPrefixes(model);
+  const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || NETWORK.includes(name) || prefixes.some(prefix => name.startsWith(prefix)));
+  return {
+    ...Object.fromEntries(keys),
+    PATH: env.PATH,
+    HOME: dir,
+    TMPDIR: join(dir, "tmp"),
+    PI_CODING_AGENT_DIR: join(dir, "agent"),
+    PI_OFFLINE: "1",
+    PI_TELEMETRY: "0",
+  };
+}
+
+// The agent can read its keys in /proc/self/environ, so the runner removes their values from its
+// transcript. A proxy URL can hold a password.
+export const secretValues = (env) => Object.entries(env)
+  .filter(([name, value]) => /KEY|TOKEN|SECRET|^https?_proxy$/i.test(name) && value.length >= 16)
+  .map(([, value]) => value);
+
+// The paths to check a sandbox cannot see: the bench, and each entry of the home directory except
+// those that hold a mount, which bwrap creates as empty directories.
+export function hiddenPaths(benchDir, home, mounts) {
+  const entries = home ? readdirSync(home).map(name => join(home, name)) : [];
+  return [benchDir, ...entries.filter(path => !mounts.some(mount => under(mount, path)))];
+}
+
+function runSandboxed(command, mounts) {
+  const dir = sandboxDir(null);
+  try {
+    return execFileSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...command], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+// The error line of a failed command's stderr, such as Node's SyntaxError, or else its last line.
+const stderrOf = (error) => {
+  const lines = (error.stderr?.toString() ?? "").trim().split("\n").filter(Boolean);
+  return lines.find(line => /^\w*Error\b/.test(line)) ?? lines.at(-1) ?? error.message;
+};
+
+// Throws when bwrap cannot create a sandbox; returns the paths a sandbox can see.
+export function visiblePaths(paths, mounts) {
+  const probe = ["sh", "-c", 'for p; do [ -e "$p" ] && echo "$p"; done; true', "sh", ...paths];
+  try {
+    return runSandboxed(probe, mounts).split("\n").filter(Boolean);
+  } catch (error) {
+    throw new Error(`bwrap cannot create a sandbox: ${stderrOf(error)}`);
+  }
+}
+
+// pi's --version as the sandbox runs it. Throws when it does not start there, as when `pi` is a
+// standalone binary or a version manager's shim rather than Node's script.
+export function sandboxedPiVersion({ command, mounts }) {
+  try {
+    return runSandboxed([...command, "--version"], mounts).trim();
+  } catch (error) {
+    throw new Error(`${command.join(" ")} --version failed in the sandbox: ${stderrOf(error)}`);
+  }
+}

bench/run.mjs

@@ -22,6 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo
 import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs";
 import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs";
 import { checkRule, gradeAnswer } from "./grade.mjs";
+import { hiddenPaths, piLauncher, providerPrefixes, sandboxArgs, sandboxDir, sandboxedPiVersion, sandboxEnv, secretValues, visiblePaths } from "./lib/sandbox.mjs";
 
 const benchDir = dirname(fileURLToPath(import.meta.url));
 if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env"));
@@ -281,6 +282,45 @@ function trialArgs(plan, prompt) {
   return piArgs({ prompt, systemPrompt: experiment.system_prompt, allow: plan.allowlist, model: plan.model, thinking: plan.thinking });
 }
 
+// Set by the smoke gate when the allowlist has a built-in tool: agents then run in a sandbox, with
+// this pi command and its read-only mounts besides /usr. Open sandbox directories are removed on
+// shutdown.
+let sandbox = null;
+const sandboxDirs = new Set();
+
+function checkSandbox(agent) {
+  const launcher = piLauncher();
+  let visible;
+  try {
+    visible = visiblePaths(hiddenPaths(benchDir, process.env.HOME, launcher.mounts), launcher.mounts);
+  } catch (error) {
+    fail(`${experiment.files.allowlist} has a built-in tool, so agents run in a bubblewrap sandbox, and ${error.message}; see "Sandbox" in bench/README.md`);
+  }
+  if (visible.length) fail(`the agent sandbox can see ${visible.join(", ")}; pi or Node is installed in a directory that holds it, so install them in a directory of their own`);
+  let version;
+  try {
+    version = `pi ${sandboxedPiVersion(launcher)}`;
+  } catch (error) {
+    fail(`the agent sandbox runs pi with Node, and ${error.message}; use an npm install of pi, with no version manager's shim for pi or Node on PATH`);
+  }
+  if (version !== agent) fail(`the agent sandbox runs ${version}, but \`pi --version\` is ${agent}`);
+  sandbox = launcher;
+}
+
+// pi as the bench user, or in a sandbox that gets only the model's and Kernel's keys.
+async function runPi({ agentDir, mcpConfig, model, ...options }) {
+  if (!sandbox) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) });
+  const dir = sandboxDir(mcpConfig);
+  sandboxDirs.add(dir);
+  const env = sandboxEnv(model, dir);
+  try {
+    return await runAgent({ ...options, command: "bwrap", args: [...sandboxArgs(dir, sandbox.mounts), "--", ...sandbox.command, ...options.args], env, redact: secretValues(env) });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+    sandboxDirs.delete(dir);
+  }
+}
+
 const stderrTail = (file) => (existsSync(file) ? readFileSync(file, "utf8") : "").trim().split("\n").slice(-3).join(" ");
 
 function logSmokeSpend(cost_usd) {
@@ -301,16 +341,18 @@ async function smokeGate(plan, probe, signedIn) {
     plan.kernel_cli ??= kernelCli;
   }
 
-  const plain = await runAgent({
+  if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkSandbox(agent);
+
+  const plain = await runPi({
     args: piArgs({ prompt: "Reply with the single word READY.", systemPrompt: "Answer briefly.", model: plan.model, thinking: plan.thinking }),
-    env: agentEnv(join(dir, "plain-agent"), null),
+    agentDir: join(dir, "plain-agent"), mcpConfig: null, model: plan.model,
     cwd: dir, capMs: 120_000, transcriptPath: join(dir, "plain.jsonl"), stderrPath: join(dir, "plain.stderr"),
   });
   if (stopping) return;
   const { final, answer, cost_usd } = summarize(plain.events, []);
   logSmokeSpend(cost_usd);
   if (final?.stopReason !== "stop" || !/READY/.test(answer)) {
-    fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key.`);
+    fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key${sandbox ? `; in the sandbox the agent gets only KERNEL_API_KEY and ${providerPrefixes(plan.model).map(prefix => `${prefix}*`).join(", ")}` : ""}.`);
   }
   // pi prices a model missing from its table at 0, which would disable the spend cap.
   if (!(cost_usd > 0)) fail(`pi reported a cost of ${cost_usd} for ${plan.model}, so --max-budget-usd cannot apply`);
@@ -321,9 +363,9 @@ async function smokeGate(plan, probe, signedIn) {
   const setup = await setUpBrowser(plan, probe, probeArm, "smoke");
   if (setup.stopped) return;
   if (setup.error) fail(`${probeArm} setup failed for ${probe.intent.id} after ${setup.attempts} browsers: ${setup.error}`);
-  const configured = await runAgent({
+  const configured = await runPi({
     args: trialArgs(plan, "Reply with the single word READY. Do not call any tools."),
-    env: agentEnv(join(dir, "configured-agent"), mcpFor(plan, setup.browser)),
+    agentDir: join(dir, "configured-agent"), mcpConfig: mcpFor(plan, setup.browser), model: plan.model,
     cwd: dir, capMs: 120_000, transcriptPath: join(dir, "configured.jsonl"), stderrPath: join(dir, "configured.stderr"),
   });
   await releaseBrowser(setup.browser);
@@ -417,6 +459,8 @@ async function shutdown(code, message) {
   for (const child of runningAgents) child.kill("SIGKILL");
   await Promise.allSettled([...creating]);
   await Promise.allSettled([...openBrowsers].map(id => browsers.deleteBrowser(id)));
+  // After the browsers, so a directory a dying agent still writes to cannot stop their deletion.
+  for (const dir of sandboxDirs) rmSync(dir, { recursive: true, force: true, maxRetries: 3 });
   process.exit(code);
 }
 
@@ -533,9 +577,9 @@ async function runTrial(plan, intentPlan, arm, key) {
 
     const agentStartedAt = Date.now();
     row.agent_started_at = new Date(agentStartedAt).toISOString();
-    const agent = await runAgent({
+    const agent = await runPi({
       args: trialArgs(plan, trialPrompt(intent, plan.backend === "kernel" ? browser : null)),
-      env: agentEnv(join(dir, "agent"), mcpFor(plan, browser)),
+      agentDir: join(dir, "agent"), mcpConfig: mcpFor(plan, browser), model: plan.model,
       cwd: dir, capMs: intentPlan.cap_sec * 1000, maxTurns: intentPlan.max_turns, maxBudgetUsd: plan.max_budget_usd,
       transcriptPath: join(dir, "transcript.jsonl"), stderrPath: join(dir, "stderr.txt"),
     });

bench/test/agent.test.mjs

@@ -188,6 +188,19 @@ test("the runner stops pi at the turn, spend and time caps, and when a cost is m
   assert.deepEqual([slow.limit, slow.durationMs], ["time cap", 300]);
 });
 
+test("the runner redacts the given strings from the transcript, including one split across streamed deltas", async () => {
+  const key = "sk-ant-0123456789abcdef";
+  const read = { type: "tool_execution_end", toolName: "read", result: { content: [{ type: "text", text: `ANTHROPIC_API_KEY=${key}\0PATH=/usr/bin` }] } };
+  const delta = (text) => ({ type: "message_update", assistantMessageEvent: { type: "text_delta", contentIndex: 0, delta: text } });
+  const echo = { type: "message_end", message: { role: "assistant", content: [{ type: "text", text: `It is ${key}` }], usage: { cost: { total: 0.01 } } } };
+  const { events } = await runEvents([read, delta(`It is ${key.slice(0, 10)}`), delta(key.slice(10)), echo, { type: "agent_settled" }], { redact: [key] });
+  assert.equal(events[0].result.content[0].text, "ANTHROPIC_API_KEY=[redacted]\0PATH=/usr/bin");
+  assert.deepEqual(events.map(event => event.type), ["tool_execution_end", "message_end", "agent_settled"]);
+  assert.equal(events[1].message.content[0].text, "It is [redacted]");
+  // Without strings to redact, the transcript keeps message_update.
+  assert.equal((await runEvents([delta("hi"), { type: "agent_settled" }])).events[0].type, "message_update");
+});
+
 test("a failed invocation keeps its error, whether the call, the tool or the MCP server failed", () => {
   const invoke = (id) => call(id, "mcp__kernel__webmcp", { action: "invoke", tool_ref: "wmcp_a" });
   const summary = summarize([