diff --git a/bench/README.md b/bench/README.md index 586c0c39..e35fb7f5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -28,7 +28,7 @@ bench/ tasks//.json one intent per file; domain.json holds per-domain settings schemas/ JSON Schemas for answers prompts/system.md the system prompt both arms get - allowlist.txt the kernel MCP tools the agent gets, and no others + allowlist.txt the tools the agent gets, and no others: the kernel MCP tools and pi's read mcp.json the kernel MCP server; reads KERNEL_API_KEY .env.example keys run.mjs loads from bench/.env run.mjs grade.mjs report.mjs lib/ test/ @@ -41,6 +41,24 @@ Needs the `kernel` and `pi` CLIs, Node 22.13+, git, `KERNEL_API_KEY` and the mod key (such as `OPENAI_API_KEY`), from the environment or `bench/.env`. Each agent gets its own `PI_CODING_AGENT_DIR`, so stored pi logins and MCP servers are not used. +### Sandbox + +pi cuts an MCP result over 20KB, such as a long WebMCP tool list, and saves the full text to a +file, so the agent gets pi's `read` tool to open it. So that `read` cannot reach the bench's own +files, agents on Kernel browsers run in a [bubblewrap](https://github.com/containers/bubblewrap) +sandbox (`bwrap`, Linux only; `apt install bubblewrap`). It sees `/usr`, the `/etc` files DNS and +TLS need, its own processes and a fresh directory with its pi config and `TMPDIR`; not your home, +this checkout, other trials or the rest of `/tmp` and `/etc`. It shares the network. The agent gets +only `KERNEL_API_KEY`, the variables of its model's provider (`ANTHROPIC_*` for `anthropic/...`, +`GEMINI_*` for `google/...`), the proxy variables and `NODE_EXTRA_CA_CERTS`; the values of keys +and proxy URLs are redacted from transcripts, which then leave out pi's streamed `message_update` +events. If Node or pi is installed outside `/usr`, such as under `~/.nvm` or `~/.local`, only Node's +binary and pi's package are mounted, read-only, and pi runs as `node `, so it must be +an npm install rather than a standalone binary or a version manager's shim. Unprivileged user +namespaces must be allowed, which some distributions restrict with AppArmor; the smoke gate stops +the run if `bwrap` cannot create a sandbox, if one can see this checkout or your home, or if pi +does not start in one. + Run it as a Hypeship service under a hold that covers the whole run; holds cannot be renewed. This pilot is 105 trials and can take about 8 hours if every trial reaches its cap: @@ -108,6 +126,8 @@ Before any trial, stopping on the first failure: 1. the hold covers the remaining jobs if every trial reaches its cap after 180s of setup (the hold is checked again before each job, and no job starts that it cannot outlast); 2. `KERNEL_API_KEY` is set (Kernel browsers), and on resume pi is the same version; + with Kernel browsers, a sandbox works, can see neither this checkout nor your home, and runs the + same pi; 3. a zero-tool pi run answers and reports a cost above zero (pi prices an unknown model at 0, which would disable the spend cap); 4. a treatment setup on the first pending intent works end to end; diff --git a/bench/allowlist.txt b/bench/allowlist.txt index f054315a..e212e45b 100644 --- a/bench/allowlist.txt +++ b/bench/allowlist.txt @@ -2,3 +2,4 @@ mcp__kernel__browser_repl mcp__kernel__computer_action mcp__kernel__execute_playwright_code mcp__kernel__webmcp +read diff --git a/bench/lib/agent.mjs b/bench/lib/agent.mjs index 51264646..b099e750 100644 --- a/bench/lib/agent.mjs +++ b/bench/lib/agent.mjs @@ -31,10 +31,12 @@ export function parseRecord(line) { } // The clock runs from launch to agent_settled. pi has no turn or spend cap, so the runner -// enforces both; a missing cost stops the agent, since the spend cap needs it. -export function runAgent({ args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, transcriptPath, stderrPath }) { +// enforces both; a missing cost stops the agent, since the spend cap needs it. Each string in +// `redact` is replaced in the transcript, which then leaves out message_update: its streamed deltas +// can split a string, and message_end holds the whole message. +export function runAgent({ command = "pi", args, env, cwd, capMs, maxTurns = Infinity, maxBudgetUsd = Infinity, redact = [], transcriptPath, stderrPath }) { return new Promise((resolve, reject) => { - const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); runningAgents.add(child); const start = performance.now(); const elapsed = () => Math.round(performance.now() - start); @@ -61,7 +63,8 @@ export function runAgent({ args, env, cwd, capMs, maxTurns = Infinity, maxBudget const onRecord = (line) => { if (!line.trim()) return; - const event = parseRecord(line); + const event = parseRecord(redact.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), line)); + if (redact.length && event.type === "message_update") return; event._t = elapsed(); events.push(event); transcript.write(JSON.stringify(event) + "\n"); diff --git a/bench/lib/sandbox.mjs b/bench/lib/sandbox.mjs new file mode 100644 index 00000000..d301df09 --- /dev/null +++ b/bench/lib/sandbox.mjs @@ -0,0 +1,129 @@ +// Agents with a built-in tool such as `read` run in a bubblewrap sandbox, so they cannot read the +// bench's files: intents and their rules, results, other trials, the home directory or keys. The +// sandbox shows /usr, the /etc files DNS and TLS need, the agent's own processes, and a fresh +// directory with its pi config and TMPDIR, where pi saves the full text of a long MCP result. + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { delimiter, join } from "node:path"; + +const ETC = [ + "/etc/resolv.conf", "/etc/hosts", "/etc/nsswitch.conf", "/etc/gai.conf", "/etc/ssl", "/etc/pki", + "/etc/ca-certificates", "/etc/passwd", "/etc/group", "/etc/localtime", "/etc/alternatives", +]; + +const under = (path, dir) => path === dir || path.startsWith(`${dir}/`); + +const which = (name, env) => (env.PATH ?? "").split(delimiter).map(dir => join(dir, name)).find(path => existsSync(path)); + +// How a sandbox runs pi: with the host's node, mounting Node's binary and pi's package by themselves +// when they are outside /usr, so the rest of a prefix such as ~/.local or ~/.nvm stays hidden. +export function piLauncher(env = process.env, execPath = process.execPath) { + const node = realpathSync(which("node", env) ?? execPath); + const cli = realpathSync(which("pi", env)); + const pkg = cli.match(/^(.*\/node_modules\/(?:@[^/]+\/)?[^/]+)\//)?.[1] ?? cli; + return { mounts: [...new Set([node, pkg])].filter(path => !under(path, "/usr")), command: [node, cli] }; +} + +export function sandboxArgs(dir, mounts = []) { + return [ + "--unshare-all", "--share-net", "--die-with-parent", "--new-session", + "--ro-bind", "/usr", "/usr", + ...["/bin", "/sbin", "/lib", "/lib64"].flatMap(path => ["--ro-bind-try", path, path]), + ...ETC.flatMap(path => ["--ro-bind-try", path, path]), + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", + ...mounts.flatMap(path => ["--ro-bind", path, path]), + "--bind", dir, dir, "--chdir", dir, + ]; +} + +// A fresh directory for one agent, with its mcp.json. It is under /tmp, not the host's TMPDIR, +// which may be in the home directory and would then show the sandbox an entry of it. +export function sandboxDir(mcpConfig) { + const dir = mkdtempSync("/tmp/xwebmcp-agent-"); + mkdirSync(join(dir, "agent")); + mkdirSync(join(dir, "tmp")); + if (mcpConfig) writeFileSync(join(dir, "agent", "mcp.json"), JSON.stringify(mcpConfig, null, 2) + "\n"); + return dir; +} + +// pi's providers whose variables are not named after them, as in pi's getApiKeyEnvVars. +const PROVIDER_PREFIXES = { + "amazon-bedrock": ["AWS_"], "cloudflare-ai-gateway": ["CLOUDFLARE_"], "cloudflare-workers-ai": ["CLOUDFLARE_"], + "github-copilot": ["COPILOT_"], google: ["GEMINI_"], "google-vertex": ["GOOGLE_CLOUD_"], huggingface: ["HF_"], + "kimi-coding": ["KIMI_"], moonshotai: ["MOONSHOT_"], "moonshotai-cn": ["MOONSHOT_"], "opencode-go": ["OPENCODE_"], + "qwen-token-plan-individual": ["QWEN_TOKEN_PLAN_"], "vercel-ai-gateway": ["AI_GATEWAY_"], +}; + +// The model's provider is the part of --model before the slash. Its variables, such as +// ANTHROPIC_API_KEY for anthropic/... or GEMINI_API_KEY for google/..., are the only keys the agent +// gets besides Kernel's. +export function providerPrefixes(model) { + const provider = model.split("/")[0]; + return PROVIDER_PREFIXES[provider] ?? [`${provider.toUpperCase().replace(/-/g, "_")}_`]; +} + +// pi's HTTP client reads the proxy variables, and Node the extra CA file an intercepting proxy needs. +const NETWORK = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy", "NODE_EXTRA_CA_CERTS"]; + +export function sandboxEnv(model, dir, env = process.env) { + const prefixes = providerPrefixes(model); + const keys = Object.entries(env).filter(([name]) => name === "KERNEL_API_KEY" || NETWORK.includes(name) || prefixes.some(prefix => name.startsWith(prefix))); + return { + ...Object.fromEntries(keys), + PATH: env.PATH, + HOME: dir, + TMPDIR: join(dir, "tmp"), + PI_CODING_AGENT_DIR: join(dir, "agent"), + PI_OFFLINE: "1", + PI_TELEMETRY: "0", + }; +} + +// The agent can read its keys in /proc/self/environ, so the runner removes their values from its +// transcript. A proxy URL can hold a password. +export const secretValues = (env) => Object.entries(env) + .filter(([name, value]) => /KEY|TOKEN|SECRET|^https?_proxy$/i.test(name) && value.length >= 16) + .map(([, value]) => value); + +// The paths to check a sandbox cannot see: the bench, and each entry of the home directory except +// those that hold a mount, which bwrap creates as empty directories. +export function hiddenPaths(benchDir, home, mounts) { + const entries = home ? readdirSync(home).map(name => join(home, name)) : []; + return [benchDir, ...entries.filter(path => !mounts.some(mount => under(mount, path)))]; +} + +function runSandboxed(command, mounts) { + const dir = sandboxDir(null); + try { + return execFileSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...command], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// The error line of a failed command's stderr, such as Node's SyntaxError, or else its last line. +const stderrOf = (error) => { + const lines = (error.stderr?.toString() ?? "").trim().split("\n").filter(Boolean); + return lines.find(line => /^\w*Error\b/.test(line)) ?? lines.at(-1) ?? error.message; +}; + +// Throws when bwrap cannot create a sandbox; returns the paths a sandbox can see. +export function visiblePaths(paths, mounts) { + const probe = ["sh", "-c", 'for p; do [ -e "$p" ] && echo "$p"; done; true', "sh", ...paths]; + try { + return runSandboxed(probe, mounts).split("\n").filter(Boolean); + } catch (error) { + throw new Error(`bwrap cannot create a sandbox: ${stderrOf(error)}`); + } +} + +// pi's --version as the sandbox runs it. Throws when it does not start there, as when `pi` is a +// standalone binary or a version manager's shim rather than Node's script. +export function sandboxedPiVersion({ command, mounts }) { + try { + return runSandboxed([...command, "--version"], mounts).trim(); + } catch (error) { + throw new Error(`${command.join(" ")} --version failed in the sandbox: ${stderrOf(error)}`); + } +} diff --git a/bench/run.mjs b/bench/run.mjs index 88b09114..9c5d273a 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -22,6 +22,7 @@ import { composeBatch, domainPatterns, gitShow, matchesUrlPattern, registeredToo import { blockedAttempts, writeGuardExtension } from "./lib/guard.mjs"; import { readPage, readPageArg, stepTimeline } from "./lib/page.mjs"; import { checkRule, gradeAnswer } from "./grade.mjs"; +import { hiddenPaths, piLauncher, providerPrefixes, sandboxArgs, sandboxDir, sandboxedPiVersion, sandboxEnv, secretValues, visiblePaths } from "./lib/sandbox.mjs"; const benchDir = dirname(fileURLToPath(import.meta.url)); if (existsSync(join(benchDir, ".env"))) process.loadEnvFile(join(benchDir, ".env")); @@ -281,6 +282,45 @@ function trialArgs(plan, prompt) { return piArgs({ prompt, systemPrompt: experiment.system_prompt, allow: plan.allowlist, model: plan.model, thinking: plan.thinking }); } +// Set by the smoke gate when the allowlist has a built-in tool: agents then run in a sandbox, with +// this pi command and its read-only mounts besides /usr. Open sandbox directories are removed on +// shutdown. +let sandbox = null; +const sandboxDirs = new Set(); + +function checkSandbox(agent) { + const launcher = piLauncher(); + let visible; + try { + visible = visiblePaths(hiddenPaths(benchDir, process.env.HOME, launcher.mounts), launcher.mounts); + } catch (error) { + fail(`${experiment.files.allowlist} has a built-in tool, so agents run in a bubblewrap sandbox, and ${error.message}; see "Sandbox" in bench/README.md`); + } + if (visible.length) fail(`the agent sandbox can see ${visible.join(", ")}; pi or Node is installed in a directory that holds it, so install them in a directory of their own`); + let version; + try { + version = `pi ${sandboxedPiVersion(launcher)}`; + } catch (error) { + fail(`the agent sandbox runs pi with Node, and ${error.message}; use an npm install of pi, with no version manager's shim for pi or Node on PATH`); + } + if (version !== agent) fail(`the agent sandbox runs ${version}, but \`pi --version\` is ${agent}`); + sandbox = launcher; +} + +// pi as the bench user, or in a sandbox that gets only the model's and Kernel's keys. +async function runPi({ agentDir, mcpConfig, model, ...options }) { + if (!sandbox) return runAgent({ ...options, env: agentEnv(agentDir, mcpConfig) }); + const dir = sandboxDir(mcpConfig); + sandboxDirs.add(dir); + const env = sandboxEnv(model, dir); + try { + return await runAgent({ ...options, command: "bwrap", args: [...sandboxArgs(dir, sandbox.mounts), "--", ...sandbox.command, ...options.args], env, redact: secretValues(env) }); + } finally { + rmSync(dir, { recursive: true, force: true }); + sandboxDirs.delete(dir); + } +} + const stderrTail = (file) => (existsSync(file) ? readFileSync(file, "utf8") : "").trim().split("\n").slice(-3).join(" "); function logSmokeSpend(cost_usd) { @@ -301,16 +341,18 @@ async function smokeGate(plan, probe, signedIn) { plan.kernel_cli ??= kernelCli; } - const plain = await runAgent({ + if (plan.allowlist.some(tool => !tool.startsWith("mcp__"))) checkSandbox(agent); + + const plain = await runPi({ args: piArgs({ prompt: "Reply with the single word READY.", systemPrompt: "Answer briefly.", model: plan.model, thinking: plan.thinking }), - env: agentEnv(join(dir, "plain-agent"), null), + agentDir: join(dir, "plain-agent"), mcpConfig: null, model: plan.model, cwd: dir, capMs: 120_000, transcriptPath: join(dir, "plain.jsonl"), stderrPath: join(dir, "plain.stderr"), }); if (stopping) return; const { final, answer, cost_usd } = summarize(plain.events, []); logSmokeSpend(cost_usd); if (final?.stopReason !== "stop" || !/READY/.test(answer)) { - fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key.`); + fail(`a zero-tool pi run did not answer: ${final?.errorMessage || stderrTail(join(dir, "plain.stderr")) || "no answer"}. Check --model and the provider's API key${sandbox ? `; in the sandbox the agent gets only KERNEL_API_KEY and ${providerPrefixes(plan.model).map(prefix => `${prefix}*`).join(", ")}` : ""}.`); } // pi prices a model missing from its table at 0, which would disable the spend cap. if (!(cost_usd > 0)) fail(`pi reported a cost of ${cost_usd} for ${plan.model}, so --max-budget-usd cannot apply`); @@ -321,9 +363,9 @@ async function smokeGate(plan, probe, signedIn) { const setup = await setUpBrowser(plan, probe, probeArm, "smoke"); if (setup.stopped) return; if (setup.error) fail(`${probeArm} setup failed for ${probe.intent.id} after ${setup.attempts} browsers: ${setup.error}`); - const configured = await runAgent({ + const configured = await runPi({ args: trialArgs(plan, "Reply with the single word READY. Do not call any tools."), - env: agentEnv(join(dir, "configured-agent"), mcpFor(plan, setup.browser)), + agentDir: join(dir, "configured-agent"), mcpConfig: mcpFor(plan, setup.browser), model: plan.model, cwd: dir, capMs: 120_000, transcriptPath: join(dir, "configured.jsonl"), stderrPath: join(dir, "configured.stderr"), }); await releaseBrowser(setup.browser); @@ -417,6 +459,8 @@ async function shutdown(code, message) { for (const child of runningAgents) child.kill("SIGKILL"); await Promise.allSettled([...creating]); await Promise.allSettled([...openBrowsers].map(id => browsers.deleteBrowser(id))); + // After the browsers, so a directory a dying agent still writes to cannot stop their deletion. + for (const dir of sandboxDirs) rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); process.exit(code); } @@ -533,9 +577,9 @@ async function runTrial(plan, intentPlan, arm, key) { const agentStartedAt = Date.now(); row.agent_started_at = new Date(agentStartedAt).toISOString(); - const agent = await runAgent({ + const agent = await runPi({ args: trialArgs(plan, trialPrompt(intent, plan.backend === "kernel" ? browser : null)), - env: agentEnv(join(dir, "agent"), mcpFor(plan, browser)), + agentDir: join(dir, "agent"), mcpConfig: mcpFor(plan, browser), model: plan.model, cwd: dir, capMs: intentPlan.cap_sec * 1000, maxTurns: intentPlan.max_turns, maxBudgetUsd: plan.max_budget_usd, transcriptPath: join(dir, "transcript.jsonl"), stderrPath: join(dir, "stderr.txt"), }); diff --git a/bench/test/agent.test.mjs b/bench/test/agent.test.mjs index cc9e2e01..9229e733 100644 --- a/bench/test/agent.test.mjs +++ b/bench/test/agent.test.mjs @@ -188,6 +188,19 @@ test("the runner stops pi at the turn, spend and time caps, and when a cost is m assert.deepEqual([slow.limit, slow.durationMs], ["time cap", 300]); }); +test("the runner redacts the given strings from the transcript, including one split across streamed deltas", async () => { + const key = "sk-ant-0123456789abcdef"; + const read = { type: "tool_execution_end", toolName: "read", result: { content: [{ type: "text", text: `ANTHROPIC_API_KEY=${key}\0PATH=/usr/bin` }] } }; + const delta = (text) => ({ type: "message_update", assistantMessageEvent: { type: "text_delta", contentIndex: 0, delta: text } }); + const echo = { type: "message_end", message: { role: "assistant", content: [{ type: "text", text: `It is ${key}` }], usage: { cost: { total: 0.01 } } } }; + const { events } = await runEvents([read, delta(`It is ${key.slice(0, 10)}`), delta(key.slice(10)), echo, { type: "agent_settled" }], { redact: [key] }); + assert.equal(events[0].result.content[0].text, "ANTHROPIC_API_KEY=[redacted]\0PATH=/usr/bin"); + assert.deepEqual(events.map(event => event.type), ["tool_execution_end", "message_end", "agent_settled"]); + assert.equal(events[1].message.content[0].text, "It is [redacted]"); + // Without strings to redact, the transcript keeps message_update. + assert.equal((await runEvents([delta("hi"), { type: "agent_settled" }])).events[0].type, "message_update"); +}); + test("a failed invocation keeps its error, whether the call, the tool or the MCP server failed", () => { const invoke = (id) => call(id, "mcp__kernel__webmcp", { action: "invoke", tool_ref: "wmcp_a" }); const summary = summarize([ diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs index 712e74a7..de112d46 100644 --- a/bench/test/run.test.mjs +++ b/bench/test/run.test.mjs @@ -240,3 +240,28 @@ test("a task on a domain with no tools runs control alone", { timeout: 120_000 } assert.ok(!log().some(entry => entry[0] === "add")); assert.match(run("uncovered").stdout, /nothing to run/); }); + +test("a run with a built-in tool stops before any agent when bwrap cannot create a sandbox", { timeout: 120_000 }, () => { + const { run, log } = workspace(); + const result = run("no-sandbox", { STUB_BWRAP_FAILS: "1" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /bwrap cannot create a sandbox: bwrap: setting up uid map: Permission denied/); + assert.deepEqual(log().filter(entry => entry[0] === "create"), []); +}); + +test("a run stops before any agent when the sandbox can see the bench", { timeout: 120_000 }, () => { + const { run, bench } = workspace(); + const result = run("leaky", { STUB_SANDBOX_LEAKS: "1" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`the agent sandbox can see ${bench}`)); +}); + +test("a run stops before any browser when pi is a shim the sandbox cannot run with Node", { timeout: 120_000 }, () => { + const { run, log } = workspace(); + const shim = mkdtempSync(join(tmpdir(), "bench-shim-")); + writeFileSync(join(shim, "pi"), `#!/bin/sh\nexec ${process.execPath} ${join(benchDir, "test/stubs/pi")} "$@"\n`, { mode: 0o755 }); + const result = run("shimmed", { PATH: [shim, join(benchDir, "test/stubs"), dirname(process.execPath), process.env.PATH].join(delimiter) }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /the agent sandbox runs pi with Node, and .* --version failed in the sandbox/); + assert.deepEqual(log().filter(entry => entry[0] === "create"), []); +}); diff --git a/bench/test/sandbox.test.mjs b/bench/test/sandbox.test.mjs new file mode 100644 index 00000000..d95488ea --- /dev/null +++ b/bench/test/sandbox.test.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, join } from "node:path"; +import test from "node:test"; +import { hiddenPaths, piLauncher, sandboxArgs, sandboxDir, sandboxedPiVersion, sandboxEnv, secretValues } from "../lib/sandbox.mjs"; + +test("a sandboxed agent gets only Kernel's key, the model provider's variables and its own directories", () => { + const env = sandboxEnv("anthropic/claude-haiku-5-5", "/tmp/xwebmcp-agent-1", { + PATH: "/usr/bin", HOME: "/home/me", KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", OPENAI_API_KEY: "o", GH_TOKEN: "g", + }); + assert.deepEqual(env, { + KERNEL_API_KEY: "k", ANTHROPIC_API_KEY: "a", PATH: "/usr/bin", HOME: "/tmp/xwebmcp-agent-1", + TMPDIR: "/tmp/xwebmcp-agent-1/tmp", PI_CODING_AGENT_DIR: "/tmp/xwebmcp-agent-1/agent", PI_OFFLINE: "1", PI_TELEMETRY: "0", + }); + // A provider whose key is not named after it. + const google = sandboxEnv("google/gemini-3-pro", "/tmp/xwebmcp-agent-1", { GEMINI_API_KEY: "g", GOOGLE_APPLICATION_CREDENTIALS: "/home/me/key.json" }); + assert.equal(google.GEMINI_API_KEY, "g"); + assert.ok(!("GOOGLE_APPLICATION_CREDENTIALS" in google)); + // The proxy variables pi's HTTP client reads. + const proxied = sandboxEnv("openai/gpt", "/tmp/xwebmcp-agent-1", { HTTPS_PROXY: "http://proxy:3128", no_proxy: "localhost", NODE_EXTRA_CA_CERTS: "/etc/ssl/proxy.pem" }); + assert.deepEqual([proxied.HTTPS_PROXY, proxied.no_proxy, proxied.NODE_EXTRA_CA_CERTS], ["http://proxy:3128", "localhost", "/etc/ssl/proxy.pem"]); +}); + +test("only long values of keys, tokens, secrets and proxy URLs are redacted", () => { + const key = "sk-ant-0123456789abcdef"; + const proxy = "http://user:pass@proxy:3128"; + assert.deepEqual(secretValues({ + ANTHROPIC_API_KEY: key, ANTHROPIC_BASE_URL: "https://api.anthropic.com/v1", HF_TOKEN: "short", https_proxy: proxy, NO_PROXY: "localhost,127.0.0.1", + }), [key, proxy]); +}); + +// A pi installed under a prefix that holds other files, as with an npm prefix of ~/.local. +function installPi(script) { + const prefix = mkdtempSync(join(tmpdir(), "bench-prefix-")); + const pkg = join(prefix, "lib/node_modules/@scope/pi"); + mkdirSync(join(pkg, "dist"), { recursive: true }); + writeFileSync(join(pkg, "dist/cli.js"), script); + mkdirSync(join(prefix, "bin")); + symlinkSync(join(pkg, "dist/cli.js"), join(prefix, "bin/pi")); + mkdirSync(join(prefix, "share")); + writeFileSync(join(prefix, "share/secret.txt"), "secret"); + return { prefix: realpathSync(prefix), env: { PATH: [join(prefix, "bin"), dirname(process.execPath)].join(delimiter) } }; +} + +test("the sandbox mounts pi's package and Node's binary, not their prefixes", () => { + const { prefix, env } = installPi(""); + try { + const node = realpathSync(process.execPath); + const { mounts, command } = piLauncher(env); + assert.deepEqual(mounts, [node, join(prefix, "lib/node_modules/@scope/pi")].filter(path => !path.startsWith("/usr/"))); + assert.deepEqual(command, [node, join(prefix, "lib/node_modules/@scope/pi/dist/cli.js")]); + } finally { + rmSync(prefix, { recursive: true, force: true }); + } +}); + +test("the sandbox binds its own directory over an empty /tmp and nothing from home", () => { + const args = sandboxArgs("/tmp/xwebmcp-agent-1", ["/home/me/.nvm/versions/node/v22"]); + const bound = []; + for (let i = 0; i < args.length; i++) if (args[i].startsWith("--") && args[i].includes("bind")) bound.push(args[i + 2]); + assert.ok(bound.includes("/usr") && bound.includes("/tmp/xwebmcp-agent-1")); + assert.deepEqual(bound.filter(path => path.startsWith("/home")), ["/home/me/.nvm/versions/node/v22"]); + assert.ok(!bound.includes("/etc")); + assert.ok(args.indexOf("--tmpfs") < args.indexOf("--bind")); +}); + +test("the paths checked hidden are the bench and the home entries that hold no mount", () => { + const home = mkdtempSync(join(tmpdir(), "bench-home-")); + for (const name of [".nvm", "xwebmcp", ".config"]) mkdirSync(join(home, name)); + const paths = hiddenPaths(join(home, "xwebmcp/bench"), home, [join(home, ".nvm/versions/node/v22")]); + assert.deepEqual(paths.sort(), [join(home, ".config"), join(home, "xwebmcp"), join(home, "xwebmcp/bench")]); +}); + +const bwrapWorks = spawnSync("bwrap", [...sandboxArgs(tmpdir()), "--", "true"]).status === 0; + +test("in a real sandbox the agent sees its directory, not other files or processes", { skip: !bwrapWorks && "bwrap cannot create a sandbox here" }, () => { + const other = mkdtempSync(join(tmpdir(), "bench-other-")); + writeFileSync(join(other, "secret.txt"), "secret"); + const { prefix, env } = installPi(`const fs = require("fs"); +fs.writeFileSync(process.env.TMPDIR + "/pi-mcp-1.txt", "full output"); +console.log(JSON.stringify({ + secret: fs.existsSync(${JSON.stringify(join(other, "secret.txt"))}), + etc: fs.existsSync("/etc/hostname"), + prefix: fs.existsSync(require("path").join(__dirname, "../../../../../share/secret.txt")), + mcp: JSON.parse(fs.readFileSync(process.env.PI_CODING_AGENT_DIR + "/mcp.json", "utf8")), + processes: fs.readdirSync("/proc").filter(name => /^\\d+$/.test(name)).length, + cwd: process.cwd(), +}));`); + const dir = sandboxDir({ mcpServers: { kernel: {} } }); + try { + const { mounts, command } = piLauncher(env); + const result = spawnSync("bwrap", [...sandboxArgs(dir, mounts), "--", ...command], { encoding: "utf8", env: sandboxEnv("openai/gpt", dir, env) }); + assert.equal(result.status, 0, result.stderr); + const seen = JSON.parse(result.stdout); + assert.deepEqual({ ...seen, processes: seen.processes <= 3 }, { secret: false, etc: false, prefix: false, mcp: { mcpServers: { kernel: {} } }, processes: true, cwd: dir }); + assert.match(dir, /^\/tmp\/xwebmcp-agent-/); + assert.equal(readFileSync(join(dir, "tmp/pi-mcp-1.txt"), "utf8"), "full output"); + } finally { + rmSync(dir, { recursive: true, force: true }); + rmSync(other, { recursive: true, force: true }); + rmSync(prefix, { recursive: true, force: true }); + } +}); + +test("a sandbox reports pi's version, and fails when pi is a shim rather than Node's script", { skip: !bwrapWorks && "bwrap cannot create a sandbox here" }, () => { + const { prefix, env } = installPi(`if (process.argv[2] === "--version") console.log("1.2.3");`); + const shim = mkdtempSync(join(tmpdir(), "bench-shim-")); + writeFileSync(join(shim, "pi"), `#!/bin/sh\nexec ${join(prefix, "bin/pi")} "$@"\n`, { mode: 0o755 }); + try { + assert.equal(sandboxedPiVersion(piLauncher(env)), "1.2.3"); + const shimmed = piLauncher({ PATH: [shim, env.PATH].join(delimiter) }); + assert.throws(() => sandboxedPiVersion(shimmed), /--version failed in the sandbox: /); + } finally { + rmSync(prefix, { recursive: true, force: true }); + rmSync(shim, { recursive: true, force: true }); + } +}); diff --git a/bench/test/stubs/bwrap b/bench/test/stubs/bwrap new file mode 100755 index 00000000..5fc064de --- /dev/null +++ b/bench/test/stubs/bwrap @@ -0,0 +1,21 @@ +#!/usr/bin/env node +// A stand-in for bwrap in run.test.mjs: runs the command in --chdir with the caller's environment, +// without mounts. The visibility probe sees nothing unless STUB_SANDBOX_LEAKS is set, and every +// sandbox fails when STUB_BWRAP_FAILS is set. +import { spawnSync } from "node:child_process"; + +if (process.env.STUB_BWRAP_FAILS) { + console.error("bwrap: setting up uid map: Permission denied"); + process.exit(1); +} +const args = process.argv.slice(2); +const dashes = args.indexOf("--"); +const options = args.slice(0, dashes); +const [command, ...rest] = args.slice(dashes + 1); +const chdir = options.includes("--chdir") ? options[options.indexOf("--chdir") + 1] : process.cwd(); +if (command === "sh" && rest[0] === "-c") { + if (process.env.STUB_SANDBOX_LEAKS) console.log(rest.slice(3).join("\n")); + process.exit(0); +} +const result = spawnSync(command, rest, { stdio: "inherit", cwd: chdir }); +process.exit(result.status ?? 1);