Add isolated_paths declaration to the docker driver

docs/DOCKER_ISOLATION.md

@@ -42,6 +42,8 @@ The agent launcher clears inheritable, ambient, and bounding capabilities and se
 
 Local plugins are projected on the host into manifest-verified bundles containing only supported discovery subtrees. Absolute, escaping, broken, and excluded-target symlinks fail closed. The complete repository is mounted only at a root-inaccessible grader path; the agent sees the sanitized bundle under `/opt/coder-eval/agent-skills`.
 
+`sandbox.docker.isolated_paths` lets a task or experiment declare host paths that must stay outside everything the agent can read, typically a suite's fixture tree beside the skill under test. The declaration mounts nothing: before the container starts, the runner checks each declared path against every agent-visible mount source and against the manifests of the sanitized plugin bundles, and fails the run if either would carry content from under it. Sources mounted below the root-only grader parent are exempt, since that placement is exactly what keeps them private. Requires `agent_isolation`.
+
 Older/custom images must declare `org.coder-eval.agent-isolation=uid-gid-v1`. A protected run rejects an image without that label before making an LLM call. Images derived with `FROM coder-eval-agent:<current-version>` inherit it. Runtime-kit injection into an unrelated base does not yet provide the required Linux users and `setpriv` launchers, so it is not compatible with protected mode.
 
 ## Running a task in Docker

docs/TASK_DEFINITION_GUIDE.md

@@ -587,6 +587,20 @@ Malformed responses, oversized output, request-budget exhaustion, and service st
 
 `passthrough_argv_prefixes` is for deliberately public live operations such as `uip docsai ask`. `mockd` invokes the real tool only when argv begins with one of these typed prefixes, caches the response in memory for the run, and never reveals the executable path to the agent. Do not use a broad prefix such as `[or]` or `[auth]`. `protected_mocks` and `record_cli` cannot claim the same tool name.
 
+### Declaring Paths That Must Stay Private
+
+`isolated_paths` declares host paths the evaluated agent must not be able to read. It mounts nothing; it is a contract the runner checks before any container starts, so a suite states the boundary it depends on instead of trusting an implementation detail. Entries expand `~` and `$VAR`, need not exist on disk, and require the Docker driver with its default `agent_isolation`:
+
+```yaml
+sandbox:
+  driver: docker
+  docker:
+    isolated_paths:
+      - "$SKILLS_REPO_PATH/tests"     # fixtures the graded skill must not read
+```
+
+The run fails at startup if a declared path overlaps an agent-readable mount, or if the sanitized plugin bundle would project a file from under it. Raw task, reference, template, and plugin sources mounted below the root-only grader parent are exempt: keeping them there is how a declared path stays private.
+
 ## Template Sources
 
 Tasks can start with preset files instead of an empty sandbox. Multiple sources are applied sequentially (last wins for conflicts).

src/coder_eval/isolation/docker_runner.py

@@ -42,7 +42,7 @@
     PreservationMode,
     ResourceLimits,
 )
-from coder_eval.plugin_bundle import stage_bundle
+from coder_eval.plugin_bundle import BundleManifest, stage_bundle
 from coder_eval.streaming.callbacks import safe_emit
 from coder_eval.streaming.wire import deserialize_event, has_prefix
 from coder_eval.utils import get_default_docker_image_tag
@@ -524,6 +524,9 @@ def __init__(
         # only manifest-verified projections; raw sources are mounted under the
         # root-only grader parent at unrelated container paths.
         self._agent_plugin_mounts: list[tuple[Path, str]] = []
+        # (plugin source, manifest of what its bundle projects) pairs, consulted by
+        # the sandbox.docker.isolated_paths guard.
+        self._plugin_bundle_manifests: list[tuple[Path, BundleManifest]] = []
         self._private_source_mounts: list[tuple[Path, str]] = []
         self._host_to_private_paths: dict[str, str] = {}
         self._host_plugin_to_agent_paths: dict[str, str] = {}
@@ -625,6 +628,11 @@ async def run(self) -> EvaluationResult:
             # under `staging` and records it on self._claude_mount_src for
             # _build_argv to mount. Cleaned up with `staging` in the finally.
             await asyncio.to_thread(self._prepare_host_mounts, staging)
+            # Every agent-visible mount source is known by now (bundles from
+            # _prepare_isolated_sources, the ~/.claude copy from
+            # _prepare_host_mounts), so the declaration can be proven before any
+            # container starts.
+            await asyncio.to_thread(self._enforce_isolated_paths)
             argv = self._build_argv(input_dir, output_dir, container_name=container_name, image=image)
             logger.info("Running task '%s' in docker: %s", self.rt.task.task_id, " ".join(argv))
             # Prime the heartbeat before the container starts so the
@@ -751,6 +759,7 @@ def _prepare_isolated_sources(self, staging: Path) -> None:
         """
 
         self._agent_plugin_mounts = []
+        self._plugin_bundle_manifests = []
         self._private_source_mounts = []
         self._host_to_private_paths = {}
         self._host_plugin_to_agent_paths = {}
@@ -781,6 +790,7 @@ def _prepare_isolated_sources(self, staging: Path) -> None:
             public_target = f"{CONTAINER_AGENT_SKILLS_DIR}/plugin-{index}"
             private_target = f"{CONTAINER_PRIVATE_PLUGIN_DIR}/plugin-{index}"
             self._agent_plugin_mounts.append((bundle_source, public_target))
+            self._plugin_bundle_manifests.append((source, manifest))
             self._register_private_mount(source, private_target)
             self._host_plugin_to_agent_paths[str(source)] = public_target
             logger.info(
@@ -844,6 +854,54 @@ def _prepare_isolated_sources(self, staging: Path) -> None:
             mock_root.chmod(0o555)
             self._mock_fixture_mount = mock_root
 
+    def _agent_visible_mount_sources(self) -> list[tuple[str, Path]]:
+        """Host paths whose content the evaluated agent can read inside the container.
+
+        Every other mount the runner renders lands below the root-only grader
+        parent (or the mockd fixture parent), so it is outside this set by
+        construction. Returns ``(label, host source)`` pairs; the label names the
+        agent-visible destination for the error message.
+        """
+
+        sources = [(f"plugin bundle mount {target}", source.resolve()) for source, target in self._agent_plugin_mounts]
+        if self._claude_mount_src is not None:
+            agent_claude = f"{AGENT_HOME}/.claude"
+            sources.append((f"agent home mount {agent_claude}", (Path.home() / ".claude").resolve()))
+        return sources
+
+    def _enforce_isolated_paths(self) -> None:
+        """Fail closed when a declared isolated path would become agent-visible.
+
+        ``sandbox.docker.isolated_paths`` is a declaration, not a mount
+        instruction: the runner proves that no agent-visible surface carries
+        content from a declared path, either as an agent-readable mount or as a
+        sanitized plugin bundle projection. Private grader mounts are exempt --
+        keeping a raw source below the root-only parent is the isolation working
+        as intended. A declared path that does not exist on the host has nothing
+        to expose, so it resolves non-strictly and passes.
+        """
+
+        declarations = self._docker_config.isolated_paths
+        if not declarations:
+            return
+        visible = self._agent_visible_mount_sources()
+        for raw in declarations:
+            declared = Path(os.path.expandvars(os.path.expanduser(raw))).resolve()
+            for label, source in visible:
+                if declared == source or declared in source.parents or source in declared.parents:
+                    raise DockerRunError(
+                        f"docker.isolated_paths entry {raw!r} resolves to {declared}, which overlaps the "
+                        + f"agent-visible {label} (host source {source})"
+                    )
+            for source, manifest in self._plugin_bundle_manifests:
+                for relative in (*manifest.files, *manifest.symlinks):
+                    entry = source / relative
+                    if entry == declared or declared in entry.parents:
+                        raise DockerRunError(
+                            f"docker.isolated_paths entry {raw!r} resolves to {declared}, which the sanitized "
+                            + f"plugin bundle for {source} projects to the agent as {relative}"
+                        )
+
     def _register_external_private_path(self, source: Path, target: str) -> None:
         source = source.resolve()
         task_dir = self.rt.task_file.parent.resolve() if self.rt.task_file else None

src/coder_eval/models/sandbox.py

@@ -293,6 +293,16 @@ class DockerDriverConfig(BaseModel):
         default_factory=list,
         description="Extra `-v src:dst[:ro]` mount specs forwarded to `docker run`. Validated for basic syntax.",
     )
+    isolated_paths: list[str] = MergeField(
+        strategy="replace",
+        default_factory=list,
+        description=(
+            "Host paths that must not be reachable by the evaluated agent. The run fails at startup if any "
+            "declared path would become agent-visible -- through an agent-readable mount or through a "
+            "sanitized plugin bundle projection. Entries expand `~` and `$VAR`; a path that does not exist "
+            "on the host has nothing to expose and passes. Requires agent_isolation."
+        ),
+    )
 
     @field_validator("working_dir")
     @classmethod
@@ -565,4 +575,12 @@ def validate_template_sources(self) -> SandboxConfig:
             overlap = sorted(recorded & set(tools))
             if overlap:
                 raise ValueError(f"protected_mocks and record_cli cannot both provide: {overlap}")
+        if self.docker.isolated_paths:
+            if self.driver != "docker":
+                raise ValueError("sandbox.docker.isolated_paths requires driver: docker")
+            if not self.docker.agent_isolation:
+                raise ValueError(
+                    "sandbox.docker.isolated_paths requires docker.agent_isolation: true -- the declaration "
+                    + "cannot be enforced without the agent identity boundary"
+                )
         return self

tests/test_isolated_paths.py

@@ -0,0 +1,221 @@
+"""Tests for ``sandbox.docker.isolated_paths``, the declared agent-invisible set.
+
+The field is a contract rather than a mount instruction: the docker runner proves
+that no agent-visible surface (an agent-readable mount, or a sanitized plugin
+bundle projection) carries content from a declared path, and fails closed before
+a container starts. Private grader mounts are exempt -- keeping a raw source
+below the root-only parent is the isolation working as intended.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import ValidationError
+
+from coder_eval.isolation.docker_runner import DockerRunError, DockerRunner
+from coder_eval.models import (
+    AgentKind,
+    ClaudeCodeAgentConfig,
+    DockerDriverConfig,
+    ExperimentDefaults,
+    ExperimentDefinition,
+    ExperimentVariant,
+    FileExistsCriterion,
+    SandboxConfig,
+    TaskDefinition,
+    TemplateDirSource,
+)
+from coder_eval.orchestration.experiment import resolve_task_for_variant
+from coder_eval.orchestration.overrides import apply_overrides
+
+
+def _write(path: Path, text: str = "data") -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+    return path
+
+
+def _runner(
+    tmp_path: Path,
+    isolated_paths: list[str],
+    *,
+    plugin: Path | None = None,
+    template_dir: Path | None = None,
+) -> DockerRunner:
+    template_sources = [TemplateDirSource(path=str(template_dir))] if template_dir is not None else None
+    task = TaskDefinition(
+        task_id="isolated-paths",
+        description="test",
+        initial_prompt="work",
+        agent=ClaudeCodeAgentConfig(
+            type=AgentKind.CLAUDE_CODE,
+            plugins=[{"type": "local", "path": str(plugin)}] if plugin is not None else None,
+        ),
+        sandbox=SandboxConfig(
+            driver="docker",
+            docker=DockerDriverConfig(agent_isolation=True, isolated_paths=isolated_paths),
+            template_sources=template_sources,
+        ),
+        success_criteria=[FileExistsCriterion(description="done", path="done.txt")],
+    )
+    rt = MagicMock(task=task, task_file=tmp_path / "task" / "task.yaml", run_dir=tmp_path / "run")
+    return DockerRunner(rt)
+
+
+class TestAgentVisibleMounts:
+    def test_declared_path_inside_an_agent_visible_mount_fails(self, tmp_path: Path) -> None:
+        bundle = tmp_path / "bundle"
+        _write(bundle / "skills" / "demo" / "SKILL.md", "public")
+        declared = bundle / "skills"
+
+        runner = _runner(tmp_path, [str(declared)])
+        runner._agent_plugin_mounts = [(bundle, "/opt/coder-eval/agent-skills/plugin-0")]
+
+        with pytest.raises(DockerRunError) as excinfo:
+            runner._enforce_isolated_paths()
+
+        message = str(excinfo.value)
+        assert str(declared) in message
+        assert str(bundle) in message
+        assert "/opt/coder-eval/agent-skills/plugin-0" in message
+
+    def test_declared_parent_of_an_agent_visible_mount_fails(self, tmp_path: Path) -> None:
+        """Containment counts in both directions: the mount sits inside the declaration."""
+
+        bundle = tmp_path / "workspace" / "bundle"
+        bundle.mkdir(parents=True)
+        runner = _runner(tmp_path, [str(tmp_path / "workspace")])
+        runner._agent_plugin_mounts = [(bundle, "/opt/coder-eval/agent-skills/plugin-0")]
+
+        with pytest.raises(DockerRunError, match="overlaps the agent-visible"):
+            runner._enforce_isolated_paths()
+
+    def test_agent_home_mount_is_agent_visible(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+        """``~`` expands, and the ~/.claude copy mounted at the agent's HOME is agent-visible."""
+
+        home = tmp_path / "home"
+        _write(home / ".claude" / "plugins" / "repo" / "skills" / "demo" / "SKILL.md")
+        monkeypatch.setenv("HOME", str(home))
+        monkeypatch.setenv("USERPROFILE", str(home))
+
+        runner = _runner(tmp_path, ["~/.claude/plugins/repo"])
+        runner._claude_mount_src = tmp_path / "staging" / "claude-home"
+
+        with pytest.raises(DockerRunError, match="agent home mount"):
+            runner._enforce_isolated_paths()
+
+    def test_env_var_in_declaration_is_expanded(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+        bundle = tmp_path / "bundle"
+        bundle.mkdir()
+        monkeypatch.setenv("ISOLATED_TEST_DIR", str(bundle))
+
+        runner = _runner(tmp_path, ["$ISOLATED_TEST_DIR"])
+        runner._agent_plugin_mounts = [(bundle, "/opt/coder-eval/agent-skills/plugin-0")]
+
+        with pytest.raises(DockerRunError, match=r"\$ISOLATED_TEST_DIR"):
+            runner._enforce_isolated_paths()
+
+    def test_unrelated_path_passes(self, tmp_path: Path) -> None:
+        bundle = tmp_path / "bundle"
+        bundle.mkdir()
+        elsewhere = tmp_path / "elsewhere"
+        elsewhere.mkdir()
+
+        runner = _runner(tmp_path, [str(elsewhere)])
+        runner._agent_plugin_mounts = [(bundle, "/opt/coder-eval/agent-skills/plugin-0")]
+
+        runner._enforce_isolated_paths()
+
+    def test_missing_declared_path_passes(self, tmp_path: Path) -> None:
+        """Nothing on disk means nothing to expose; declarations resolve non-strictly."""
+
+        runner = _runner(tmp_path, [str(tmp_path / "never" / "created")])
+
+        runner._enforce_isolated_paths()
+
+
+class TestPluginBundleProjection:
+    """The motivating shape: a skills checkout whose tests tree must stay private."""
+
+    def _prepared_runner(self, tmp_path: Path, declared: str) -> DockerRunner:
+        plugin = tmp_path / "skills-repo"
+        _write(plugin / "skills" / "demo" / "SKILL.md", "public")
+        _write(plugin / "tests" / "fixtures" / "golden.json", "expected answers")
+        runner = _runner(tmp_path, [declared], plugin=plugin, template_dir=plugin / "tests")
+        staging = tmp_path / "staging"
+        staging.mkdir()
+        runner._prepare_isolated_sources(staging)
+        return runner
+
+    def test_declared_tests_tree_outside_the_projection_passes(self, tmp_path: Path) -> None:
+        runner = self._prepared_runner(tmp_path, str(tmp_path / "skills-repo" / "tests"))
+
+        # The raw checkout is mounted, but only below the root-only grader parent.
+        assert any(source == tmp_path / "skills-repo" for source, _ in runner._private_source_mounts)
+        # A projection exists, so the declaration is checked against real entries.
+        assert [relative for _, manifest in runner._plugin_bundle_manifests for relative in manifest.files]
+        runner._enforce_isolated_paths()
+
+    def test_declared_path_the_bundle_projects_fails(self, tmp_path: Path) -> None:
+        runner = self._prepared_runner(tmp_path, str(tmp_path / "skills-repo" / "skills"))
+
+        with pytest.raises(DockerRunError) as excinfo:
+            runner._enforce_isolated_paths()
+
+        message = str(excinfo.value)
+        assert str(tmp_path / "skills-repo" / "skills") in message
+        assert "skills/demo/SKILL.md" in message
+
+
+class TestConfigValidation:
+    def test_isolated_paths_requires_docker_driver(self) -> None:
+        with pytest.raises(ValidationError, match="requires driver: docker"):
+            SandboxConfig(driver="tempdir", docker=DockerDriverConfig(isolated_paths=["/some/dir"]))
+
+    def test_isolated_paths_requires_agent_isolation(self) -> None:
+        with pytest.raises(ValidationError, match=re.escape("requires docker.agent_isolation")):
+            SandboxConfig(
+                driver="docker",
+                docker=DockerDriverConfig(agent_isolation=False, isolated_paths=["/some/dir"]),
+            )
+
+    def test_empty_declaration_is_unconstrained(self) -> None:
+        SandboxConfig(driver="tempdir", docker=DockerDriverConfig(agent_isolation=False))
+
+
+class TestConfigMerge:
+    """The declaration must survive the layered resolver and the CLI override path."""
+
+    def _task(self) -> TaskDefinition:
+        return TaskDefinition(
+            task_id="t",
+            description="x",
+            initial_prompt="hi",
+            agent=ClaudeCodeAgentConfig(type=AgentKind.CLAUDE_CODE),
+            sandbox=SandboxConfig(driver="docker"),
+            success_criteria=[FileExistsCriterion(description="done", path="done.txt")],
+        )
+
+    def test_experiment_defaults_layer_and_cli_override_agree(self) -> None:
+        base = ExperimentDefinition(
+            experiment_id="default",
+            variants=[ExperimentVariant(variant_id="default")],
+        )
+        experiment = ExperimentDefinition(
+            experiment_id="test",
+            defaults=ExperimentDefaults(
+                sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(isolated_paths=["/repo/tests"]))
+            ),
+            variants=[ExperimentVariant(variant_id="v")],
+        )
+        layered, _, _ = resolve_task_for_variant(base, self._task(), experiment, experiment.variants[0])
+
+        overridden = self._task()
+        apply_overrides(overridden, {"sandbox.docker.isolated_paths": ["/repo/tests"]})
+
+        assert layered.sandbox.docker.isolated_paths == ["/repo/tests"]
+        assert layered.sandbox.docker.model_dump() == overridden.sandbox.docker.model_dump()