From edd87af21118f52cdb34e19bc549ce95f1049479 Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Thu, 17 Sep 2026 13:53:49 +0530 Subject: [PATCH 1/6] feat(sandbox): structured Docker diagnostics + actionable CLI pre-flight gate Add a frozen DockerDiagnostic dataclass carrying a probe reason plus platform-specific recovery hints, and refactor DockerBackend.diagnose() to return it for every failure mode (CLI missing on PATH, daemon unreachable, and probe timeout). is_available() now delegates to the structured diagnostic instead of a bare boolean probe. Surface the diagnostic in evalguard audit run's pre-flight gate: when an explicit '--backend docker' is requested but cannot come up, print the structured reason and color-coded hints, and point at the '--backend auto' graceful fallback instead of a bare warning. Closes. Ref. --- evalguard/cli/audit.py | 20 +++++ evalguard/sandbox/backends/docker.py | 106 +++++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/evalguard/cli/audit.py b/evalguard/cli/audit.py index d0a1540..f7d61d9 100644 --- a/evalguard/cli/audit.py +++ b/evalguard/cli/audit.py @@ -17,6 +17,7 @@ ) from evalguard.rewardhack.mutations import ALL_MUTATION_STRATEGIES from evalguard.rewardhack.tamper_detector import TamperDetector +from evalguard.sandbox.backends.docker import DockerBackend, DockerDiagnostic from evalguard.sandbox.profiles import SandboxProfile console = Console() @@ -55,6 +56,25 @@ def run_audit( workspace_base_dir=sb_profile.task_workspace_root, ) + # Pre-flight: emit actionable, color-coded guidance when an explicit isolation + # backend is requested but cannot come up on this host. This surfaces the + # structured DockerDiagnostic (probe reason + platform hints) instead of a + # bare warning, and hints at the `--backend auto` graceful fallback. + if backend == "docker" and not DockerBackend.is_available(): + diag: DockerDiagnostic = DockerBackend.diagnose() + console.print( + f"\n[bold red]Backend '{backend}' requested but unavailable on this host.[/bold red]" + ) + console.print(f" [bold]Reason:[/bold] {diag.reason}") + if diag.hints: + console.print(" [bold yellow]Try one of the following:[/bold yellow]") + for hint in diag.hints: + console.print(f" - {hint}") + console.print( + "\n To run with automatic backend detection instead, retry with:\n" + " evalguard audit run --backend auto" + ) + try: audit_record: TaskAuditRecord = adapter.run_task_with_audit( agent=command, diff --git a/evalguard/sandbox/backends/docker.py b/evalguard/sandbox/backends/docker.py index 2d5526e..ce85cf7 100644 --- a/evalguard/sandbox/backends/docker.py +++ b/evalguard/sandbox/backends/docker.py @@ -5,6 +5,8 @@ import logging import shutil import subprocess +import sys +from dataclasses import dataclass from pathlib import Path from evalguard.sandbox.backends.base import CommandResult, SandboxBackend @@ -13,6 +15,43 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class DockerDiagnostic: + """Structured result of probing the local Docker installation. + + ``available`` is ``True`` only when both the CLI binary is on ``PATH`` + and the daemon answers ``docker info`` successfully. + """ + + available: bool + reason: str + hints: tuple[str, ...] = () + + @classmethod + def ok(cls) -> DockerDiagnostic: + """Return a diagnostic representing a healthy Docker installation.""" + return cls(available=True, reason="Docker daemon is reachable and healthy.") + + @staticmethod + def platform_hints() -> tuple[str, ...]: + """Return actionable instructions for the current platform.""" + if sys.platform == "darwin": + return ( + "Start Docker Desktop:\n open -a Docker", + "Wait for the whale icon to stop animating before re-running.", + ) + if sys.platform.startswith("win"): + return ( + "Start Docker Desktop from the Start menu or run:\n & \"C:\\Program Files\\Docker\\Docker\\Docker Desktop.exe\"", + "If Docker Desktop reports 'no daemon', relaunch it and retry.", + ) + return ( + "Start the Docker daemon (needs sudo unless you are in the docker group):\n sudo systemctl start docker", + "Enable auto-start on boot:\n sudo systemctl enable --now docker", + "After starting, verify with:\n docker info", + ) + + class DockerBackend(SandboxBackend): """Hermetic isolation backend using Docker containers.""" @@ -32,21 +71,72 @@ def backend_name(cls) -> str: return "docker" @classmethod - def is_available(cls) -> bool: - """Check if docker binary is present and daemon is reachable.""" - if not shutil.which("docker"): - return False + def diagnose(cls) -> DockerDiagnostic: + """Probe the local Docker installation and return a structured reason. + + This is the diagnostic core used by :meth:`is_available` and surfaced + by the CLI when a requested backend fails to come up. It distinguishes + three failure modes so callers can emit actionable guidance: + binary missing on ``PATH``, daemon unreachable, and probe timeout. + """ + docker_bin = shutil.which("docker") + if docker_bin is None: + hints = ( + "Install the Docker CLI and add it to PATH:", + " https://docs.docker.com/get-docker/", + DockerDiagnostic.platform_hints()[0], + ) + return DockerDiagnostic( + available=False, + reason=( + "Docker CLI not found on PATH. The 'docker' executable is required " + "before the daemon can be reached." + ), + hints=hints, + ) + try: res = subprocess.run( ["docker", "info"], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stderr=subprocess.PIPE, timeout=3, check=False, ) - return res.returncode == 0 - except Exception: - return False + except subprocess.TimeoutExpired: + return DockerDiagnostic( + available=False, + reason=( + "Docker daemon did not respond to 'docker info' within 3s. " + "The CLI is present but the daemon is likely not running." + ), + hints=DockerDiagnostic.platform_hints(), + ) + except Exception as exc: # pragma: no cover - defensive + return DockerDiagnostic( + available=False, + reason=f"Docker probe failed unexpectedly: {exc}", + hints=DockerDiagnostic.platform_hints(), + ) + + if res.returncode != 0: + stderr_tail = (res.stderr or b"").decode("utf-8", errors="replace").strip().splitlines() + detail = stderr_tail[-1] if stderr_tail else "docker info failed" + return DockerDiagnostic( + available=False, + reason=( + f"Docker daemon is unreachable ({detail}). The CLI is present " + "but the daemon is not answering 'docker info'." + ), + hints=DockerDiagnostic.platform_hints(), + ) + + return DockerDiagnostic.ok() + + @classmethod + def is_available(cls) -> bool: + """Check if docker binary is present and daemon is reachable.""" + return cls.diagnose().available def setup(self) -> None: """Start an idle container with workspace volume mounts.""" From e073bce26fd41d6730e8869bb6c5243b7a45356d Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Sat, 17 Oct 2026 19:10:09 +0530 Subject: [PATCH 2/6] style: fix Docker hint formatting --- evalguard/sandbox/backends/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evalguard/sandbox/backends/docker.py b/evalguard/sandbox/backends/docker.py index ce85cf7..ffcfbf9 100644 --- a/evalguard/sandbox/backends/docker.py +++ b/evalguard/sandbox/backends/docker.py @@ -42,7 +42,7 @@ def platform_hints() -> tuple[str, ...]: ) if sys.platform.startswith("win"): return ( - "Start Docker Desktop from the Start menu or run:\n & \"C:\\Program Files\\Docker\\Docker\\Docker Desktop.exe\"", + 'Start Docker Desktop from the Start menu or run:\n & "C:\\Program Files\\Docker\\Docker\\Docker Desktop.exe"', "If Docker Desktop reports 'no daemon', relaunch it and retry.", ) return ( From 5dcb90f380182c5261d70b6ffbc76a5db888f85d Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Sat, 17 Oct 2026 19:11:04 +0530 Subject: [PATCH 3/6] test(sandbox): cover Docker diagnostic failure modes --- .../test_sandbox/test_backends_extended.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/test_sandbox/test_backends_extended.py b/tests/unit/test_sandbox/test_backends_extended.py index bc99329..271ac41 100644 --- a/tests/unit/test_sandbox/test_backends_extended.py +++ b/tests/unit/test_sandbox/test_backends_extended.py @@ -135,3 +135,43 @@ def test_gvisor_backend_mocked() -> None: backend.teardown() assert backend._is_running is False + +def test_docker_diagnose_reports_missing_cli() -> None: + with patch("evalguard.sandbox.backends.docker.shutil.which", return_value=None): + diagnostic = DockerBackend.diagnose() + + assert diagnostic.available is False + assert "CLI not found on PATH" in diagnostic.reason + assert any("docs.docker.com/get-docker" in hint for hint in diagnostic.hints) + assert DockerBackend.is_available() is False + + +def test_docker_diagnose_includes_daemon_error() -> None: + failed_result = MagicMock(returncode=1, stdout="", stderr=b"permission denied\\nCannot connect to the Docker daemon") + with ( + patch("evalguard.sandbox.backends.docker.shutil.which", return_value="/usr/bin/docker"), + patch("evalguard.sandbox.backends.docker.subprocess.run", return_value=failed_result), + ): + diagnostic = DockerBackend.diagnose() + + assert diagnostic.available is False + assert "Cannot connect to the Docker daemon" in diagnostic.reason + assert diagnostic.hints + + +def test_docker_diagnose_handles_probe_timeout() -> None: + import subprocess + + with ( + patch("evalguard.sandbox.backends.docker.shutil.which", return_value="/usr/bin/docker"), + patch( + "evalguard.sandbox.backends.docker.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd=["docker", "info"], timeout=3), + ), + ): + diagnostic = DockerBackend.diagnose() + + assert diagnostic.available is False + assert "within 3s" in diagnostic.reason + assert diagnostic.hints + From 81bf97fa17256d87b53739202cf2056b5d69dc9f Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Sat, 17 Oct 2026 19:14:35 +0530 Subject: [PATCH 4/6] test(sandbox): keep Docker absence mocked for availability check --- tests/unit/test_sandbox/test_backends_extended.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_sandbox/test_backends_extended.py b/tests/unit/test_sandbox/test_backends_extended.py index 271ac41..71acc6b 100644 --- a/tests/unit/test_sandbox/test_backends_extended.py +++ b/tests/unit/test_sandbox/test_backends_extended.py @@ -139,11 +139,11 @@ def test_gvisor_backend_mocked() -> None: def test_docker_diagnose_reports_missing_cli() -> None: with patch("evalguard.sandbox.backends.docker.shutil.which", return_value=None): diagnostic = DockerBackend.diagnose() + assert DockerBackend.is_available() is False assert diagnostic.available is False assert "CLI not found on PATH" in diagnostic.reason assert any("docs.docker.com/get-docker" in hint for hint in diagnostic.hints) - assert DockerBackend.is_available() is False def test_docker_diagnose_includes_daemon_error() -> None: From 0afbcfbf6c02f4e5fc05f3b7563d6af2ec51aa5c Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Sat, 17 Oct 2026 19:15:32 +0530 Subject: [PATCH 5/6] style(test): format Docker diagnostic coverage --- tests/unit/test_sandbox/test_backends_extended.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_sandbox/test_backends_extended.py b/tests/unit/test_sandbox/test_backends_extended.py index 71acc6b..e3e86de 100644 --- a/tests/unit/test_sandbox/test_backends_extended.py +++ b/tests/unit/test_sandbox/test_backends_extended.py @@ -136,6 +136,7 @@ def test_gvisor_backend_mocked() -> None: backend.teardown() assert backend._is_running is False + def test_docker_diagnose_reports_missing_cli() -> None: with patch("evalguard.sandbox.backends.docker.shutil.which", return_value=None): diagnostic = DockerBackend.diagnose() @@ -147,7 +148,9 @@ def test_docker_diagnose_reports_missing_cli() -> None: def test_docker_diagnose_includes_daemon_error() -> None: - failed_result = MagicMock(returncode=1, stdout="", stderr=b"permission denied\\nCannot connect to the Docker daemon") + failed_result = MagicMock( + returncode=1, stdout="", stderr=b"permission denied\nCannot connect to the Docker daemon" + ) with ( patch("evalguard.sandbox.backends.docker.shutil.which", return_value="/usr/bin/docker"), patch("evalguard.sandbox.backends.docker.subprocess.run", return_value=failed_result), @@ -174,4 +177,3 @@ def test_docker_diagnose_handles_probe_timeout() -> None: assert diagnostic.available is False assert "within 3s" in diagnostic.reason assert diagnostic.hints - From da9c1802c61d9258fcbf617242d60d8683136d0c Mon Sep 17 00:00:00 2001 From: Aditya Suryavanshi Date: Sat, 17 Oct 2026 19:21:53 +0530 Subject: [PATCH 6/6] fix(cli): reuse Docker preflight diagnostic --- evalguard/cli/audit.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/evalguard/cli/audit.py b/evalguard/cli/audit.py index f7d61d9..a5cc635 100644 --- a/evalguard/cli/audit.py +++ b/evalguard/cli/audit.py @@ -60,20 +60,21 @@ def run_audit( # backend is requested but cannot come up on this host. This surfaces the # structured DockerDiagnostic (probe reason + platform hints) instead of a # bare warning, and hints at the `--backend auto` graceful fallback. - if backend == "docker" and not DockerBackend.is_available(): + if backend == "docker": diag: DockerDiagnostic = DockerBackend.diagnose() - console.print( - f"\n[bold red]Backend '{backend}' requested but unavailable on this host.[/bold red]" - ) - console.print(f" [bold]Reason:[/bold] {diag.reason}") - if diag.hints: - console.print(" [bold yellow]Try one of the following:[/bold yellow]") - for hint in diag.hints: - console.print(f" - {hint}") - console.print( - "\n To run with automatic backend detection instead, retry with:\n" - " evalguard audit run --backend auto" - ) + if not diag.available: + console.print( + f"\n[bold red]Backend '{backend}' requested but unavailable on this host.[/bold red]" + ) + console.print(f" [bold]Reason:[/bold] {diag.reason}") + if diag.hints: + console.print(" [bold yellow]Try one of the following:[/bold yellow]") + for hint in diag.hints: + console.print(f" - {hint}") + console.print( + "\n To run with automatic backend detection instead, retry with:\n" + " evalguard audit run --backend auto" + ) try: audit_record: TaskAuditRecord = adapter.run_task_with_audit(