diff --git a/cmd/root/eval.go b/cmd/root/eval.go index c0e75d29d..487c4b935 100644 --- a/cmd/root/eval.go +++ b/cmd/root/eval.go @@ -43,7 +43,8 @@ func newEvalCmd() *cobra.Command { cmd.Flags().StringVar(&flags.JudgeModel, "judge-model", defaultJudgeModel, "Model to use for relevance checking (format: provider/model)") cmd.Flags().StringVar(&flags.outputDir, "output", "", "Directory for results and logs (default: /results)") cmd.Flags().StringSliceVar(&flags.Only, "only", nil, "Only run evaluations with file names matching these patterns (can be specified multiple times)") - cmd.Flags().StringVar(&flags.BaseImage, "base-image", "", "Custom base Docker image for running evaluations") + cmd.Flags().StringVar(&flags.BaseImage, "base-image", "", "Custom base image for running evaluations") + cmd.Flags().StringVar(&flags.ContainerRuntime, "container-runtime", evaluation.DefaultContainerRuntime, "Container runtime executable for building and running evaluations") cmd.Flags().BoolVar(&flags.KeepContainers, "keep-containers", false, "Keep containers after evaluation (don't use --rm)") cmd.Flags().StringSliceVarP(&flags.EnvVars, "env", "e", nil, "Environment variables to pass to container (KEY or KEY=VALUE)") cmd.Flags().IntVar(&flags.Repeat, "repeat", 1, "Number of times to repeat each evaluation (useful for computing baselines)") @@ -101,6 +102,7 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr fmt.Fprintf(logFile, "Evals dir: %s\n", evalsDir) fmt.Fprintf(logFile, "Judge model: %s\n", f.JudgeModel) fmt.Fprintf(logFile, "Concurrency: %d\n", f.Concurrency) + fmt.Fprintf(logFile, "Container runtime: %s\n", f.ContainerRuntime) fmt.Fprintf(logFile, "\n") // Create tee writer to write to both console and log file diff --git a/cmd/root/eval_test.go b/cmd/root/eval_test.go new file mode 100644 index 000000000..fb9fc567c --- /dev/null +++ b/cmd/root/eval_test.go @@ -0,0 +1,29 @@ +package root + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvalContainerRuntimeFlagDefaultsToDocker(t *testing.T) { + t.Parallel() + + cmd := newEvalCmd() + + flag := cmd.Flags().Lookup("container-runtime") + require.NotNil(t, flag, "eval must expose --container-runtime") + assert.Equal(t, "docker", flag.DefValue) +} + +func TestEvalContainerRuntimeFlagAcceptsCustomExecutable(t *testing.T) { + t.Parallel() + + cmd := newEvalCmd() + require.NoError(t, cmd.Flags().Parse([]string{"--container-runtime", "podman"})) + + value, err := cmd.Flags().GetString("container-runtime") + require.NoError(t, err) + assert.Equal(t, "podman", value) +} diff --git a/docs/features/cli/index.md b/docs/features/cli/index.md index 5292a3027..23cac6ecf 100644 --- a/docs/features/cli/index.md +++ b/docs/features/cli/index.md @@ -461,7 +461,8 @@ $ docker agent eval | [|./evals] [flags] | `--judge-model` | `anthropic/claude-opus-5` | Model for LLM-as-a-judge relevance scoring (format: `provider/model`) | | `--output ` | `/results` | Directory for results, logs, and session databases | | `--only ` | (all) | Only run evals with file names matching these patterns (repeatable) | -| `--base-image` | (default) | Custom base Docker image for eval containers | +| `--base-image` | (default) | Custom base image for eval containers | +| `--container-runtime` | `docker` | Container runtime executable for building and running evaluations (e.g. `podman`) | | `--keep-containers` | `false` | Keep containers after evaluation (don't remove with `--rm`) | | `-e, --env` | (none) | Environment variables to pass to container (`KEY` or `KEY=VALUE`, repeatable) | | `--repeat ` | `1` | Number of times to repeat each evaluation (useful for computing baselines) | @@ -476,6 +477,7 @@ $ docker agent eval agent.yaml -c 8 # 8 concurrent evaluat $ docker agent eval agent.yaml --keep-containers # keep containers for debugging $ docker agent eval agent.yaml --only "auth*" # only run matching evals $ docker agent eval agent.yaml --repeat 5 # repeat each eval 5 times +$ docker agent eval agent.yaml --container-runtime podman # use a Docker-compatible runtime such as Podman ``` See [Evaluation](../evaluation/index.md) for details on creating eval sessions and interpreting results. diff --git a/docs/features/evaluation/index.md b/docs/features/evaluation/index.md index 8b1163afd..03c36a9fc 100644 --- a/docs/features/evaluation/index.md +++ b/docs/features/evaluation/index.md @@ -15,9 +15,9 @@ _Measure agent quality with automated evaluations — tool call accuracy, respon The `docker agent eval` command runs your agent against a set of recorded sessions and scores the results. Each eval session captures a user question, the expected tool calls, and criteria the response must satisfy. Docker Agent replays the question, compares the agent's behavior to expectations, and produces a report. > [!NOTE] -> **Docker required** +> **Container runtime required** > -> Evaluations run inside Docker containers for isolation. Each eval gets a clean environment with optional setup scripts. Docker Desktop (or Docker Engine) must be running. +> Evaluations run inside containers for isolation. Each eval gets a clean environment with optional setup scripts. A running Docker-compatible container CLI/runtime is required: Docker Desktop or Docker Engine by default, or another Docker-compatible runtime such as Podman selected with `--container-runtime`. ## Quick Start @@ -39,6 +39,9 @@ $ docker agent eval agent.yaml --repeat 5 # Repeat a specific eval 5 times $ docker agent eval agent.yaml --only "auth*" --repeat 5 + +# Use a Docker-compatible runtime such as Podman +$ docker agent eval agent.yaml --container-runtime podman ``` ## Eval Directory Structure @@ -160,7 +163,8 @@ $ docker agent eval | [|./evals] | `--judge-model` | `anthropic/claude-opus-5` | Model for LLM-as-a-judge relevance scoring | | `--output` | `/results` | Directory for results, logs, and session databases | | `--only` | (all) | Only run evals with file names matching these patterns | -| `--base-image` | (default) | Custom base Docker image for eval containers (see [Custom Base Images](#custom-base-images)) | +| `--base-image` | (default) | Custom base image for eval containers (see [Custom Base Images](#custom-base-images)) | +| `--container-runtime` | `docker` | Container runtime executable for building and running evaluations (e.g. `podman`) | | `--keep-containers` | `false` | Keep containers after evaluation (don't remove with `--rm`) | | `-e, --env` | (none) | Environment variables to pass to container (`KEY` or `KEY=VALUE`) | | `--repeat` | `1` | Number of times to repeat each evaluation (useful for computing baselines) | @@ -219,7 +223,7 @@ After a run completes, Docker Agent produces: > [!TIP] > **Debugging Failed Evals** > -> Use `--keep-containers` to preserve containers after evaluation. You can then inspect them with `docker exec` to understand why an eval failed. The session database (`.db` file) contains the full conversation history for each eval. +> Use `--keep-containers` to preserve containers after evaluation. You can then inspect them with your selected runtime's `exec` command (`docker exec` by default, `podman exec` with `--container-runtime podman`) to understand why an eval failed. The session database (`.db` file) contains the full conversation history for each eval. ```bash $ docker agent eval demo.yaml ./evals diff --git a/pkg/evaluation/build.go b/pkg/evaluation/build.go index c2d9c4669..268260562 100644 --- a/pkg/evaluation/build.go +++ b/pkg/evaluation/build.go @@ -27,7 +27,7 @@ var ( dockerfileCustomTemplate = template.Must(template.New("DockerfileCustom").Parse(dockerfileCustomTmpl)) ) -// imageKey uniquely identifies a Docker image build configuration. +// imageKey uniquely identifies a container image build configuration. type imageKey struct { workingDir string image string @@ -79,7 +79,8 @@ func (r *Runner) resolveBaseImage(evals *session.EvalCriteria) string { return r.BaseImage } -// buildEvalImage builds a Docker image for an evaluation. +// buildEvalImage builds a container image for an evaluation using the +// configured container runtime. func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria) (string, error) { var buildContext string var data struct { @@ -110,16 +111,17 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria return "", fmt.Errorf("executing dockerfile template: %w", err) } - cmd := exec.CommandContext(ctx, "docker", "build", "-q", "-f-", ".") + containerRuntime := r.containerRuntimeOrDefault() + cmd := exec.CommandContext(ctx, containerRuntime, "build", "-q", "-f-", ".") cmd.Dir = buildContext cmd.Stdin = &dockerfile output, err := cmd.Output() if err != nil { if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - return "", fmt.Errorf("docker build failed: %s", string(exitErr.Stderr)) + return "", fmt.Errorf("%s build failed: %s", containerRuntime, string(exitErr.Stderr)) } - return "", fmt.Errorf("docker build failed: %w", err) + return "", fmt.Errorf("%s build failed: %w", containerRuntime, err) } return strings.TrimSpace(string(output)), nil diff --git a/pkg/evaluation/eval.go b/pkg/evaluation/eval.go index 58f127539..1dd822def 100644 --- a/pkg/evaluation/eval.go +++ b/pkg/evaluation/eval.go @@ -37,7 +37,7 @@ type Runner struct { judge *Judge runConfig *config.RuntimeConfig - // imageCache caches built Docker images by (workingDir, image) pair. + // imageCache caches built container images by (workingDir, image) pair. imageCache map[imageKey]string imageCacheMu sync.Mutex @@ -132,7 +132,7 @@ func (r *Runner) Run(ctx context.Context, ttyOut, out io.Writer, isTTY bool) ([] } } - // Pre-build all unique Docker images in parallel before running evaluations. + // Pre-build all unique container images in parallel before running evaluations. // This avoids serialized builds when multiple workers need the same image. if err := r.preBuildImages(ctx, out, evals); err != nil { return nil, fmt.Errorf("pre-building images: %w", err) @@ -246,7 +246,7 @@ func (r *Runner) loadEvalSessions(ctx context.Context) ([]InputSession, error) { return evals, nil } -// preBuildImages pre-builds all unique Docker images needed for the evaluations. +// preBuildImages pre-builds all unique container images needed for the evaluations. // Concurrent calls for the same (workingDir, image) pair are deduplicated by // getOrBuildImage's singleflight, so we simply iterate over all evals. func (r *Runner) preBuildImages(ctx context.Context, out io.Writer, evals []InputSession) error { @@ -261,7 +261,7 @@ func (r *Runner) preBuildImages(ctx context.Context, out io.Writer, evals []Inpu unique[imageKey{workingDir: criteria.WorkingDir, image: criteria.Image}] = struct{}{} } - fmt.Fprintf(out, "Pre-building %d Docker image(s)...\n", len(unique)) + fmt.Fprintf(out, "Pre-building %d container image(s)...\n", len(unique)) type buildResult struct { title string @@ -407,8 +407,8 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string, ) var env []string - // addEnv forwards a variable to the container: "-e NAME" tells docker to - // pass it through, and NAME=VALUE sets it on the docker process. + // addEnv forwards a variable to the container: "-e NAME" tells the runtime + // CLI to pass it through, and NAME=VALUE sets it on the CLI process. addEnv := func(name, value string) { args = append(args, "-e", name) env = append(env, name+"="+value) @@ -465,13 +465,14 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string, } args = append(args, questions...) - cmd := exec.CommandContext(ctx, "docker", args...) + containerRuntime := r.containerRuntimeOrDefault() + cmd := exec.CommandContext(ctx, containerRuntime, args...) cmd.Env = append(env, os.Environ()...) - // On cancellation send SIGINT instead of the default SIGKILL: the docker - // CLI proxies SIGINT to the container (SIGKILL is never proxied and would - // leave the container running daemon-side). The container's exit also - // triggers --rm removal. WaitDelay force-kills the CLI if the container - // doesn't stop in time. + // On cancellation send SIGINT instead of the default SIGKILL: the + // Docker-compatible CLI proxies SIGINT to the container (SIGKILL is never + // proxied and would leave the container running daemon-side). The + // container's exit also triggers --rm removal. WaitDelay force-kills the + // CLI if the container doesn't stop in time. cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } @@ -486,7 +487,7 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string, } if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("starting docker run: %w", err) + return nil, fmt.Errorf("starting %s run: %w", containerRuntime, err) } var stderrData []byte diff --git a/pkg/evaluation/eval_test.go b/pkg/evaluation/eval_test.go index 28f2e1bf9..465354234 100644 --- a/pkg/evaluation/eval_test.go +++ b/pkg/evaluation/eval_test.go @@ -1056,6 +1056,90 @@ func TestRunDockerAgentInContainerHelperProcess(*testing.T) { } } +func TestContainerRuntimeOrDefault(t *testing.T) { + t.Parallel() + + empty := Config{} + assert.Equal(t, "docker", empty.containerRuntimeOrDefault(), "empty config must fall back to docker") + + custom := Config{ContainerRuntime: "podman"} + assert.Equal(t, "podman", custom.containerRuntimeOrDefault()) +} + +// writeFakeContainerRuntime writes a POSIX shell script standing in for a +// Docker-compatible container runtime CLI: it records its arguments to +// argsFile and prints output on stdout. No daemon is involved. +func writeFakeContainerRuntime(t *testing.T, path, argsFile, output string) { + t.Helper() + script := "#!/bin/sh\necho \"$@\" > \"" + argsFile + "\"\necho '" + output + "'\n" + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) +} + +// TestRunDockerAgentInContainerUsesConfiguredRuntime proves that container +// runs are executed with the configured runtime executable instead of the +// docker CLI. +func TestRunDockerAgentInContainerUsesConfiguredRuntime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the fake container runtime executable is a POSIX shell script") + } + t.Parallel() + + tmpDir := t.TempDir() + argsFile := filepath.Join(tmpDir, "args") + fakeRuntime := filepath.Join(tmpDir, "fake-podman") + writeFakeContainerRuntime(t, fakeRuntime, argsFile, `{"type":"agent_choice","content":"ok"}`) + + runner := newRunner( + config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")), + &config.RuntimeConfig{EnvProviderForTests: environment.NewNoEnvProvider()}, + nil, + Config{ContainerRuntime: fakeRuntime}, + ) + + events, err := runner.runDockerAgentInContainer(t.Context(), "image-id", []string{"question"}, "") + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, "agent_choice", events[0]["type"]) + + args, err := os.ReadFile(argsFile) + require.NoError(t, err) + got := string(args) + assert.True(t, strings.HasPrefix(got, "run "), "fake runtime must receive the run subcommand, got: %s", got) + assert.Contains(t, got, "image-id") +} + +// TestBuildEvalImageUsesConfiguredRuntime proves that image builds shell out +// to the configured runtime executable instead of the docker CLI. +func TestBuildEvalImageUsesConfiguredRuntime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the fake container runtime executable is a POSIX shell script") + } + t.Parallel() + + tmpDir := t.TempDir() + argsFile := filepath.Join(tmpDir, "args") + fakeRuntime := filepath.Join(tmpDir, "fake-podman") + writeFakeContainerRuntime(t, fakeRuntime, argsFile, "sha256:fake-image-id") + + evalsDir := filepath.Join(tmpDir, "evals") + require.NoError(t, os.Mkdir(evalsDir, 0o755)) + + runner := newRunner( + config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")), + &config.RuntimeConfig{EnvProviderForTests: environment.NewNoEnvProvider()}, + nil, + Config{EvalsDir: evalsDir, ContainerRuntime: fakeRuntime}, + ) + + imageID, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{}) + require.NoError(t, err) + assert.Equal(t, "sha256:fake-image-id", imageID) + + args, err := os.ReadFile(argsFile) + require.NoError(t, err) + assert.Equal(t, "build -q -f- .", strings.TrimSpace(string(args))) +} + func TestNeedsJudge(t *testing.T) { t.Parallel() diff --git a/pkg/evaluation/save.go b/pkg/evaluation/save.go index a2cb15a94..9407b5f38 100644 --- a/pkg/evaluation/save.go +++ b/pkg/evaluation/save.go @@ -426,11 +426,12 @@ func SaveRunSessionsJSON(run *EvalRun, outputDir string) (string, error) { Timestamp: run.Timestamp, Duration: run.Duration.Round(time.Millisecond).String(), Config: RunOutputConfig{ - Agent: run.Config.AgentFilename, - JudgeModel: run.Config.JudgeModel, - Concurrency: run.Config.Concurrency, - EvalsDir: run.Config.EvalsDir, - BaseImage: run.Config.BaseImage, + Agent: run.Config.AgentFilename, + JudgeModel: run.Config.JudgeModel, + Concurrency: run.Config.Concurrency, + EvalsDir: run.Config.EvalsDir, + BaseImage: run.Config.BaseImage, + ContainerRuntime: run.Config.ContainerRuntime, }, Summary: run.Summary, Sessions: sessions, diff --git a/pkg/evaluation/save_test.go b/pkg/evaluation/save_test.go index ee4bdf352..f43047c29 100644 --- a/pkg/evaluation/save_test.go +++ b/pkg/evaluation/save_test.go @@ -291,6 +291,48 @@ func TestSaveRunSessionsJSON(t *testing.T) { assert.Equal(t, "no explanation given", sess2Loaded.EvalResult.Checks.Relevance.Results[1].Reason) } +func TestSaveRunSessionsJSONContainerRuntime(t *testing.T) { + t.Parallel() + + save := func(t *testing.T, cfg Config) []byte { + t.Helper() + run := &EvalRun{ + Name: "test-runtime-001", + Timestamp: time.Now(), + Config: cfg, + } + + sessionsPath, err := SaveRunSessionsJSON(run, t.TempDir()) + require.NoError(t, err) + + data, err := os.ReadFile(sessionsPath) + require.NoError(t, err) + return data + } + + t.Run("recorded when configured", func(t *testing.T) { + t.Parallel() + + data := save(t, Config{ContainerRuntime: "podman"}) + + var output RunOutput + require.NoError(t, json.Unmarshal(data, &output)) + assert.Equal(t, "podman", output.Config.ContainerRuntime) + }) + + t.Run("omitted when empty", func(t *testing.T) { + t.Parallel() + + data := save(t, Config{}) + + var raw struct { + Config map[string]any `json:"config"` + } + require.NoError(t, json.Unmarshal(data, &raw)) + assert.NotContains(t, raw.Config, "container_runtime") + }) +} + func TestSaveRunSessionsWithCost(t *testing.T) { t.Parallel() diff --git a/pkg/evaluation/types.go b/pkg/evaluation/types.go index b33fefa71..06172144a 100644 --- a/pkg/evaluation/types.go +++ b/pkg/evaluation/types.go @@ -1,6 +1,7 @@ package evaluation import ( + "cmp" "fmt" "time" @@ -131,25 +132,37 @@ type RunOutput struct { // RunOutputConfig captures the evaluation run configuration. type RunOutputConfig struct { - Agent string `json:"agent"` - JudgeModel string `json:"judge_model,omitempty"` - Concurrency int `json:"concurrency"` - EvalsDir string `json:"evals_dir"` - BaseImage string `json:"base_image,omitempty"` + Agent string `json:"agent"` + JudgeModel string `json:"judge_model,omitempty"` + Concurrency int `json:"concurrency"` + EvalsDir string `json:"evals_dir"` + BaseImage string `json:"base_image,omitempty"` + ContainerRuntime string `json:"container_runtime,omitempty"` } // Config holds configuration for evaluation runs. type Config struct { - AgentFilename string // Path to the agent configuration file - EvalsDir string // Directory containing evaluation files - JudgeModel string // Model for relevance checking (format: provider/model, optional) - Concurrency int // Number of concurrent runs (0 = number of CPUs) - TTYFd int // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd())) - Only []string // Only run evaluations matching these patterns - BaseImage string // Custom base Docker image for running evaluations - KeepContainers bool // If true, don't remove containers after evaluation (skip --rm) - EnvVars []string // Environment variables to pass: KEY (value from env) or KEY=VALUE (explicit) - Repeat int // Number of times to repeat each evaluation (default 1) + AgentFilename string // Path to the agent configuration file + EvalsDir string // Directory containing evaluation files + JudgeModel string // Model for relevance checking (format: provider/model, optional) + Concurrency int // Number of concurrent runs (0 = number of CPUs) + TTYFd int // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd())) + Only []string // Only run evaluations matching these patterns + BaseImage string // Custom base image for running evaluations + KeepContainers bool // If true, don't remove containers after evaluation (skip --rm) + EnvVars []string // Environment variables to pass: KEY (value from env) or KEY=VALUE (explicit) + Repeat int // Number of times to repeat each evaluation (default 1) + ContainerRuntime string // Docker-compatible container runtime executable (default "docker") +} + +// DefaultContainerRuntime is the container runtime executable used when +// Config.ContainerRuntime is empty, keeping the historical Docker behavior. +const DefaultContainerRuntime = "docker" + +// containerRuntimeOrDefault returns the container runtime executable to +// invoke, falling back to DefaultContainerRuntime when none is configured. +func (c *Config) containerRuntimeOrDefault() string { + return cmp.Or(c.ContainerRuntime, DefaultContainerRuntime) } // Session helper functions