feat(eval): support configurable container runtime

cmd/root/eval.go

@@ -43,7 +43,8 @@ func newEvalCmd() *cobra.Command {
 	cmd.Flags().StringVar(&flags.JudgeModel, "judge-model", defaultJudgeModel, "Model to use for relevance checking (format: provider/model)")
 	cmd.Flags().StringVar(&flags.outputDir, "output", "", "Directory for results and logs (default: <eval-dir>/results)")
 	cmd.Flags().StringSliceVar(&flags.Only, "only", nil, "Only run evaluations with file names matching these patterns (can be specified multiple times)")
-	cmd.Flags().StringVar(&flags.BaseImage, "base-image", "", "Custom base Docker image for running evaluations")
+	cmd.Flags().StringVar(&flags.BaseImage, "base-image", "", "Custom base image for running evaluations")
+	cmd.Flags().StringVar(&flags.ContainerRuntime, "container-runtime", evaluation.DefaultContainerRuntime, "Container runtime executable for building and running evaluations")
 	cmd.Flags().BoolVar(&flags.KeepContainers, "keep-containers", false, "Keep containers after evaluation (don't use --rm)")
 	cmd.Flags().StringSliceVarP(&flags.EnvVars, "env", "e", nil, "Environment variables to pass to container (KEY or KEY=VALUE)")
 	cmd.Flags().IntVar(&flags.Repeat, "repeat", 1, "Number of times to repeat each evaluation (useful for computing baselines)")
@@ -101,6 +102,7 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr
 	fmt.Fprintf(logFile, "Evals dir: %s\n", evalsDir)
 	fmt.Fprintf(logFile, "Judge model: %s\n", f.JudgeModel)
 	fmt.Fprintf(logFile, "Concurrency: %d\n", f.Concurrency)
+	fmt.Fprintf(logFile, "Container runtime: %s\n", f.ContainerRuntime)
 	fmt.Fprintf(logFile, "\n")
 
 	// Create tee writer to write to both console and log file

cmd/root/eval_test.go

@@ -0,0 +1,29 @@
+package root
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEvalContainerRuntimeFlagDefaultsToDocker(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+
+	flag := cmd.Flags().Lookup("container-runtime")
+	require.NotNil(t, flag, "eval must expose --container-runtime")
+	assert.Equal(t, "docker", flag.DefValue)
+}
+
+func TestEvalContainerRuntimeFlagAcceptsCustomExecutable(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NoError(t, cmd.Flags().Parse([]string{"--container-runtime", "podman"}))
+
+	value, err := cmd.Flags().GetString("container-runtime")
+	require.NoError(t, err)
+	assert.Equal(t, "podman", value)
+}

docs/features/cli/index.md

@@ -461,7 +461,8 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals] [flags]
 | `--judge-model`     | `anthropic/claude-opus-5` | Model for LLM-as-a-judge relevance scoring (format: `provider/model`)      |
 | `--output <dir>`    | `<eval-dir>/results`                 | Directory for results, logs, and session databases                         |
 | `--only <pattern>`  | (all)                                | Only run evals with file names matching these patterns (repeatable)        |
-| `--base-image`      | (default)                            | Custom base Docker image for eval containers                               |
+| `--base-image`      | (default)                            | Custom base image for eval containers                                      |
+| `--container-runtime` | `docker`                           | Container runtime executable for building and running evaluations (e.g. `podman`) |
 | `--keep-containers` | `false`                              | Keep containers after evaluation (don't remove with `--rm`)                |
 | `-e, --env`         | (none)                               | Environment variables to pass to container (`KEY` or `KEY=VALUE`, repeatable) |
 | `--repeat <n>`      | `1`                                  | Number of times to repeat each evaluation (useful for computing baselines) |
@@ -476,6 +477,7 @@ $ docker agent eval agent.yaml -c 8                       # 8 concurrent evaluat
 $ docker agent eval agent.yaml --keep-containers          # keep containers for debugging
 $ docker agent eval agent.yaml --only "auth*"             # only run matching evals
 $ docker agent eval agent.yaml --repeat 5                 # repeat each eval 5 times
+$ docker agent eval agent.yaml --container-runtime podman # use a Docker-compatible runtime such as Podman
 ```
 
 See [Evaluation](../evaluation/index.md) for details on creating eval sessions and interpreting results.

docs/features/evaluation/index.md

@@ -15,9 +15,9 @@ _Measure agent quality with automated evaluations — tool call accuracy, respon
 The `docker agent eval` command runs your agent against a set of recorded sessions and scores the results. Each eval session captures a user question, the expected tool calls, and criteria the response must satisfy. Docker Agent replays the question, compares the agent's behavior to expectations, and produces a report.
 
 > [!NOTE]
-> **Docker required**
+> **Container runtime required**
 >
-> Evaluations run inside Docker containers for isolation. Each eval gets a clean environment with optional setup scripts. Docker Desktop (or Docker Engine) must be running.
+> Evaluations run inside containers for isolation. Each eval gets a clean environment with optional setup scripts. A running Docker-compatible container CLI/runtime is required: Docker Desktop or Docker Engine by default, or another Docker-compatible runtime such as Podman selected with `--container-runtime`.
 
 ## Quick Start
 
@@ -39,6 +39,9 @@ $ docker agent eval agent.yaml --repeat 5
 
 # Repeat a specific eval 5 times
 $ docker agent eval agent.yaml --only "auth*" --repeat 5
+
+# Use a Docker-compatible runtime such as Podman
+$ docker agent eval agent.yaml --container-runtime podman
 ```
 
 ## Eval Directory Structure
@@ -160,7 +163,8 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals]
 | `--judge-model`     | `anthropic/claude-opus-5` | Model for LLM-as-a-judge relevance scoring                        |
 | `--output`          | `<eval-dir>/results`  | Directory for results, logs, and session databases                |
 | `--only`            | (all)                       | Only run evals with file names matching these patterns            |
-| `--base-image`      | (default)                   | Custom base Docker image for eval containers (see [Custom Base Images](#custom-base-images)) |
+| `--base-image`      | (default)                   | Custom base image for eval containers (see [Custom Base Images](#custom-base-images)) |
+| `--container-runtime` | `docker`                  | Container runtime executable for building and running evaluations (e.g. `podman`) |
 | `--keep-containers` | `false`                     | Keep containers after evaluation (don't remove with `--rm`)       |
 | `-e, --env`         | (none)                      | Environment variables to pass to container (`KEY` or `KEY=VALUE`) |
 | `--repeat`          | `1`                         | Number of times to repeat each evaluation (useful for computing baselines) |
@@ -219,7 +223,7 @@ After a run completes, Docker Agent produces:
 > [!TIP]
 > **Debugging Failed Evals**
 >
-> Use `--keep-containers` to preserve containers after evaluation. You can then inspect them with `docker exec` to understand why an eval failed. The session database (`.db` file) contains the full conversation history for each eval.
+> Use `--keep-containers` to preserve containers after evaluation. You can then inspect them with your selected runtime's `exec` command (`docker exec` by default, `podman exec` with `--container-runtime podman`) to understand why an eval failed. The session database (`.db` file) contains the full conversation history for each eval.
 
 ```bash
 $ docker agent eval demo.yaml ./evals

pkg/evaluation/build.go

@@ -27,7 +27,7 @@ var (
 	dockerfileCustomTemplate = template.Must(template.New("DockerfileCustom").Parse(dockerfileCustomTmpl))
 )
 
-// imageKey uniquely identifies a Docker image build configuration.
+// imageKey uniquely identifies a container image build configuration.
 type imageKey struct {
 	workingDir string
 	image      string
@@ -79,7 +79,8 @@ func (r *Runner) resolveBaseImage(evals *session.EvalCriteria) string {
 	return r.BaseImage
 }
 
-// buildEvalImage builds a Docker image for an evaluation.
+// buildEvalImage builds a container image for an evaluation using the
+// configured container runtime.
 func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria) (string, error) {
 	var buildContext string
 	var data struct {
@@ -110,16 +111,17 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria
 		return "", fmt.Errorf("executing dockerfile template: %w", err)
 	}
 
-	cmd := exec.CommandContext(ctx, "docker", "build", "-q", "-f-", ".")
+	containerRuntime := r.containerRuntimeOrDefault()
+	cmd := exec.CommandContext(ctx, containerRuntime, "build", "-q", "-f-", ".")
 	cmd.Dir = buildContext
 	cmd.Stdin = &dockerfile
 
 	output, err := cmd.Output()
 	if err != nil {
 		if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
-			return "", fmt.Errorf("docker build failed: %s", string(exitErr.Stderr))
+			return "", fmt.Errorf("%s build failed: %s", containerRuntime, string(exitErr.Stderr))
 		}
-		return "", fmt.Errorf("docker build failed: %w", err)
+		return "", fmt.Errorf("%s build failed: %w", containerRuntime, err)
 	}
 
 	return strings.TrimSpace(string(output)), nil

pkg/evaluation/eval.go

@@ -37,7 +37,7 @@ type Runner struct {
 	judge       *Judge
 	runConfig   *config.RuntimeConfig
 
-	// imageCache caches built Docker images by (workingDir, image) pair.
+	// imageCache caches built container images by (workingDir, image) pair.
 	imageCache   map[imageKey]string
 	imageCacheMu sync.Mutex
 
@@ -132,7 +132,7 @@ func (r *Runner) Run(ctx context.Context, ttyOut, out io.Writer, isTTY bool) ([]
 		}
 	}
 
-	// Pre-build all unique Docker images in parallel before running evaluations.
+	// Pre-build all unique container images in parallel before running evaluations.
 	// This avoids serialized builds when multiple workers need the same image.
 	if err := r.preBuildImages(ctx, out, evals); err != nil {
 		return nil, fmt.Errorf("pre-building images: %w", err)
@@ -246,7 +246,7 @@ func (r *Runner) loadEvalSessions(ctx context.Context) ([]InputSession, error) {
 	return evals, nil
 }
 
-// preBuildImages pre-builds all unique Docker images needed for the evaluations.
+// preBuildImages pre-builds all unique container images needed for the evaluations.
 // Concurrent calls for the same (workingDir, image) pair are deduplicated by
 // getOrBuildImage's singleflight, so we simply iterate over all evals.
 func (r *Runner) preBuildImages(ctx context.Context, out io.Writer, evals []InputSession) error {
@@ -261,7 +261,7 @@ func (r *Runner) preBuildImages(ctx context.Context, out io.Writer, evals []Inpu
 		unique[imageKey{workingDir: criteria.WorkingDir, image: criteria.Image}] = struct{}{}
 	}
 
-	fmt.Fprintf(out, "Pre-building %d Docker image(s)...\n", len(unique))
+	fmt.Fprintf(out, "Pre-building %d container image(s)...\n", len(unique))
 
 	type buildResult struct {
 		title string
@@ -407,8 +407,8 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string,
 	)
 
 	var env []string
-	// addEnv forwards a variable to the container: "-e NAME" tells docker to
-	// pass it through, and NAME=VALUE sets it on the docker process.
+	// addEnv forwards a variable to the container: "-e NAME" tells the runtime
+	// CLI to pass it through, and NAME=VALUE sets it on the CLI process.
 	addEnv := func(name, value string) {
 		args = append(args, "-e", name)
 		env = append(env, name+"="+value)
@@ -465,13 +465,14 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string,
 	}
 	args = append(args, questions...)
 
-	cmd := exec.CommandContext(ctx, "docker", args...)
+	containerRuntime := r.containerRuntimeOrDefault()
+	cmd := exec.CommandContext(ctx, containerRuntime, args...)
 	cmd.Env = append(env, os.Environ()...)
-	// On cancellation send SIGINT instead of the default SIGKILL: the docker
-	// CLI proxies SIGINT to the container (SIGKILL is never proxied and would
-	// leave the container running daemon-side). The container's exit also
-	// triggers --rm removal. WaitDelay force-kills the CLI if the container
-	// doesn't stop in time.
+	// On cancellation send SIGINT instead of the default SIGKILL: the
+	// Docker-compatible CLI proxies SIGINT to the container (SIGKILL is never
+	// proxied and would leave the container running daemon-side). The
+	// container's exit also triggers --rm removal. WaitDelay force-kills the
+	// CLI if the container doesn't stop in time.
 	cmd.Cancel = func() error {
 		return cmd.Process.Signal(os.Interrupt)
 	}
@@ -486,7 +487,7 @@ func (r *Runner) runDockerAgentInContainer(ctx context.Context, imageID string,
 	}
 
 	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("starting docker run: %w", err)
+		return nil, fmt.Errorf("starting %s run: %w", containerRuntime, err)
 	}
 
 	var stderrData []byte

pkg/evaluation/eval_test.go

@@ -1056,6 +1056,90 @@ func TestRunDockerAgentInContainerHelperProcess(*testing.T) {
 	}
 }
 
+func TestContainerRuntimeOrDefault(t *testing.T) {
+	t.Parallel()
+
+	empty := Config{}
+	assert.Equal(t, "docker", empty.containerRuntimeOrDefault(), "empty config must fall back to docker")
+
+	custom := Config{ContainerRuntime: "podman"}
+	assert.Equal(t, "podman", custom.containerRuntimeOrDefault())
+}
+
+// writeFakeContainerRuntime writes a POSIX shell script standing in for a
+// Docker-compatible container runtime CLI: it records its arguments to
+// argsFile and prints output on stdout. No daemon is involved.
+func writeFakeContainerRuntime(t *testing.T, path, argsFile, output string) {
+	t.Helper()
+	script := "#!/bin/sh\necho \"$@\" > \"" + argsFile + "\"\necho '" + output + "'\n"
+	require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
+}
+
+// TestRunDockerAgentInContainerUsesConfiguredRuntime proves that container
+// runs are executed with the configured runtime executable instead of the
+// docker CLI.
+func TestRunDockerAgentInContainerUsesConfiguredRuntime(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("the fake container runtime executable is a POSIX shell script")
+	}
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+	argsFile := filepath.Join(tmpDir, "args")
+	fakeRuntime := filepath.Join(tmpDir, "fake-podman")
+	writeFakeContainerRuntime(t, fakeRuntime, argsFile, `{"type":"agent_choice","content":"ok"}`)
+
+	runner := newRunner(
+		config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")),
+		&config.RuntimeConfig{EnvProviderForTests: environment.NewNoEnvProvider()},
+		nil,
+		Config{ContainerRuntime: fakeRuntime},
+	)
+
+	events, err := runner.runDockerAgentInContainer(t.Context(), "image-id", []string{"question"}, "")
+	require.NoError(t, err)
+	require.Len(t, events, 1)
+	assert.Equal(t, "agent_choice", events[0]["type"])
+
+	args, err := os.ReadFile(argsFile)
+	require.NoError(t, err)
+	got := string(args)
+	assert.True(t, strings.HasPrefix(got, "run "), "fake runtime must receive the run subcommand, got: %s", got)
+	assert.Contains(t, got, "image-id")
+}
+
+// TestBuildEvalImageUsesConfiguredRuntime proves that image builds shell out
+// to the configured runtime executable instead of the docker CLI.
+func TestBuildEvalImageUsesConfiguredRuntime(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("the fake container runtime executable is a POSIX shell script")
+	}
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+	argsFile := filepath.Join(tmpDir, "args")
+	fakeRuntime := filepath.Join(tmpDir, "fake-podman")
+	writeFakeContainerRuntime(t, fakeRuntime, argsFile, "sha256:fake-image-id")
+
+	evalsDir := filepath.Join(tmpDir, "evals")
+	require.NoError(t, os.Mkdir(evalsDir, 0o755))
+
+	runner := newRunner(
+		config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")),
+		&config.RuntimeConfig{EnvProviderForTests: environment.NewNoEnvProvider()},
+		nil,
+		Config{EvalsDir: evalsDir, ContainerRuntime: fakeRuntime},
+	)
+
+	imageID, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{})
+	require.NoError(t, err)
+	assert.Equal(t, "sha256:fake-image-id", imageID)
+
+	args, err := os.ReadFile(argsFile)
+	require.NoError(t, err)
+	assert.Equal(t, "build -q -f- .", strings.TrimSpace(string(args)))
+}
+
 func TestNeedsJudge(t *testing.T) {
 	t.Parallel()
 

pkg/evaluation/save.go

@@ -426,11 +426,12 @@ func SaveRunSessionsJSON(run *EvalRun, outputDir string) (string, error) {
 		Timestamp: run.Timestamp,
 		Duration:  run.Duration.Round(time.Millisecond).String(),
 		Config: RunOutputConfig{
-			Agent:       run.Config.AgentFilename,
-			JudgeModel:  run.Config.JudgeModel,
-			Concurrency: run.Config.Concurrency,
-			EvalsDir:    run.Config.EvalsDir,
-			BaseImage:   run.Config.BaseImage,
+			Agent:            run.Config.AgentFilename,
+			JudgeModel:       run.Config.JudgeModel,
+			Concurrency:      run.Config.Concurrency,
+			EvalsDir:         run.Config.EvalsDir,
+			BaseImage:        run.Config.BaseImage,
+			ContainerRuntime: run.Config.ContainerRuntime,
 		},
 		Summary:  run.Summary,
 		Sessions: sessions,

pkg/evaluation/save_test.go

@@ -291,6 +291,48 @@ func TestSaveRunSessionsJSON(t *testing.T) {
 	assert.Equal(t, "no explanation given", sess2Loaded.EvalResult.Checks.Relevance.Results[1].Reason)
 }
 
+func TestSaveRunSessionsJSONContainerRuntime(t *testing.T) {
+	t.Parallel()
+
+	save := func(t *testing.T, cfg Config) []byte {
+		t.Helper()
+		run := &EvalRun{
+			Name:      "test-runtime-001",
+			Timestamp: time.Now(),
+			Config:    cfg,
+		}
+
+		sessionsPath, err := SaveRunSessionsJSON(run, t.TempDir())
+		require.NoError(t, err)
+
+		data, err := os.ReadFile(sessionsPath)
+		require.NoError(t, err)
+		return data
+	}
+
+	t.Run("recorded when configured", func(t *testing.T) {
+		t.Parallel()
+
+		data := save(t, Config{ContainerRuntime: "podman"})
+
+		var output RunOutput
+		require.NoError(t, json.Unmarshal(data, &output))
+		assert.Equal(t, "podman", output.Config.ContainerRuntime)
+	})
+
+	t.Run("omitted when empty", func(t *testing.T) {
+		t.Parallel()
+
+		data := save(t, Config{})
+
+		var raw struct {
+			Config map[string]any `json:"config"`
+		}
+		require.NoError(t, json.Unmarshal(data, &raw))
+		assert.NotContains(t, raw.Config, "container_runtime")
+	})
+}
+
 func TestSaveRunSessionsWithCost(t *testing.T) {
 	t.Parallel()