feat(webui): modernize reference client and sandbox job workflows

cmd/odek/bg_tools.go

@@ -53,7 +53,8 @@ type bgRuntime struct {
 	// container is the sandbox container name, readable after construction
 	// (surfaces that start the sandbox after building tools bind it late,
 	// before the agent runs — jobs only spawn once the agent iterates).
-	container atomic.Value
+	container    atomic.Value
+	serveSandbox *serveSandboxLease
 }
 
 // backgroundSettingsFromResolved maps the resolved background config onto
@@ -111,8 +112,8 @@ func newBackgroundRuntime(s BackgroundSettings, sessionID, containerName string,
 		if name == "" {
 			return []string{"sh", "-c", command}, nil, nil
 		}
-		argv, followUp := wrapSandboxCommand(name, command)
-		return argv, followUp, nil
+		argv, followUp := wrapBackgroundSandboxCommand(name, command)
+		return append([]string{"docker"}, argv...), followUp, nil
 	}
 	var obs bgproc.Observer
 	if emit != nil {
@@ -379,7 +380,15 @@ func (t *bgStartTool) Call(args string) (string, error) {
 	if err := t.shell.checkApproval(p.Command, "background job"); err != nil {
 		return "", err
 	}
-	job, err := t.rt.mgr.Start(t.rt.session, p.Command, "", time.Duration(p.TimeoutSeconds)*time.Second)
+	opts := bgproc.SpawnOptions{}
+	if t.rt.serveSandbox != nil {
+		var err error
+		opts, err = t.rt.serveSandbox.acquire()
+		if err != nil {
+			return "", err
+		}
+	}
+	job, err := t.rt.mgr.StartWithOptions(t.rt.session, p.Command, "", time.Duration(p.TimeoutSeconds)*time.Second, opts)
 	if err != nil {
 		return "", err
 	}
@@ -555,3 +564,14 @@ func jobRuntimeSeconds(j bgproc.Job) float64 {
 	}
 	return end.Sub(j.StartedAt).Seconds()
 }
+
+// Background commands need their own process group inside the container so
+// the pidfile follow-up can kill descendants without affecting other jobs.
+// Run setsid as a child of a waiting shell so it is not already a process-group
+// leader and does not fork away from docker exec. This also supports BusyBox
+// setsid, which has no --wait option.
+func wrapBackgroundSandboxCommand(name, command string) ([]string, func()) {
+	argv, followUp := wrapSandboxCommand(name, command)
+	argv = append(append(append([]string{}, argv[:4]...), "sh", "-c", `setsid "$@" & wait $!`, "odek-bg"), argv[4:]...)
+	return argv, followUp
+}

cmd/odek/hermetic_test.go

@@ -42,6 +42,7 @@ func TestHermetic_TestProcessIsolatedFromOperatorConfig(t *testing.T) {
 	keep := map[string]bool{
 		"ODEK_NO_SANDBOX":               true,
 		"ODEK_E2E":                      true,
+		"ODEK_BG_SANDBOX_TEST_IMAGE":    true,
 		"ODEK_TEST_HOME":                true,
 		"ODEK_TEST_KEEP_REAL_HOME":      true,
 		"ODEK_SUPPRESS_SANDBOX_WARNING": true,

cmd/odek/main.go

@@ -2352,10 +2352,9 @@ func setupSandbox(tools []odek.Tool, cfg sandboxConfig) (containerName string, c
 		return "", nil, fmt.Errorf("failed to create sandbox container %q: %w\n  hint: make sure Docker is running, or disable sandbox with --no-sandbox", containerName, err)
 	}
 
-	cleanup = func() error {
-		fmt.Fprintf(os.Stderr, "odek: destroying sandbox container %s...\n", containerName)
-		return exec.Command("docker", "rm", "-f", containerName).Run()
-	}
+	cleanup = newSandboxCleanup(containerName, func(ctx context.Context) error {
+		return exec.CommandContext(ctx, "docker", "rm", "-f", containerName).Run()
+	})
 
 	applySandboxToolBindings(tools, containerName)
 	return containerName, cleanup, nil

cmd/odek/sandbox_cleanup.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+)
+
+// newSandboxCleanup is shared by every execution surface. Successful cleanup
+// is idempotent; failure remains retryable and is reported even when a deferred
+// Agent.Close caller cannot return the error to its caller.
+func newSandboxCleanup(container string, remove func(context.Context) error) func() error {
+	var mu sync.Mutex
+	var removed bool
+	return func() error {
+		mu.Lock()
+		defer mu.Unlock()
+		if removed {
+			return nil
+		}
+		fmt.Fprintf(os.Stderr, "odek: destroying sandbox container %s...\n", container)
+		err := retrySandboxRemoval(remove, 5*time.Second)
+		if err != nil {
+			err = fmt.Errorf("sandbox cleanup failed for %s after 3 attempts: %w", container, err)
+			fmt.Fprintf(os.Stderr, "odek: %v; container may still be running. Remove it with: docker rm -f %s\n", err, container)
+			return err
+		}
+		removed = true
+		return nil
+	}
+}
+
+// Each attempt has its own deadline so a timed-out Docker client can recover
+// on the next attempt. Callers must honor ctx; production uses CommandContext.
+func retrySandboxRemoval(remove func(context.Context) error, timeout time.Duration) error {
+	var err error
+	for attempt := 0; attempt < 3; attempt++ {
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		err = remove(ctx)
+		cancel()
+		if err == nil {
+			return nil
+		}
+		if attempt < 2 {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+	return err
+}

cmd/odek/sandbox_cleanup_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestStandaloneSandboxCleanupRetriesAndIsIdempotent(t *testing.T) {
+	var attempts atomic.Int32
+	cleanup := newSandboxCleanup("standalone-test", func(ctx context.Context) error {
+		if _, ok := ctx.Deadline(); !ok {
+			t.Error("removal has no deadline")
+		}
+		if attempts.Add(1) < 3 {
+			return errors.New("Docker temporarily unavailable")
+		}
+		return nil
+	})
+	var wg sync.WaitGroup
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := cleanup(); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+	wg.Wait()
+	if got := attempts.Load(); got != 3 {
+		t.Fatalf("removal attempts: %d, want 3", got)
+	}
+}
+
+func TestStandaloneSandboxCleanupReportsExhaustionAndCanRecover(t *testing.T) {
+	failure := errors.New("Docker unavailable")
+	attempts := 0
+	fail := true
+	cleanup := newSandboxCleanup("standalone-test", func(context.Context) error {
+		attempts++
+		if fail {
+			return failure
+		}
+		return nil
+	})
+	var got error
+	diagnostic := captureStderrDuring(t, func() { got = cleanup() })
+	if !errors.Is(got, failure) || attempts != 3 {
+		t.Fatalf("cleanup: %v, attempts %d", got, attempts)
+	}
+	if !strings.Contains(diagnostic, "after 3 attempts") || !strings.Contains(diagnostic, "docker rm -f standalone-test") {
+		t.Fatalf("missing actionable cleanup failure: %s", diagnostic)
+	}
+	fail = false
+	if err := cleanup(); err != nil {
+		t.Fatal(err)
+	}
+	if attempts != 4 {
+		t.Fatalf("failed cleanup could not retry: %d", attempts)
+	}
+}
+
+func TestSandboxRemovalRetriesTimedOutAttemptWithFreshContext(t *testing.T) {
+	attempts := 0
+	var previous context.Context
+	err := retrySandboxRemoval(func(ctx context.Context) error {
+		attempts++
+		if previous != nil && previous.Err() == nil {
+			t.Error("previous attempt context not cancelled")
+		}
+		previous = ctx
+		if ctx.Err() != nil {
+			t.Error("new attempt already cancelled")
+		}
+		if attempts == 1 {
+			<-ctx.Done()
+			return ctx.Err()
+		}
+		return nil
+	}, 10*time.Millisecond)
+	if err != nil || attempts != 2 {
+		t.Fatalf("timeout recovery: %v, attempts %d", err, attempts)
+	}
+	if previous.Err() == nil {
+		t.Fatal("successful attempt context not released")
+	}
+}