From 9c956f2b1ed791ad1cf116a50f61eab72c1f1377 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:30:06 +0000 Subject: [PATCH 1/3] macOS guests 5/6: ingress for macOS guests Route ingress to macOS guests the same way as Linux guests: Caddy resolves the target through hypeman's DNS server and proxies to the guest's IP, which for a macOS guest is its current vmnet lease. Caddy and DNS now start on macOS-only servers, and the API and resolver no longer refuse macOS targets. The Caddy daemon manager finds and checks its process with ps on macOS, which has no /proc. macOS builds still do not embed Caddy: make build-caddy-darwin builds the same version and DNS modules into bin/caddy, and hypeman looks for it next to its own binary before PATH. --- Makefile | 9 ++- cmd/api/api/ingress.go | 6 -- cmd/api/api/macos.go | 30 -------- cmd/api/api/macos_test.go | 41 +--------- cmd/api/main.go | 12 +-- docs/macos-experimental.md | 54 ++++++++++++- docs/macos-guest-agent.md | 4 +- docs/macos-support.md | 2 +- lib/ingress/binaries_darwin.go | 16 ++-- lib/ingress/daemon.go | 69 ----------------- lib/ingress/daemon_darwin.go | 64 ++++++++++++++++ lib/ingress/daemon_darwin_test.go | 36 +++++++++ lib/ingress/daemon_linux.go | 80 ++++++++++++++++++++ lib/instances/ingress_resolver.go | 6 -- lib/instances/ingress_resolver_macos_test.go | 13 +++- 15 files changed, 266 insertions(+), 176 deletions(-) create mode 100644 lib/ingress/daemon_darwin.go create mode 100644 lib/ingress/daemon_darwin_test.go create mode 100644 lib/ingress/daemon_linux.go diff --git a/Makefile b/Makefile index 99550631f..5e7a4cab0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded +.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy build-caddy-darwin ensure-caddy-binaries release-prep clean build-embedded # Directory where local binaries will be installed BIN_DIR ?= $(CURDIR)/bin @@ -93,6 +93,13 @@ build-caddy-binaries: $(XCADDY) @chmod +x lib/ingress/binaries/caddy/$(CADDY_VERSION)/*/caddy @echo "Caddy binaries built successfully with DNS modules" +# Build Caddy with the same DNS modules for macOS hosts, which find it next to +# hypeman (bin/caddy) or on PATH instead of embedding it. +build-caddy-darwin: $(XCADDY) | $(BIN_DIR) + GOOS=darwin GOARCH=arm64 $(XCADDY) build $(CADDY_VERSION) \ + $(CADDY_DNS_MODULES) \ + --output $(BIN_DIR)/caddy + # Build Caddy for current architecture only (faster for development) build-caddy: $(XCADDY) @echo "Building Caddy $(CADDY_VERSION) with DNS modules for current architecture..." diff --git a/cmd/api/api/ingress.go b/cmd/api/api/ingress.go index a58854802..da6678da7 100644 --- a/cmd/api/api/ingress.go +++ b/cmd/api/api/ingress.go @@ -36,12 +36,6 @@ func (s *ApiService) ListIngresses(ctx context.Context, request oapi.ListIngress // CreateIngress creates a new ingress resource func (s *ApiService) CreateIngress(ctx context.Context, request oapi.CreateIngressRequestObject) (oapi.CreateIngressResponseObject, error) { - if s.Config != nil && s.Config.MacOSOnly { - return oapi.CreateIngress501JSONResponse{Code: "unsupported", Message: "ingress is not implemented in macOS-only mode"}, nil - } - if reason := s.macOSIngressTargetReason(ctx, request.Body.Rules); reason != "" { - return oapi.CreateIngress501JSONResponse{Code: "unsupported", Message: reason}, nil - } log := logger.FromContext(ctx) // Convert OAPI request to domain request diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 9dfa1e9fd..5dc42cbdb 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -10,8 +10,6 @@ import ( "runtime" "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/ingress" - "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/oapi" ) @@ -35,34 +33,6 @@ func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error return img.MacOS != nil, nil } -// macOSIngressTargetReason refuses routes to macOS guests. Their addresses are NAT -// leases, and no ingress path has been proven for them, so mixed-mode servers must -// not route public traffic to them. A target that cannot be looked up is refused. -// Pattern hostnames resolve their target per request, where the ingress resolver -// refuses macOS guests, and a missing target is left to the ingress manager's 400. -func (s *ApiService) macOSIngressTargetReason(ctx context.Context, rules []oapi.IngressRule) string { - if !macOSHost || s.InstanceManager == nil { - return "" - } - for _, rule := range rules { - match := ingress.IngressMatch{Hostname: rule.Match.Hostname} - if match.IsPattern() { - continue - } - inst, err := s.InstanceManager.GetInstance(ctx, rule.Target.Instance) - if errors.Is(err, instances.ErrNotFound) { - continue - } - if err != nil { - return "cannot verify ingress target is not a macOS guest" - } - if inst.MacOS != nil { - return "ingress cannot target macOS guests" - } - } - return "" -} - // Linux defaults for CreateInstanceRequest in openapi.yaml. const ( linuxDefaultSize = "1GB" diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index 69743fe56..ce19cce6b 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -2,8 +2,6 @@ package api import ( "context" - "errors" - "fmt" "io" "net/http" "net/http/httptest" @@ -18,14 +16,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestMacOSOnlyRejectsBuildersAndIngress(t *testing.T) { +func TestMacOSOnlyRejectsBuilders(t *testing.T) { s := &ApiService{Config: &config.Config{MacOSOnly: true}} build, err := s.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{}) require.NoError(t, err) require.IsType(t, oapi.CreateBuild501JSONResponse{}, build) - ingress, err := s.CreateIngress(context.Background(), oapi.CreateIngressRequestObject{}) - require.NoError(t, err) - require.IsType(t, oapi.CreateIngress501JSONResponse{}, ingress) } func TestMacOSExecRejectedBeforeWebsocketUpgrade(t *testing.T) { @@ -177,37 +172,3 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } - -type macOSIngressTargetManager struct { - instances.Manager - inst *instances.Instance - err error -} - -func (m macOSIngressTargetManager) GetInstance(context.Context, string) (*instances.Instance, error) { - return m.inst, m.err -} - -func TestIngressRejectsMacOSTargets(t *testing.T) { - useMacOSHost(t) - rule := oapi.IngressRule{Match: oapi.IngressMatch{Hostname: "app.example.test"}, Target: oapi.IngressTarget{Instance: "mac", Port: 8080}} - macos := &instances.Instance{StoredMetadata: instances.StoredMetadata{MacOS: &images.MacOSImage{}}} - s := &ApiService{InstanceManager: macOSIngressTargetManager{inst: macos}} - require.Contains(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "macOS") - - linux := &instances.Instance{} - s = &ApiService{InstanceManager: macOSIngressTargetManager{inst: linux}} - require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule})) - - s = &ApiService{InstanceManager: macOSIngressTargetManager{err: errors.New("lookup failed")}} - require.NotEmpty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "an unverifiable target is refused") - - s = &ApiService{InstanceManager: macOSIngressTargetManager{err: fmt.Errorf("lookup: %w", instances.ErrNotFound)}} - require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "the ingress manager reports a missing target as 400") - - pattern := rule - pattern.Match.Hostname = "{instance}.example.test" - pattern.Target.Instance = "{instance}" - s = &ApiService{InstanceManager: macOSIngressTargetManager{err: errors.New("lookup failed")}} - require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{pattern}), "pattern targets resolve per request, where the resolver refuses macOS guests") -} diff --git a/cmd/api/main.go b/cmd/api/main.go index 47bc41a3a..347b395ff 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -407,15 +407,11 @@ func run() error { // Initialize ingress manager (starts Caddy daemon and DNS server for dynamic upstreams) logger.Info("Initializing ingress manager...") - if !cfg.MacOSOnly { - if err := app.IngressManager.Initialize(app.Ctx); err != nil { - logger.Error("failed to initialize ingress manager", "error", err) - return fmt.Errorf("initialize ingress manager: %w", err) - } - logger.Info("Ingress manager initialized", "listen_addr", cfg.Caddy.ListenAddress, "admin", app.IngressManager.AdminURL()) - } else { - logger.Info("macOS-only mode: ingress is unsupported; skipping Caddy and DNS startup") + if err := app.IngressManager.Initialize(app.Ctx); err != nil { + logger.Error("failed to initialize ingress manager", "error", err) + return fmt.Errorf("initialize ingress manager: %w", err) } + logger.Info("Ingress manager initialized", "listen_addr", cfg.Caddy.ListenAddress, "admin", app.IngressManager.AdminURL()) // Create router r := chi.NewRouter() diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 0c183a0e9..26d1416e1 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -110,13 +110,59 @@ Unsupported operations are rejected explicitly. The status of each operation is [macos-support.md](macos-support.md), not repeated here. An opt-in `macos_only: true` config requires Darwin/arm64 and VZ, refuses Linux -instance creates/starts and skips Linux kernel/initrd downloads, builders and -Caddy/DNS initialization. Ingress creation and builds are rejected in that mode. +instance creates/starts and skips Linux kernel/initrd downloads and builders. +Builds are rejected in that mode. Caddy and DNS start as on Linux. The default is false. `listen_address: 127.0.0.1` can bind the HTTP API locally. No host sudo, bridge creation or launchd installation is required for the spike. For controlled Chrome testing after autologin, use `open -na "Google Chrome"` with a dedicated `--user-data-dir` and guest-local `--remote-debugging-port=9222`. macOS can reopen Chrome after a power-off without the original debugging flags; -`open -a` may just activate that existing process and ignore the flags. Tunnel -CDP via SSH rather than exposing it on the guest network. +`open -a` may just activate that existing process and ignore the flags. To reach +CDP from off the host, route it through ingress (below) or tunnel it via SSH. + +## Ingress + +Ingress routes to macOS guests as it does to Linux guests: Caddy on the host +resolves the target instance through hypeman's DNS server on each request and +proxies to the guest's IP. For a macOS guest that IP is its current vmnet DHCP +lease, the address the instance reports. A guest that is not running, or has no +lease, has no upstream, and Caddy returns 502. As on Linux, the guest service must +listen on the guest's network interface, not only on loopback. + +Caddy is not embedded in macOS builds. `make build-caddy-darwin` builds the same +Caddy version and DNS modules as Linux into `bin/caddy`. Hypeman looks for `caddy` +next to its own binary, then on `PATH`, and fails to start without it. + +Caddy needs local network access to reach guests. On macOS 15.5 and later, allow +the vmnet subnet for every process once per host, then restart (Apple TN3179): + +```sh +sudo defaults write com.apple.network.local-network AllowedEthernetLocalNetworkAddresses -array "192.168.64.0/24" +``` + +Without it, only root has access. On the QA host (macOS 27.0), a launchd daemon +running as a regular user was refused, and so was a hypeman left running after the +SSH session that started it ended: dials to guests failed with "no route to host" +and Caddy returned 502. With the subnet allowed, the same daemon reached guests. + +Starting a macOS guest also needs the hypeman user logged in on the host. After a +host restart with nobody logged in, start failed with `VZErrorDomain Code=-9` +("Unable to access security information") and succeeded once the user logged in. + +Saving a macOS guest's memory (standby and memory snapshots) also needs that +user's screen unlocked. VZ encrypts saved state with a Secure Enclave key that +macOS creates only while the user's data is unlocked, and a locked screen locks +it. On the QA host, standby failed with `VZErrorDomain Code=11` ("permission +denied"; the log shows `errSecInteractionNotAllowed` from the key creation) until +the screen was unlocked. Turn off the automatic screen lock for that user: + +```sh +sysadminctl -screenLock off -password - +``` + +On the QA host (macOS 27.0), vmnet attaches each guest to the NAT bridge as a +private member: guests cannot reach each other, but they can reach anything the +host listens on, including Caddy's listeners, sshd and Screen Sharing. Hosts that +run guests for different tenants should not expose other host services on the +bridge. Ingress hostnames reach a guest through Caddy, as they do from outside. diff --git a/docs/macos-guest-agent.md b/docs/macos-guest-agent.md index 95ebe9b5b..8e09d2859 100644 --- a/docs/macos-guest-agent.md +++ b/docs/macos-guest-agent.md @@ -98,8 +98,8 @@ at boot; the host does not call `GetAgentInfo` for them. Permission, cancellation, and command errors are reported. An RPC response is not proof the VMM has exited: the host must wait for teardown. - **Network identity reconfiguration:** returns gRPC `Unimplemented` on Darwin. - Current VZ NAT uses guest DHCP; no static-address, MAC-rekey, ingress, or network - policy parity is claimed. + Current VZ NAT uses guest DHCP; no static-address, MAC-rekey or network policy + parity is claimed. - **GPU status:** Linux NVIDIA initialization reporting is not macOS graphics readiness. A Darwin guest without that device reports the existing unknown state. diff --git a/docs/macos-support.md b/docs/macos-support.md index a57072825..e0a6b529a 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -24,7 +24,7 @@ owns the behavior. | Update (PATCH), VM restart policy | Partial | Expiration/TTL and `restart_policy: always` or `never` are supported. Manual stop suppresses automatic restart. A shim that dies without removing its socket (for example, SIGKILL) reads as Stopped, so `always` restarts it. `on_failure` is rejected: no workload exit status is available. | PR 1 | | Exec health checks | Agent-gated | Supported at create and PATCH; `type: none` disables them. HTTP/TCP checks remain unsupported. | PR 3 | | Volumes, env and commands, credentials and auto-standby policies, passthrough, shaping | Unsupported | Rejected at create, start, or update. | — | -| Ingress | Unsupported | Refused in macOS-only mode and for macOS targets. | — | +| Ingress | Experimental | Routes to a running guest's vmnet lease, as on Linux, including in macOS-only mode. Caddy is not embedded on macOS (`make build-caddy-darwin`). New path needs native QA. | PR 5 | | Snapshot schedules | Experimental | State-based stopped or memory capture, existing cadence/retention, host save/restore gate and live configuration probe. New path needs native QA. | PR 4 | Limits that apply everywhere: one active VM per machine identifier, no clean-base diff --git a/lib/ingress/binaries_darwin.go b/lib/ingress/binaries_darwin.go index 1a2ba408b..f9570e2bb 100644 --- a/lib/ingress/binaries_darwin.go +++ b/lib/ingress/binaries_darwin.go @@ -4,7 +4,9 @@ package ingress import ( "fmt" + "os" "os/exec" + "path/filepath" "github.com/kernel/hypeman/lib/paths" ) @@ -13,13 +15,17 @@ import ( const CaddyVersion = "v2.10.2" // ErrCaddyNotEmbedded indicates Caddy is not embedded on macOS. -// Users should install Caddy via Homebrew or download from caddyserver.com. -var ErrCaddyNotEmbedded = fmt.Errorf("caddy binary is not embedded on macOS; install via: brew install caddy") +var ErrCaddyNotEmbedded = fmt.Errorf("caddy binary is not embedded on macOS; build it with `make build-caddy-darwin` and install it next to hypeman or on PATH") -// ExtractCaddyBinary on macOS attempts to find Caddy in PATH. +// ExtractCaddyBinary on macOS finds Caddy next to the hypeman binary, then on PATH. // Unlike Linux, we don't embed the binary on macOS. func ExtractCaddyBinary(p *paths.Paths) (string, error) { - // Try to find caddy in PATH + if exe, err := os.Executable(); err == nil { + path := filepath.Join(filepath.Dir(exe), "caddy") + if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0111 != 0 { + return path, nil + } + } path, err := exec.LookPath("caddy") if err != nil { return "", ErrCaddyNotEmbedded @@ -27,7 +33,7 @@ func ExtractCaddyBinary(p *paths.Paths) (string, error) { return path, nil } -// GetCaddyBinaryPath returns path to Caddy, looking in PATH on macOS. +// GetCaddyBinaryPath returns path to Caddy on macOS. func GetCaddyBinaryPath(p *paths.Paths) (string, error) { return ExtractCaddyBinary(p) } diff --git a/lib/ingress/daemon.go b/lib/ingress/daemon.go index fa0eb3ada..7e0bbd3f0 100644 --- a/lib/ingress/daemon.go +++ b/lib/ingress/daemon.go @@ -382,72 +382,3 @@ func (d *CaddyDaemon) isAdminResponding() bool { // Caddy returns 200 for /config/ when running return resp.StatusCode == http.StatusOK } - -// isProcessRunning checks if a process with the given PID is running. -// Returns false for zombie processes (which have exited but not been reaped). -func (d *CaddyDaemon) isProcessRunning(pid int) bool { - // Check if process exists - proc, err := os.FindProcess(pid) - if err != nil { - return false - } - err = proc.Signal(syscall.Signal(0)) - if err != nil { - return false - } - - // Check if it's a zombie process by reading /proc/[pid]/stat - // A zombie has state 'Z' and shouldn't be considered "running" - statPath := fmt.Sprintf("/proc/%d/stat", pid) - data, err := os.ReadFile(statPath) - if err != nil { - // Can't read stat, process might have exited - return false - } - - // Format: pid (comm) state ... - // Find the state character after the closing parenthesis - statStr := string(data) - closeParenIdx := strings.LastIndex(statStr, ")") - if closeParenIdx == -1 || closeParenIdx+2 >= len(statStr) { - return false - } - state := statStr[closeParenIdx+2] // Skip ") " to get state char - - // Z = zombie, X = dead - return state != 'Z' && state != 'X' -} - -// findCaddyPID tries to find the Caddy process PID by scanning /proc. -// It matches both the binary name and our specific config path to avoid -// colliding with other Caddy instances or other hypeman instances on the same server. -func (d *CaddyDaemon) findCaddyPID() int { - entries, err := os.ReadDir("/proc") - if err != nil { - return 0 - } - - configPath := d.paths.CaddyConfig() - for _, entry := range entries { - if !entry.IsDir() { - continue - } - pid, err := strconv.Atoi(entry.Name()) - if err != nil { - continue - } - - cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)) - if err != nil { - continue - } - - cmdStr := string(cmdline) - // Match caddy run command with our specific config path - if strings.Contains(cmdStr, "caddy") && strings.Contains(cmdStr, "run") && strings.Contains(cmdStr, configPath) { - return pid - } - } - - return 0 -} diff --git a/lib/ingress/daemon_darwin.go b/lib/ingress/daemon_darwin.go new file mode 100644 index 000000000..34c44860a --- /dev/null +++ b/lib/ingress/daemon_darwin.go @@ -0,0 +1,64 @@ +//go:build darwin + +package ingress + +import ( + "bufio" + "bytes" + "context" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" +) + +// isProcessRunning checks if a process with the given PID is running. +// Returns false for zombie processes (which have exited but not been reaped). +func (d *CaddyDaemon) isProcessRunning(pid int) bool { + proc, err := os.FindProcess(pid) + if err != nil || proc.Signal(syscall.Signal(0)) != nil { + return false + } + out, err := ps("-o", "stat=", "-p", strconv.Itoa(pid)) + if err != nil { + return false + } + state := strings.TrimSpace(string(out)) + return state != "" && !strings.HasPrefix(state, "Z") +} + +// findCaddyPID finds the Caddy process running our config. macOS has no /proc, so +// it scans ps output the same way the Linux version scans cmdlines. +func (d *CaddyDaemon) findCaddyPID() int { + out, err := ps("-axww", "-o", "pid=,command=") + if err != nil { + return 0 + } + return caddyPIDFromPS(out, d.paths.CaddyConfig()) +} + +func caddyPIDFromPS(out []byte, configPath string) int { + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + pidField, cmd, ok := strings.Cut(strings.TrimSpace(scanner.Text()), " ") + if !ok { + continue + } + pid, err := strconv.Atoi(pidField) + if err != nil { + continue + } + if strings.Contains(cmd, "caddy") && strings.Contains(cmd, "run") && strings.Contains(cmd, configPath) { + return pid + } + } + return 0 +} + +func ps(args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return exec.CommandContext(ctx, "/bin/ps", args...).Output() +} diff --git a/lib/ingress/daemon_darwin_test.go b/lib/ingress/daemon_darwin_test.go new file mode 100644 index 000000000..ff0cae913 --- /dev/null +++ b/lib/ingress/daemon_darwin_test.go @@ -0,0 +1,36 @@ +//go:build darwin + +package ingress + +import ( + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCaddyPIDFromPS(t *testing.T) { + out := []byte(` 101 /usr/sbin/sshd + 202 /opt/caddy run --config /other/caddy.json + 303 /Users/me/bin/caddy run --config /data/caddy/caddy.json --resume +`) + require.Equal(t, 303, caddyPIDFromPS(out, "/data/caddy/caddy.json")) + require.Zero(t, caddyPIDFromPS(out, "/missing/caddy.json")) +} + +func TestIsProcessRunningOnDarwin(t *testing.T) { + d := &CaddyDaemon{} + require.True(t, d.isProcessRunning(os.Getpid())) + + cmd := exec.Command("/bin/sleep", "60") + require.NoError(t, cmd.Start()) + pid := cmd.Process.Pid + require.True(t, d.isProcessRunning(pid)) + require.NoError(t, cmd.Process.Kill()) + // Not yet reaped, so the process is a zombie. + require.Eventually(t, func() bool { return !d.isProcessRunning(pid) }, 5*time.Second, 10*time.Millisecond, "a zombie is not running") + _ = cmd.Wait() + require.False(t, d.isProcessRunning(pid)) +} diff --git a/lib/ingress/daemon_linux.go b/lib/ingress/daemon_linux.go new file mode 100644 index 000000000..6ea4ae8b8 --- /dev/null +++ b/lib/ingress/daemon_linux.go @@ -0,0 +1,80 @@ +//go:build !darwin + +package ingress + +import ( + "fmt" + "os" + "strconv" + "strings" + "syscall" +) + +// isProcessRunning checks if a process with the given PID is running. +// Returns false for zombie processes (which have exited but not been reaped). +func (d *CaddyDaemon) isProcessRunning(pid int) bool { + // Check if process exists + proc, err := os.FindProcess(pid) + if err != nil { + return false + } + err = proc.Signal(syscall.Signal(0)) + if err != nil { + return false + } + + // Check if it's a zombie process by reading /proc/[pid]/stat + // A zombie has state 'Z' and shouldn't be considered "running" + statPath := fmt.Sprintf("/proc/%d/stat", pid) + data, err := os.ReadFile(statPath) + if err != nil { + // Can't read stat, process might have exited + return false + } + + // Format: pid (comm) state ... + // Find the state character after the closing parenthesis + statStr := string(data) + closeParenIdx := strings.LastIndex(statStr, ")") + if closeParenIdx == -1 || closeParenIdx+2 >= len(statStr) { + return false + } + state := statStr[closeParenIdx+2] // Skip ") " to get state char + + // Z = zombie, X = dead + return state != 'Z' && state != 'X' +} + +// findCaddyPID tries to find the Caddy process PID by scanning /proc. +// It matches both the binary name and our specific config path to avoid +// colliding with other Caddy instances or other hypeman instances on the same server. +func (d *CaddyDaemon) findCaddyPID() int { + entries, err := os.ReadDir("/proc") + if err != nil { + return 0 + } + + configPath := d.paths.CaddyConfig() + for _, entry := range entries { + if !entry.IsDir() { + continue + } + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + + cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)) + if err != nil { + continue + } + + cmdStr := string(cmdline) + // Match caddy run command with our specific config path + if strings.Contains(cmdStr, "caddy") && strings.Contains(cmdStr, "run") && strings.Contains(cmdStr, configPath) { + return pid + } + } + + return 0 +} diff --git a/lib/instances/ingress_resolver.go b/lib/instances/ingress_resolver.go index 3379329f8..17c2bf7a9 100644 --- a/lib/instances/ingress_resolver.go +++ b/lib/instances/ingress_resolver.go @@ -35,12 +35,6 @@ func (r *IngressResolver) ResolveInstanceIP(ctx context.Context, nameOrID string return "", fmt.Errorf("instance not found: %s", nameOrID) } - // macOS guests sit behind vmnet NAT, and their addresses are observed leases. Ingress - // must not route to them, and the public route must not reach them either. - if inst.MacOS != nil { - return "", fmt.Errorf("instance %s is a macOS guest and is not an ingress target", nameOrID) - } - // Check if instance has network enabled if !inst.NetworkEnabled { return "", fmt.Errorf("instance %s has no network configured", nameOrID) diff --git a/lib/instances/ingress_resolver_macos_test.go b/lib/instances/ingress_resolver_macos_test.go index e8fb2eb36..2c809f483 100644 --- a/lib/instances/ingress_resolver_macos_test.go +++ b/lib/instances/ingress_resolver_macos_test.go @@ -17,10 +17,15 @@ func (f fakeMinPrefixManager) getInstanceWithMinIDPrefix(context.Context, string return f.inst, nil } -func TestIngressResolverRefusesMacOSGuest(t *testing.T) { +// A running macOS guest resolves to its vmnet lease, which the instance reports as its IP. +func TestIngressResolverResolvesMacOSGuestLease(t *testing.T) { inst := &Instance{StoredMetadata: StoredMetadata{Id: "mac", NetworkEnabled: true, IP: "192.168.64.2", MacOS: &images.MacOSImage{}}} r := &IngressResolver{manager: fakeMinPrefixManager{inst: inst}} - _, err := r.ResolveInstanceIP(context.Background(), "mac") - require.Error(t, err) - require.Contains(t, err.Error(), "macOS guest") + ip, err := r.ResolveInstanceIP(context.Background(), "mac") + require.NoError(t, err) + require.Equal(t, "192.168.64.2", ip) + + inst.IP = "" + _, err = r.ResolveInstanceIP(context.Background(), "mac") + require.ErrorContains(t, err, "no IP assigned", "a guest without a current lease has no target") } From 9f35f32957f6391a15252ad9262cc93ad7ece674 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:17:57 +0000 Subject: [PATCH 2/3] Give each macOS guest its own vmnet network with a fixed address Hypeman assigns a networked macOS guest a /30 from network.macos_subnet_cidr and stores its address with the instance, keeping it through stop and standby. vz-shim creates a vmnet network for that subnet with a DHCP reservation for the guest's MAC, so the address no longer comes from reading vmnet's shared lease file, which could return expired or reassigned addresses. A released address is held for five minutes before reuse, longer than ingress caches upstream addresses. A memory restore whose saved address another guest now holds is refused; a cold-boot restore gets a new address. vmnet networks need macOS 26 and Code-Hex/vz v4, which is not tagged yet, so the darwin build pins it at upstream main (787f319). vz-shim adapts to v4's caller-owned state notifications. --- .../api/capabilities_rosetta_darwin_arm64.go | 2 +- cmd/api/config/config.go | 9 + cmd/api/config/config_test.go | 13 ++ cmd/api/hypervisor_check_darwin.go | 2 +- cmd/vz-shim/macos_arm64.go | 2 +- cmd/vz-shim/macos_other.go | 2 +- cmd/vz-shim/macos_vm.go | 2 +- cmd/vz-shim/main.go | 39 ++-- cmd/vz-shim/rosetta_arm64.go | 2 +- cmd/vz-shim/rosetta_arm64_test.go | 2 +- cmd/vz-shim/rosetta_other.go | 2 +- cmd/vz-shim/save_restore_arm64.go | 2 +- cmd/vz-shim/save_restore_unsupported.go | 2 +- cmd/vz-shim/server.go | 2 +- cmd/vz-shim/vm.go | 54 ++++- config.example.darwin.yaml | 4 + docs/macos-experimental.md | 33 ++-- docs/macos-guest-agent.md | 4 +- docs/macos-support.md | 2 +- go.mod | 15 +- go.sum | 18 +- lib/hypervisor/vz/macos_config_test.go | 6 + lib/hypervisor/vz/shimconfig/config.go | 4 + lib/hypervisor/vz/starter.go | 25 ++- lib/instances/create.go | 7 +- lib/instances/delete.go | 10 +- lib/instances/macos.go | 68 ++----- lib/instances/macos_darwin_test.go | 11 -- lib/instances/macos_identity_darwin.go | 2 +- lib/instances/macos_memory_snapshot_test.go | 31 +++ lib/instances/macos_network.go | 185 ++++++++++++++++++ lib/instances/macos_network_test.go | 107 ++++++++++ lib/instances/manager.go | 6 + lib/instances/query.go | 3 - lib/instances/snapshot.go | 21 ++ lib/instances/snapshot_macos.go | 10 + lib/instances/start.go | 5 +- lib/providers/providers.go | 6 + 38 files changed, 585 insertions(+), 135 deletions(-) create mode 100644 lib/instances/macos_network.go create mode 100644 lib/instances/macos_network_test.go diff --git a/cmd/api/api/capabilities_rosetta_darwin_arm64.go b/cmd/api/api/capabilities_rosetta_darwin_arm64.go index a52b3f047..0ec2f5f6d 100644 --- a/cmd/api/api/capabilities_rosetta_darwin_arm64.go +++ b/cmd/api/api/capabilities_rosetta_darwin_arm64.go @@ -2,7 +2,7 @@ package api -import "github.com/Code-Hex/vz/v3" +import "github.com/Code-Hex/vz/v4" // rosettaInstalled reports whether Rosetta translation for Linux guests is // installed and usable right now, using the same Virtualization.framework diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 3b728c282..0bffb655b 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "net/netip" "os" "path/filepath" "runtime" @@ -67,6 +68,8 @@ type NetworkConfig struct { DNSServer string `koanf:"dns_server"` UploadBurstMultiplier int `koanf:"upload_burst_multiplier"` DownloadBurstMultiplier int `koanf:"download_burst_multiplier"` + // MacOSSubnetCIDR is the pool macOS guest networks come from: one /29 per guest. + MacOSSubnetCIDR string `koanf:"macos_subnet_cidr"` } // CaddyConfig holds Caddy reverse-proxy / ingress settings. @@ -338,6 +341,7 @@ func defaultConfig() *Config { DNSServer: "1.1.1.1", UploadBurstMultiplier: 4, DownloadBurstMultiplier: 4, + MacOSSubnetCIDR: "192.168.128.0/20", }, Caddy: CaddyConfig{ @@ -594,6 +598,11 @@ func (c *Config) Validate() error { if c.MacOSOnly && (runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" || c.Hypervisor.Default != "vz") { return fmt.Errorf("macos_only requires vz on Apple silicon") } + if pool, err := netip.ParsePrefix(c.Network.MacOSSubnetCIDR); err != nil || !pool.Addr().Is4() || pool.Bits() < 8 || pool.Bits() > 29 { + return fmt.Errorf("network.macos_subnet_cidr must be an IPv4 prefix from /8 to /29, got %q", c.Network.MacOSSubnetCIDR) + } else if linux, err := netip.ParsePrefix(c.Network.SubnetCIDR); err == nil && linux.Overlaps(pool) { + return fmt.Errorf("network.macos_subnet_cidr %s overlaps network.subnet_cidr %s", pool, linux) + } if err := c.validateAPIListenPorts(); err != nil { return err } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index efd52f208..c695d4e85 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -605,3 +605,16 @@ func TestValidateBuildersConfig(t *testing.T) { t.Fatalf("expected parsed idle ttl 24h, got %s", cfg.Builders.IdleTTLDuration) } } + +func TestValidateMacOSSubnetCIDR(t *testing.T) { + if err := defaultConfig().Validate(); err != nil { + t.Fatalf("expected the default macOS guest pool to validate, got %v", err) + } + for _, pool := range []string{"", "nonsense", "fd00::/64", "192.168.128.0/30", "10.100.4.0/24"} { + cfg := defaultConfig() + cfg.Network.MacOSSubnetCIDR = pool + if err := cfg.Validate(); err == nil { + t.Fatalf("expected macOS guest pool %q to be rejected", pool) + } + } +} diff --git a/cmd/api/hypervisor_check_darwin.go b/cmd/api/hypervisor_check_darwin.go index 51950cc2c..ca64a833f 100644 --- a/cmd/api/hypervisor_check_darwin.go +++ b/cmd/api/hypervisor_check_darwin.go @@ -6,7 +6,7 @@ import ( "fmt" "runtime" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" ) // checkHypervisorAccess verifies Virtualization.framework is available on macOS diff --git a/cmd/vz-shim/macos_arm64.go b/cmd/vz-shim/macos_arm64.go index 62962e082..e2887abad 100644 --- a/cmd/vz-shim/macos_arm64.go +++ b/cmd/vz-shim/macos_arm64.go @@ -6,7 +6,7 @@ import ( "encoding/base64" "fmt" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/macos_other.go b/cmd/vz-shim/macos_other.go index 733ded320..ed256b850 100644 --- a/cmd/vz-shim/macos_other.go +++ b/cmd/vz-shim/macos_other.go @@ -4,7 +4,7 @@ package main import ( "fmt" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/macos_vm.go b/cmd/vz-shim/macos_vm.go index 8132a7a1f..269e9af82 100644 --- a/cmd/vz-shim/macos_vm.go +++ b/cmd/vz-shim/macos_vm.go @@ -6,7 +6,7 @@ import ( "fmt" "log/slog" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/main.go b/cmd/vz-shim/main.go index ed333b7a8..950602184 100644 --- a/cmd/vz-shim/main.go +++ b/cmd/vz-shim/main.go @@ -18,7 +18,7 @@ import ( "syscall" "time" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) @@ -131,23 +131,34 @@ func main() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - // Monitor VM state - stateChanged := vm.StateChangedNotify() + // Monitor VM state. Notifications can be dropped and their values can be stale, + // so each one, and the start, re-reads the current state. + stateChanged := make(chan vz.VirtualMachineState, 8) + stopNotify, err := vm.StateChangedNotify(stateChanged) + if err != nil { + slog.Error("failed to watch VM state", "error", err) + os.Exit(1) + } + defer stopNotify() go func() { for { - select { - case <-ctx.Done(): - return - case newState := <-stateChanged: - slog.Info("VM state changed", "state", newState) - if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { - if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() { - writeStopReceipt(config.StopReceiptPath) - } - slog.Info("VM stopped, shutting down shim") - cancel() + newState := vm.State() + if newState != vz.VirtualMachineStateStopped && newState != vz.VirtualMachineStateError { + select { + case <-ctx.Done(): return + case <-stateChanged: } + newState = vm.State() + slog.Info("VM state changed", "state", newState) + } + if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { + if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() { + writeStopReceipt(config.StopReceiptPath) + } + slog.Info("VM stopped, shutting down shim") + cancel() + return } } }() diff --git a/cmd/vz-shim/rosetta_arm64.go b/cmd/vz-shim/rosetta_arm64.go index fc7a06f97..5495f7731 100644 --- a/cmd/vz-shim/rosetta_arm64.go +++ b/cmd/vz-shim/rosetta_arm64.go @@ -6,7 +6,7 @@ import ( "fmt" "log/slog" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/rosetta_arm64_test.go b/cmd/vz-shim/rosetta_arm64_test.go index 2c8962f49..25f3b5fad 100644 --- a/cmd/vz-shim/rosetta_arm64_test.go +++ b/cmd/vz-shim/rosetta_arm64_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/cmd/vz-shim/rosetta_other.go b/cmd/vz-shim/rosetta_other.go index d2f86f2bd..5f45abb06 100644 --- a/cmd/vz-shim/rosetta_other.go +++ b/cmd/vz-shim/rosetta_other.go @@ -5,7 +5,7 @@ package main import ( "fmt" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/save_restore_arm64.go b/cmd/vz-shim/save_restore_arm64.go index 29147c3eb..a5d522216 100644 --- a/cmd/vz-shim/save_restore_arm64.go +++ b/cmd/vz-shim/save_restore_arm64.go @@ -5,7 +5,7 @@ package main import ( "fmt" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" ) func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { diff --git a/cmd/vz-shim/save_restore_unsupported.go b/cmd/vz-shim/save_restore_unsupported.go index 3dbcd1845..47ce10fd0 100644 --- a/cmd/vz-shim/save_restore_unsupported.go +++ b/cmd/vz-shim/save_restore_unsupported.go @@ -6,7 +6,7 @@ import ( "fmt" "runtime" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" ) func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go index 94b53103e..100e69276 100644 --- a/cmd/vz-shim/server.go +++ b/cmd/vz-shim/server.go @@ -15,7 +15,7 @@ import ( "sync" "sync/atomic" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) diff --git a/cmd/vz-shim/vm.go b/cmd/vz-shim/vm.go index b755da866..08b373e5f 100644 --- a/cmd/vz-shim/vm.go +++ b/cmd/vz-shim/vm.go @@ -7,11 +7,13 @@ import ( "fmt" "log/slog" "net" + "net/netip" "os" "runtime" "strings" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" + "github.com/Code-Hex/vz/v4/vmnet" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" ) @@ -190,7 +192,13 @@ func configureNetwork(vmConfig *vz.VirtualMachineConfiguration, networks []shimc var devices []*vz.VirtioNetworkDeviceConfiguration for _, netConfig := range networks { - dev, err := createNATNetworkDevice(netConfig.MAC) + var dev *vz.VirtioNetworkDeviceConfiguration + var err error + if netConfig.IPv4Address != "" { + dev, err = createVmnetNetworkDevice(netConfig) + } else { + dev, err = createNATNetworkDevice(netConfig.MAC) + } if err != nil { return err } @@ -220,6 +228,48 @@ func createNATNetworkDevice(macAddr string) (*vz.VirtioNetworkDeviceConfiguratio return networkConfig, nil } +// createVmnetNetworkDevice attaches the guest to its own shared-mode vmnet network +// with the given subnet and a DHCP reservation for the guest's MAC, so the guest's +// address is the one hypeman assigned. Saved state records this configuration, so a +// guest saved on the shared NAT (no address in its config) is restored on NAT. +func createVmnetNetworkDevice(netConfig shimconfig.NetworkConfig) (*vz.VirtioNetworkDeviceConfiguration, error) { + hw, err := net.ParseMAC(netConfig.MAC) + if err != nil { + return nil, fmt.Errorf("vmnet network needs the guest MAC: %w", err) + } + subnet, err := netip.ParsePrefix(netConfig.IPv4Subnet) + if err != nil { + return nil, fmt.Errorf("parse guest subnet: %w", err) + } + address, err := netip.ParseAddr(netConfig.IPv4Address) + if err != nil { + return nil, fmt.Errorf("parse guest address: %w", err) + } + cfg, err := vmnet.NewNetworkConfiguration(vmnet.ModeShared, vmnet.WithIPv4Subnet(subnet), vmnet.WithDHCPReservation(hw, address)) + if err != nil { + return nil, fmt.Errorf("configure vmnet network %s: %w", subnet, err) + } + network, err := vmnet.NewNetwork(cfg) + if err != nil { + return nil, fmt.Errorf("create vmnet network %s: %w", subnet, err) + } + attachment, err := vz.NewVMNetNetworkDeviceAttachment(network) + if err != nil { + return nil, fmt.Errorf("create vmnet attachment: %w", err) + } + dev, err := vz.NewVirtioNetworkDeviceConfiguration(attachment) + if err != nil { + return nil, fmt.Errorf("create network config: %w", err) + } + mac, err := assignMACAddress(netConfig.MAC) + if err != nil { + return nil, err + } + dev.SetMACAddress(mac) + slog.Info("attached guest vmnet network", "subnet", subnet.String(), "address", address.String(), "mac", netConfig.MAC) + return dev, nil +} + func assignMACAddress(macAddr string) (*vz.MACAddress, error) { if macAddr == "" { mac, err := vz.NewRandomLocallyAdministeredMACAddress() diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index 31ab3ef73..e7745fb1a 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -62,8 +62,12 @@ hypervisor: # - No TAP devices, user-created bridges, or iptables rules are needed # - The following settings are IGNORED on macOS because VZ owns the network: # network.bridge_name, subnet_cidr, subnet_gateway, uplink_interface +# - macOS guests do not use the shared NAT. Each gets its own vmnet network, a /29 +# from network.macos_subnet_cidr, and keeps its address through stop and standby. +# Keep the pool clear of the shared NAT subnet and of networks the host uses. network: dns_server: 8.8.8.8 + macos_subnet_cidr: 192.168.128.0/20 # ============================================================================= # Logging diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 26d1416e1..1c6e8f73e 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -76,8 +76,15 @@ CPU and memory default to the imported template (minimum 2 CPUs and 4GB). Each instance gets its own writable complete boot disk (`boot.raw`) and auxiliary storage; instances created before that naming must be deleted and recreated. No Linux kernel, initrd, configuration disk, overlay filesystem or balloon is -attached. NAT uses the preserved MAC. The IP is observed from the host's VZ -DHCP leases, which may retain an old lease before the guest is ready. +attached. A networked guest gets its own vmnet network (macOS 26 and later): +hypeman assigns it a /29 from `network.macos_subnet_cidr` (default +`192.168.128.0/20`) and vmnet's DHCP serves the guest a reservation for its +preserved MAC. The guest's address is stored with the instance, kept through stop +and standby, and reported whether or not the guest is running. Delete releases it, +and a released address is not given to another guest for five minutes, longer than +ingress caches upstream addresses. Clones and forks get a new address. Instances +started before this change get an address at their next start; until then they +stay on vmnet's shared NAT and report none. **Without the guest agent, `Running` means the VMM is running, not that SSH, the desktop or Chrome is ready.** Unmanaged templates still need an external @@ -125,20 +132,20 @@ CDP from off the host, route it through ingress (below) or tunnel it via SSH. Ingress routes to macOS guests as it does to Linux guests: Caddy on the host resolves the target instance through hypeman's DNS server on each request and -proxies to the guest's IP. For a macOS guest that IP is its current vmnet DHCP -lease, the address the instance reports. A guest that is not running, or has no -lease, has no upstream, and Caddy returns 502. As on Linux, the guest service must -listen on the guest's network interface, not only on loopback. +proxies to the guest's IP, the address hypeman assigned it. A guest that is not +running has no network at that address, so Caddy returns 502; the address is never +another guest's. As on Linux, the guest service must listen on the guest's network +interface, not only on loopback. Caddy is not embedded in macOS builds. `make build-caddy-darwin` builds the same Caddy version and DNS modules as Linux into `bin/caddy`. Hypeman looks for `caddy` next to its own binary, then on `PATH`, and fails to start without it. Caddy needs local network access to reach guests. On macOS 15.5 and later, allow -the vmnet subnet for every process once per host, then restart (Apple TN3179): +the guest pool for every process once per host, then restart (Apple TN3179): ```sh -sudo defaults write com.apple.network.local-network AllowedEthernetLocalNetworkAddresses -array "192.168.64.0/24" +sudo defaults write com.apple.network.local-network AllowedEthernetLocalNetworkAddresses -array "192.168.128.0/20" ``` Without it, only root has access. On the QA host (macOS 27.0), a launchd daemon @@ -161,8 +168,8 @@ the screen was unlocked. Turn off the automatic screen lock for that user: sysadminctl -screenLock off -password - ``` -On the QA host (macOS 27.0), vmnet attaches each guest to the NAT bridge as a -private member: guests cannot reach each other, but they can reach anything the -host listens on, including Caddy's listeners, sshd and Screen Sharing. Hosts that -run guests for different tenants should not expose other host services on the -bridge. Ingress hostnames reach a guest through Caddy, as they do from outside. +Each guest's network is separate, so guests cannot reach each other's addresses. +A guest can still reach anything the host listens on, including Caddy's listeners, +sshd and Screen Sharing, so hosts that run guests for different tenants should not +expose other host services on guest networks. Ingress hostnames reach a guest +through Caddy, as they do from outside. diff --git a/docs/macos-guest-agent.md b/docs/macos-guest-agent.md index 8e09d2859..fda435897 100644 --- a/docs/macos-guest-agent.md +++ b/docs/macos-guest-agent.md @@ -98,8 +98,8 @@ at boot; the host does not call `GetAgentInfo` for them. Permission, cancellation, and command errors are reported. An RPC response is not proof the VMM has exited: the host must wait for teardown. - **Network identity reconfiguration:** returns gRPC `Unimplemented` on Darwin. - Current VZ NAT uses guest DHCP; no static-address, MAC-rekey or network policy - parity is claimed. + The guest's address comes from a DHCP reservation on its own vmnet network; no + MAC-rekey or network policy parity is claimed. - **GPU status:** Linux NVIDIA initialization reporting is not macOS graphics readiness. A Darwin guest without that device reports the existing unknown state. diff --git a/docs/macos-support.md b/docs/macos-support.md index e0a6b529a..2ee42b466 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -24,7 +24,7 @@ owns the behavior. | Update (PATCH), VM restart policy | Partial | Expiration/TTL and `restart_policy: always` or `never` are supported. Manual stop suppresses automatic restart. A shim that dies without removing its socket (for example, SIGKILL) reads as Stopped, so `always` restarts it. `on_failure` is rejected: no workload exit status is available. | PR 1 | | Exec health checks | Agent-gated | Supported at create and PATCH; `type: none` disables them. HTTP/TCP checks remain unsupported. | PR 3 | | Volumes, env and commands, credentials and auto-standby policies, passthrough, shaping | Unsupported | Rejected at create, start, or update. | — | -| Ingress | Experimental | Routes to a running guest's vmnet lease, as on Linux, including in macOS-only mode. Caddy is not embedded on macOS (`make build-caddy-darwin`). New path needs native QA. | PR 5 | +| Ingress | Experimental | Routes to the guest's hypeman-assigned address on its own vmnet network, as on Linux, including in macOS-only mode. Caddy is not embedded on macOS (`make build-caddy-darwin`). New path needs native QA. | PR 5 | | Snapshot schedules | Experimental | State-based stopped or memory capture, existing cadence/retention, host save/restore gate and live configuration probe. New path needs native QA. | PR 4 | Limits that apply everywhere: one active VM per machine identifier, no clean-base diff --git a/go.mod b/go.mod index 974801dcd..b3f6abde7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.10 require ( al.essio.dev/pkg/shellescape v1.6.0 - github.com/Code-Hex/vz/v3 v3.7.1 + github.com/Code-Hex/vz/v4 v4.0.0-20261028120916-787f319dcf12 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/creack/pty v1.1.24 github.com/cyphar/filepath-securejoin v0.6.1 @@ -58,8 +58,8 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 golang.org/x/net v0.55.0 - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 + golang.org/x/sync v0.21.0 + golang.org/x/sys v0.46.0 google.golang.org/grpc v1.82.2 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 google.golang.org/protobuf v1.36.11 @@ -70,7 +70,6 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect @@ -137,10 +136,10 @@ require ( go.opentelemetry.io/otel/log v0.14.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 851573b04..ad9465561 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,8 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4s github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME= github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= -github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw= -github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY= -github.com/Code-Hex/vz/v3 v3.7.1 h1:EN1yNiyrbPq+dl388nne2NySo8I94EnPppvqypA65XM= -github.com/Code-Hex/vz/v3 v3.7.1/go.mod h1:1LsW0jqW0r0cQ+IeR4hHbjdqOtSidNCVMWhStMHGho8= +github.com/Code-Hex/vz/v4 v4.0.0-20261028120916-787f319dcf12 h1:R4xOWrBwJiPLOn69qrBTFV1pFpIIHD4E8wKcxMB3vTU= +github.com/Code-Hex/vz/v4 v4.0.0-20261028120916-787f319dcf12/go.mod h1:CUuC5Mtg/oCm/V0GW/a2fwb7Ko5RD80UYYs3863I44g= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= @@ -722,6 +720,8 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -729,6 +729,8 @@ golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -744,6 +746,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -759,6 +763,8 @@ golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -771,6 +777,8 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -780,6 +788,8 @@ golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/lib/hypervisor/vz/macos_config_test.go b/lib/hypervisor/vz/macos_config_test.go index 8efb65c2c..38fbac0b0 100644 --- a/lib/hypervisor/vz/macos_config_test.go +++ b/lib/hypervisor/vz/macos_config_test.go @@ -5,6 +5,7 @@ package vz import ( "encoding/base64" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" "github.com/stretchr/testify/require" "testing" ) @@ -17,6 +18,11 @@ func TestMacOSStarterConfig(t *testing.T) { require.Equal(t, "/vm/aux.img", shim.MacAuxStoragePath) require.Equal(t, "/vm/vz-guest-stopped", shim.StopReceiptPath) require.Empty(t, buildShimConfigFromVMConfig(hypervisor.VMConfig{}, "/vm/vz.sock").StopReceiptPath, "Linux guests write no receipt") + networked := cfg + networked.Networks = []hypervisor.NetworkConfig{{MAC: "02:00:00:00:00:01", IP: "192.168.128.10", Netmask: "255.255.255.248"}} + require.Equal(t, []shimconfig.NetworkConfig{{MAC: "02:00:00:00:00:01", IPv4Subnet: "192.168.128.8/29", IPv4Address: "192.168.128.10"}}, buildShimConfigFromVMConfig(networked, "/vm/vz.sock").Networks) + linux := hypervisor.VMConfig{Networks: networked.Networks} + require.Empty(t, buildShimConfigFromVMConfig(linux, "/vm/vz.sock").Networks[0].IPv4Address, "Linux guests keep the shared NAT") for _, resources := range []struct { cpus int memory int64 diff --git a/lib/hypervisor/vz/shimconfig/config.go b/lib/hypervisor/vz/shimconfig/config.go index 897b0fb36..16f413876 100644 --- a/lib/hypervisor/vz/shimconfig/config.go +++ b/lib/hypervisor/vz/shimconfig/config.go @@ -76,6 +76,10 @@ type DiskConfig struct { // NetworkConfig represents a network interface. type NetworkConfig struct { MAC string `json:"mac"` + // IPv4Subnet and IPv4Address, when set, give the guest its own vmnet network with + // a DHCP reservation for MAC. Otherwise the guest uses vmnet's shared NAT. + IPv4Subnet string `json:"ipv4_subnet,omitempty"` + IPv4Address string `json:"ipv4_address,omitempty"` } // SnapshotManifest is persisted in snapshot directories to allow restore. diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 26346eac6..321f832dd 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -10,6 +10,8 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net" + "net/netip" "os" "os/exec" "path/filepath" @@ -17,7 +19,7 @@ import ( "syscall" "time" - "github.com/Code-Hex/vz/v3" + "github.com/Code-Hex/vz/v4" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" "github.com/kernel/hypeman/lib/logger" @@ -232,10 +234,13 @@ func buildShimConfigFromVMConfig(config hypervisor.VMConfig, socketPath string) Readonly: disk.Readonly, }) } - for _, net := range config.Networks { - cfg.Networks = append(cfg.Networks, shimconfig.NetworkConfig{ - MAC: net.MAC, - }) + for _, n := range config.Networks { + nc := shimconfig.NetworkConfig{MAC: n.MAC} + if config.MacOS != nil && n.IP != "" { + nc.IPv4Address = n.IP + nc.IPv4Subnet = guestSubnet(n.IP, n.Netmask) + } + cfg.Networks = append(cfg.Networks, nc) } return cfg } @@ -333,3 +338,13 @@ func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout ti return nil, fmt.Errorf("timeout waiting for shim socket: %s", socketPath) } + +// guestSubnet returns the subnet of ip under a dotted netmask, such as 192.168.128.0/29. +func guestSubnet(ip, netmask string) string { + addr, err := netip.ParseAddr(ip) + if err != nil { + return "" + } + ones, _ := net.IPMask(net.ParseIP(netmask).To4()).Size() + return netip.PrefixFrom(addr, ones).Masked().String() +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 4b70b7e00..9477b2202 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -468,8 +468,13 @@ func (m *manager) createInstance( }) } - if macOSNetwork := applyMacOSNetwork(stored); macOSNetwork != nil { + macOSNetwork, err := m.applyMacOSNetwork(stored) + if err != nil { + return nil, err + } + if macOSNetwork != nil { netConfig = macOSNetwork + cu.Add(func() { m.forgetMacOSAddress(id) }) } // 15. Validate and attach volumes diff --git a/lib/instances/delete.go b/lib/instances/delete.go index e33bf8ac7..42775b48a 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -198,7 +198,15 @@ func (m *manager) deleteInstanceWithOptions( } } - // 8. Delete all instance data + // 8. Hold a macOS guest's address before its data goes, so it is not reused while + // ingress may still route to it. + if stored.MacOS != nil { + if err := m.releaseMacOSAddress(id, stored.IP); err != nil { + return fmt.Errorf("release macOS guest address: %w", err) + } + } + + // 9. Delete all instance data log.DebugContext(ctx, "deleting instance data", "instance_id", id) _, dataSpanEnd := m.startLifecycleStep(ctx, "delete_instance_data", attribute.String("instance_id", id), diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9afe3a268..9e8e4d901 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -3,11 +3,9 @@ package instances import ( "bytes" "context" - "encoding/hex" "errors" "fmt" "net" - "os" "path/filepath" "strings" "syscall" @@ -179,21 +177,23 @@ func (m *manager) createBootDisk(stored *StoredMetadata, img *images.Image) erro } // usesLinuxNetwork reports whether the instance's network is a hypeman TAP -// allocation. Networked macOS guests use vmnet NAT instead. +// allocation. Networked macOS guests get their own vmnet network instead. func (s *StoredMetadata) usesLinuxNetwork() bool { return s.NetworkEnabled && s.MacOS == nil } -// applyMacOSNetwork sets the template MAC on a networked macOS guest and returns its -// network config. vmnet assigns the address, which is observed after boot, so no -// Linux TAP or static IP is allocated. Other instances get nil. -func applyMacOSNetwork(stored *StoredMetadata) *network.NetworkConfig { +// applyMacOSNetwork sets the template MAC on a networked macOS guest, assigns its +// address if it has none, and returns its network config. No Linux TAP is allocated. +// Other instances get nil. +func (m *manager) applyMacOSNetwork(stored *StoredMetadata) (*network.NetworkConfig, error) { if stored.MacOS == nil || !stored.NetworkEnabled { - return nil + return nil, nil } stored.MAC = stored.MacOS.MAC - stored.IP = "" - return &network.NetworkConfig{MAC: stored.MAC} + if err := m.assignMacOSAddress(stored); err != nil { + return nil, err + } + return &network.NetworkConfig{MAC: stored.MAC, IP: stored.IP, Netmask: macOSGuestNetmask}, nil } // admitMacOSBoot holds the macOS boot lock and checks socket paths and identity @@ -219,7 +219,7 @@ func (m *manager) macOSVMConfig(inst *Instance) hypervisor.VMConfig { MacOS: &hypervisor.MacOSPlatform{HardwareModelData: inst.MacOS.HardwareModel, MachineIdentifierData: inst.MacOS.MachineIdentifier, AuxStoragePath: filepath.Join(inst.DataDir, macOSAuxStorageFile)}, Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(inst.Id)}}} if inst.NetworkEnabled { - c.Networks = []hypervisor.NetworkConfig{{MAC: inst.MAC}} + c.Networks = []hypervisor.NetworkConfig{{MAC: inst.MAC, IP: inst.IP, Netmask: macOSGuestNetmask}} } return c } @@ -302,52 +302,6 @@ func (m *manager) admitGuestOperation(stored *StoredMetadata) error { return m.requireMacOSGuest(stored) } -// vmnet stores octets without leading zeroes. A lease is observed addressing, -// not a static-IP assignment or a guest-readiness signal. -func macOSGuestIP(mac string) string { - wanted, err := net.ParseMAC(mac) - if err != nil { - return "" - } - data, err := os.ReadFile("/var/db/dhcpd_leases") - if err != nil { - return "" - } - return macOSLeaseIP(string(data), wanted) -} - -// dhcpd appends newer leases, so the last matching block is the current one. -func macOSLeaseIP(data string, wanted net.HardwareAddr) string { - found := "" - for _, block := range strings.Split(data, "}") { - var addr, ip string - for _, line := range strings.Split(block, "\n") { - k, v, ok := strings.Cut(strings.TrimSpace(line), "=") - if !ok { - continue - } - switch k { - case "hw_address": - _, addr, _ = strings.Cut(v, ",") - case "ip_address": - ip = v - } - } - var padded []string - for _, octet := range strings.Split(addr, ":") { - if len(octet) == 1 { - octet = "0" + octet - } - padded = append(padded, octet) - } - raw, err := hex.DecodeString(strings.Join(padded, "")) - if err == nil && string(raw) == string(wanted) && net.ParseIP(ip) != nil { - found = ip - } - } - return found -} - // VMStoragePath is the disk a macOS VM process holds open, used to find that process. // Linux guests return empty: their hypervisor PID is the whole guest. func (s *StoredMetadata) VMStoragePath() string { diff --git a/lib/instances/macos_darwin_test.go b/lib/instances/macos_darwin_test.go index 036f5ec0a..795a71199 100644 --- a/lib/instances/macos_darwin_test.go +++ b/lib/instances/macos_darwin_test.go @@ -4,7 +4,6 @@ package instances import ( "errors" - "net" "os" "path/filepath" "testing" @@ -71,13 +70,3 @@ func TestMacOSStorageClone(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) }) } - -func TestMacOSLeaseNormalization(t *testing.T) { - mac, err := net.ParseMAC("02:00:01:0a:0b:ff") - require.NoError(t, err) - data := "{\n ip_address=192.168.64.7\n hw_address=1,2:0:1:a:b:ff\n}\n" - require.Equal(t, "192.168.64.7", macOSLeaseIP(data, mac)) - require.Empty(t, macOSLeaseIP("garbage", mac)) - // A later lease for the same MAC supersedes the earlier one. - require.Equal(t, "192.168.64.9", macOSLeaseIP(data+"{\n ip_address=192.168.64.9\n hw_address=1,2:0:1:a:b:ff\n}\n", mac)) -} diff --git a/lib/instances/macos_identity_darwin.go b/lib/instances/macos_identity_darwin.go index e8cfa7d28..8a4e1b215 100644 --- a/lib/instances/macos_identity_darwin.go +++ b/lib/instances/macos_identity_darwin.go @@ -3,7 +3,7 @@ package instances import ( - vz "github.com/Code-Hex/vz/v3" + vz "github.com/Code-Hex/vz/v4" ) // newMacMachineIdentifierData returns a fresh Apple machine identifier for a cloned guest. diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go index a478b4216..28fb142b8 100644 --- a/lib/instances/macos_memory_snapshot_test.go +++ b/lib/instances/macos_memory_snapshot_test.go @@ -53,6 +53,11 @@ func writeMacOSMemorySnapshotPayload(t *testing.T, m *manager, guestDir string, }} if stored.NetworkEnabled { manifest.ShimConfig.Networks = []shimconfig.NetworkConfig{{MAC: stored.MAC}} + if stored.IP != "" { + subnet, err := macOSGuestSubnet(stored.IP) + require.NoError(t, err) + manifest.ShimConfig.Networks[0].IPv4Address, manifest.ShimConfig.Networks[0].IPv4Subnet = stored.IP, subnet.String() + } } writeMacOSSnapshotManifest(t, snapshotDir, manifest) return manifest @@ -138,6 +143,9 @@ func TestMacOSMemorySnapshotRejectsInvalidPayloadBeforeStorageSwap(t *testing.T) {"disk", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Disks[0].Path = "/wrong-disk" }}, {"aux", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacAuxStoragePath = "/wrong-aux" }}, {"network", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].MAC = "02:00:00:00:00:02" }}, + {"network-address", func(m *shimconfig.SnapshotManifest) { + m.ShimConfig.Networks[0].IPv4Address, m.ShimConfig.Networks[0].IPv4Subnet = "192.168.128.2", "192.168.128.0/29" + }}, {"state-path", func(m *shimconfig.SnapshotManifest) { m.MachineStateFile = "../../other-state" }}, } { t.Run(test.name, func(t *testing.T) { @@ -473,3 +481,26 @@ func (h resumeProbeHypervisor) Resume(ctx context.Context) error { h.onResume() return h.Hypervisor.Resume(ctx) } + +// Saved memory cannot change its network address, so a memory restore whose address +// another instance has taken since is refused rather than duplicating it. +func TestMacOSMemoryRestoreRefusesAddressNowInUse(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.IP = "192.168.128.2" + require.NoError(t, m.saveMetadata(meta)) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + snapshot, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.NoError(t, err) + + meta, err = m.loadMetadata(meta.Id) + require.NoError(t, err) + meta.IP = "192.168.128.10" + require.NoError(t, m.saveMetadata(meta)) + saveAddressedInstance(t, m, "mac-other", "192.168.128.2", true) + + _, err = m.RestoreSnapshot(context.Background(), meta.Id, snapshot.Id, RestoreSnapshotRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + got, err := m.loadMetadata(meta.Id) + require.NoError(t, err) + require.Equal(t, "192.168.128.10", got.IP, "a refused restore keeps the instance's address") +} diff --git a/lib/instances/macos_network.go b/lib/instances/macos_network.go new file mode 100644 index 000000000..0680d8d61 --- /dev/null +++ b/lib/instances/macos_network.go @@ -0,0 +1,185 @@ +package instances + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "path/filepath" + "time" +) + +// Each networked macOS guest gets its own vmnet network: a /29 from the configured +// pool, with the host at its first address and the guest at its second. vmnet's DHCP +// leaves the router out of its replies on a /30, so the guest would have no default +// route. The guest's +// address is stored with the instance and kept through stop and standby, so it never +// depends on vmnet's DHCP leases. +const ( + macOSGuestSubnetBits = 29 + macOSGuestNetmask = "255.255.255.248" +) + +// DefaultMacOSGuestSubnet is the pool macOS guest networks come from by default. +var DefaultMacOSGuestSubnet = netip.MustParsePrefix("192.168.128.0/20") + +// macOSAddressReuseHold keeps a released guest address unassigned for longer than +// ingress caches upstream addresses (one minute), so a request meant for a deleted +// guest cannot reach the next guest given its address. +const macOSAddressReuseHold = 5 * time.Minute + +const macOSAddressReleasesFile = "macos-address-releases.json" + +// macOSGuestSubnet returns the /29 a macOS guest address belongs to. +func macOSGuestSubnet(ip string) (netip.Prefix, error) { + addr, err := netip.ParseAddr(ip) + if err != nil || !addr.Is4() { + return netip.Prefix{}, fmt.Errorf("invalid macOS guest address %q", ip) + } + return netip.PrefixFrom(addr, macOSGuestSubnetBits).Masked(), nil +} + +// assignMacOSAddress gives a networked macOS guest without an address the first free +// guest address in the pool. A guest that has one keeps it. Addresses held by other +// instances, by creates in progress, or released within the reuse hold are skipped. +func (m *manager) assignMacOSAddress(stored *StoredMetadata) error { + if stored.MacOS == nil || !stored.NetworkEnabled || stored.IP != "" { + return nil + } + m.macOSAddressMu.Lock() + defer m.macOSAddressMu.Unlock() + unavailable, err := m.unavailableMacOSAddresses(stored.Id) + if err != nil { + return err + } + + pool := m.macOSGuestSubnet + if !pool.IsValid() { + pool = DefaultMacOSGuestSubnet + } + size := uint64(1) << (32 - macOSGuestSubnetBits) + base := pool.Masked().Addr().As4() + start := uint64(base[0])<<24 | uint64(base[1])<<16 | uint64(base[2])<<8 | uint64(base[3]) + count := (uint64(1) << (32 - pool.Bits())) / size + for i := uint64(0); i < count; i++ { + v := start + i*size + 2 + guest := netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}) + if unavailable[guest] { + continue + } + stored.IP = guest.String() + if m.macOSPendingAddresses == nil { + m.macOSPendingAddresses = make(map[string]netip.Addr) + } + m.macOSPendingAddresses[stored.Id] = guest + return nil + } + return fmt.Errorf("%w: no free macOS guest network in %s", ErrInsufficientResources, pool) +} + +// macOSAddressAvailable reports whether id may use ip: no other instance or create +// holds it and it is not in its reuse hold. +func (m *manager) macOSAddressAvailable(id, ip string) (bool, error) { + addr, err := netip.ParseAddr(ip) + if err != nil { + return false, nil + } + m.macOSAddressMu.Lock() + defer m.macOSAddressMu.Unlock() + unavailable, err := m.unavailableMacOSAddresses(id) + if err != nil { + return false, err + } + return !unavailable[addr], nil +} + +// unavailableMacOSAddresses returns the addresses id may not take: those of other +// instances and creates in progress, and those in their reuse hold. The caller holds +// macOSAddressMu. +func (m *manager) unavailableMacOSAddresses(id string) (map[netip.Addr]bool, error) { + unavailable := make(map[netip.Addr]bool) + err := m.forEachOtherInstance(id, "list instance addresses", func(meta *metadata) error { + if addr, err := netip.ParseAddr(meta.IP); err == nil { + unavailable[addr] = true + } + return nil + }) + if err != nil { + return nil, err + } + for other, addr := range m.macOSPendingAddresses { + if other != id { + unavailable[addr] = true + } + } + held, err := m.loadMacOSAddressReleases() + if err != nil { + return nil, err + } + now := m.nowUTC() + for ip, at := range held { + if addr, err := netip.ParseAddr(ip); err == nil && now.Sub(at) < macOSAddressReuseHold { + unavailable[addr] = true + } + } + return unavailable, nil +} + +// releaseMacOSAddress starts the reuse hold for a guest's address. It runs before the +// instance's data is removed, and an error stops the delete, so an address is never +// free without its hold. +func (m *manager) releaseMacOSAddress(id, ip string) error { + m.macOSAddressMu.Lock() + defer m.macOSAddressMu.Unlock() + delete(m.macOSPendingAddresses, id) + if ip == "" { + return nil + } + held, err := m.loadMacOSAddressReleases() + if err != nil { + return err + } + now := m.nowUTC() + for addr, at := range held { + if now.Sub(at) >= macOSAddressReuseHold { + delete(held, addr) + } + } + held[ip] = now + data, err := json.Marshal(held) + if err != nil { + return err + } + path := filepath.Join(m.paths.DataDir(), macOSAddressReleasesFile) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0600); err != nil { + return fmt.Errorf("record released macOS guest address: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("record released macOS guest address: %w", err) + } + return nil +} + +// forgetMacOSAddress drops an address assigned to a create that did not finish. +func (m *manager) forgetMacOSAddress(id string) { + m.macOSAddressMu.Lock() + defer m.macOSAddressMu.Unlock() + delete(m.macOSPendingAddresses, id) +} + +func (m *manager) loadMacOSAddressReleases() (map[string]time.Time, error) { + held := make(map[string]time.Time) + data, err := os.ReadFile(filepath.Join(m.paths.DataDir(), macOSAddressReleasesFile)) + if errors.Is(err, os.ErrNotExist) { + return held, nil + } + if err != nil { + return nil, fmt.Errorf("read released macOS guest addresses: %w", err) + } + if err := json.Unmarshal(data, &held); err != nil { + return nil, fmt.Errorf("decode released macOS guest addresses: %w", err) + } + return held, nil +} diff --git a/lib/instances/macos_network_test.go b/lib/instances/macos_network_test.go new file mode 100644 index 000000000..42906fe6c --- /dev/null +++ b/lib/instances/macos_network_test.go @@ -0,0 +1,107 @@ +package instances + +import ( + "net/netip" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func macOSNetworkTestManager(t *testing.T, pool string, now *time.Time) *manager { + t.Helper() + m := &manager{paths: paths.New(t.TempDir()), macOSGuestSubnet: netip.MustParsePrefix(pool), now: func() time.Time { return *now }} + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0755)) + return m +} + +func saveAddressedInstance(t *testing.T, m *manager, id, ip string, macOS bool) { + t.Helper() + stored := StoredMetadata{Id: id, IP: ip, NetworkEnabled: true} + if macOS { + stored.MacOS = macOSGuestMetadata(1).MacOS + } + require.NoError(t, os.MkdirAll(m.paths.InstanceDir(id), 0755)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) +} + +func networkedMacOSGuest(id string) *StoredMetadata { + stored := macOSGuestMetadata(1) + stored.Id = id + stored.NetworkEnabled = true + return stored +} + +// A /27 pool holds four guest networks, whose guest addresses are .2, .10, .18 and .26. +func TestMacOSAddressAssignment(t *testing.T) { + now := time.Now() + m := macOSNetworkTestManager(t, "192.168.128.0/27", &now) + saveAddressedInstance(t, m, "mac-a", "192.168.128.2", true) + saveAddressedInstance(t, m, "linux-b", "192.168.128.10", false) + + c := networkedMacOSGuest("mac-c") + require.NoError(t, m.assignMacOSAddress(c)) + require.Equal(t, "192.168.128.18", c.IP, "addresses of every stored instance are skipped") + + d := networkedMacOSGuest("mac-d") + require.NoError(t, m.assignMacOSAddress(d)) + require.Equal(t, "192.168.128.26", d.IP, "a create in progress keeps its address before it is saved") + + require.ErrorIs(t, m.assignMacOSAddress(networkedMacOSGuest("mac-e")), ErrInsufficientResources) + + require.NoError(t, m.assignMacOSAddress(c)) + require.Equal(t, "192.168.128.18", c.IP, "a guest keeps the address it has") + offline := networkedMacOSGuest("mac-f") + offline.NetworkEnabled = false + require.NoError(t, m.assignMacOSAddress(offline)) + require.Empty(t, offline.IP) + + m.forgetMacOSAddress("mac-d") + e := networkedMacOSGuest("mac-e") + require.NoError(t, m.assignMacOSAddress(e)) + require.Equal(t, "192.168.128.26", e.IP, "an unfinished create's address is free again without a hold") +} + +func TestMacOSAddressReuseHold(t *testing.T) { + now := time.Now() + m := macOSNetworkTestManager(t, "192.168.128.0/29", &now) + a := networkedMacOSGuest("mac-a") + require.NoError(t, m.assignMacOSAddress(a)) + require.Equal(t, "192.168.128.2", a.IP) + + require.NoError(t, m.releaseMacOSAddress("mac-a", a.IP)) + now = now.Add(time.Minute) + require.ErrorIs(t, m.assignMacOSAddress(networkedMacOSGuest("mac-b")), ErrInsufficientResources, "a released address is held past ingress's upstream cache") + ok, err := m.macOSAddressAvailable("mac-c", "192.168.128.2") + require.NoError(t, err) + require.False(t, ok) + + now = now.Add(macOSAddressReuseHold) + b := networkedMacOSGuest("mac-b") + require.NoError(t, m.assignMacOSAddress(b)) + require.Equal(t, "192.168.128.2", b.IP) +} + +func TestMacOSAddressAvailable(t *testing.T) { + now := time.Now() + m := macOSNetworkTestManager(t, "192.168.128.0/28", &now) + saveAddressedInstance(t, m, "mac-a", "192.168.128.2", true) + for ip, want := range map[string]bool{"192.168.128.2": false, "192.168.128.10": true, "garbage": false} { + ok, err := m.macOSAddressAvailable("mac-b", ip) + require.NoError(t, err) + require.Equal(t, want, ok, ip) + } + ok, err := m.macOSAddressAvailable("mac-a", "192.168.128.2") + require.NoError(t, err) + require.True(t, ok, "an instance may keep its own address") +} + +func TestMacOSGuestSubnet(t *testing.T) { + subnet, err := macOSGuestSubnet("192.168.128.10") + require.NoError(t, err) + require.Equal(t, "192.168.128.8/29", subnet.String()) + _, err = macOSGuestSubnet("fd00::2") + require.Error(t, err) +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 14b23e147..b64d1c1c9 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/netip" "os" "path/filepath" "strings" @@ -88,6 +89,7 @@ type ResourceLimits struct { // ManagerConfig holds non-resource manager behavior settings. type ManagerConfig struct { MacOSOnly bool + MacOSGuestSubnet netip.Prefix // pool for macOS guest networks; zero uses DefaultMacOSGuestSubnet LifecycleEventBufferSize int FirecrackerSnapshotMemoryBackend string FirecrackerUFFDCacheMaxBytes int64 @@ -177,6 +179,9 @@ type manager struct { instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks forkMetadataMu sync.Mutex macOSBootMu sync.Mutex // Serialize admission for preserved Mac identities. + macOSGuestSubnet netip.Prefix + macOSAddressMu sync.Mutex + macOSPendingAddresses map[string]netip.Addr // assigned to creates not yet saved macOSOnly bool bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan hypervisorStateCache sync.Map // map[string]hypervisorStateCacheEntry - last observed hypervisor state per instance @@ -295,6 +300,7 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa m := &manager{ paths: p, macOSOnly: managerConfig.MacOSOnly, + macOSGuestSubnet: managerConfig.MacOSGuestSubnet, imageManager: imageManager, systemManager: systemManager, networkManager: networkManager, diff --git a/lib/instances/query.go b/lib/instances/query.go index 9cf8ffe84..44d47ebb0 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -572,9 +572,6 @@ func (m *manager) toInstanceWithStateDerivation(ctx context.Context, meta *metad HealthCheckRuntime: healthcheck.CloneRuntime(meta.HealthCheckRuntime), } refreshHypervisorPID(&inst.StoredMetadata, result.State) - if inst.MacOS != nil && inst.NetworkEnabled && inst.State.RequiresVMM() { - inst.IP = macOSGuestIP(inst.MAC) - } // If VM is stopped and exit info isn't persisted yet, populate in-memory // from the serial console log. This is read-only -- no metadata writes. diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index cf61c5c22..51866b13a 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -366,6 +366,27 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str } } prepareMacOSSnapshotRestoreMetadata(&restored, &sourceMeta.StoredMetadata, rec.Snapshot.Kind, targetState) + // The restored record carries the address the snapshot was taken with. If another + // instance has since taken it, a cold boot gets a new address; saved memory + // cannot change address, so its restore is refused. The current address is held + // if it changes, as a delete would. + if restored.IP != "" && restored.IP != sourceMeta.IP { + ok, err := m.macOSAddressAvailable(id, restored.IP) + if err != nil { + return nil, err + } + if !ok && rec.Snapshot.Kind == SnapshotKindStandby { + return nil, fmt.Errorf("%w: the snapshot's network address %s is now in use", ErrInvalidState, restored.IP) + } + if !ok { + restored.IP = "" + } + } + if sourceMeta.IP != "" && sourceMeta.IP != restored.IP { + if err := m.releaseMacOSAddress(id, sourceMeta.IP); err != nil { + return nil, fmt.Errorf("release macOS guest address: %w", err) + } + } if err := m.replaceMacOSInstanceWithSnapshotPayload(snapshotID, &metadata{StoredMetadata: restored}); err != nil { return nil, err } diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index 50d3ff42b..e120fa38f 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -172,6 +172,16 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St if len(cfg.Networks) != 1 || !strings.EqualFold(cfg.Networks[0].MAC, stored.MAC) || !strings.EqualFold(stored.MAC, stored.MacOS.MAC) { return fmt.Errorf("%w: macOS snapshot network identity does not match its metadata", ErrInvalidRequest) } + // The saved network must be the guest's own, so it resumes at its recorded address. + if cfg.Networks[0].IPv4Address != stored.IP { + return fmt.Errorf("%w: macOS snapshot network address does not match its metadata", ErrInvalidRequest) + } + if stored.IP != "" { + subnet, err := macOSGuestSubnet(stored.IP) + if err != nil || cfg.Networks[0].IPv4Subnet != subnet.String() { + return fmt.Errorf("%w: macOS snapshot network subnet does not match its metadata", ErrInvalidRequest) + } + } } else if len(cfg.Networks) != 0 { return fmt.Errorf("%w: macOS snapshot unexpectedly enables networking", ErrInvalidRequest) } diff --git a/lib/instances/start.go b/lib/instances/start.go index 2fd3fd236..898d6a166 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -108,7 +108,10 @@ func (m *manager) startInstance( defer cu.Clean() // 4. Allocate fresh network if network enabled - netConfig := applyMacOSNetwork(stored) + netConfig, err := m.applyMacOSNetwork(stored) + if err != nil { + return nil, fmt.Errorf("assign macOS guest network: %w", err) + } if stored.usesLinuxNetwork() { log.DebugContext(ctx, "allocating network for start", "instance_id", id, "network", "default") networkCtx, networkSpanEnd := m.startLifecycleStep(ctx, "allocate_network", diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 3e3320b07..8de7c33d3 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "net/netip" "os" "strconv" "strings" @@ -144,8 +145,13 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima if err := firecrackerUFFDCacheMaxBytes.UnmarshalText([]byte(cfg.Hypervisor.FirecrackerUFFDCacheMaxBytes)); err != nil { return nil, fmt.Errorf("failed to parse hypervisor.firecracker_uffd_cache_max_bytes %q: %w", cfg.Hypervisor.FirecrackerUFFDCacheMaxBytes, err) } + macOSGuestSubnet, err := netip.ParsePrefix(cfg.Network.MacOSSubnetCIDR) + if err != nil { + return nil, fmt.Errorf("failed to parse network.macos_subnet_cidr %q: %w", cfg.Network.MacOSSubnetCIDR, err) + } managerConfig := instances.ManagerConfig{ MacOSOnly: cfg.MacOSOnly, + MacOSGuestSubnet: macOSGuestSubnet, LifecycleEventBufferSize: cfg.Instances.LifecycleEventBufferSize, FirecrackerSnapshotMemoryBackend: cfg.Hypervisor.FirecrackerSnapshotMemoryBackend, FirecrackerUFFDCacheMaxBytes: int64(firecrackerUFFDCacheMaxBytes), From 0897c3f005c71e0fa55ce368d51d35991e3b95b6 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:41:04 +0000 Subject: [PATCH 3/3] Stop a macOS guest's shim instead of killing it on delete Deleting a running guest killed its hypervisor process. A vz-shim owns its guest's vmnet network, and macOS keeps a network whose owner is killed while the VM is attached, refusing that subnet until the host restarts. Delete now asks a networked macOS guest's shim to stop the VM and exit, as stop does, and kills it only if it does not exit. --- docs/macos-experimental.md | 6 +++++ lib/instances/delete.go | 14 +++++++++- lib/instances/macos_network_test.go | 42 +++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 1c6e8f73e..585453ee6 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -86,6 +86,12 @@ ingress caches upstream addresses. Clones and forks get a new address. Instances started before this change get an address at their next start; until then they stay on vmnet's shared NAT and report none. +The guest's vz-shim owns its vmnet network. If the shim is killed while the guest +runs (SIGKILL, a crash), macOS keeps the network and refuses to create that subnet +again until the host restarts, so that guest cannot start and its address cannot be +reused. Stop and delete therefore ask the shim to stop the VM and exit, and kill it +only if it does not. + **Without the guest agent, `Running` means the VMM is running, not that SSH, the desktop or Chrome is ready.** Unmanaged templates still need an external readiness check. Templates provisioned with the shared Darwin GuestService can opt diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 42775b48a..d9ea69a40 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -132,7 +132,7 @@ func (m *manager) deleteInstanceWithOptions( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "force_kill_hypervisor"), ) - err := m.killHypervisor(killCtx, &inst) + err := m.stopHypervisorForDelete(killCtx, &inst) killSpanEnd(err) if err != nil { // The hypervisor may still be running, so tearing down its vGPU, @@ -233,6 +233,18 @@ func resolveDeleteStopTimeout(stored *StoredMetadata) int { return min(resolveStopTimeout(stored), bound) } +// stopHypervisorForDelete stops the hypervisor before a delete removes the instance. +// A macOS guest's vz-shim owns its vmnet network, and macOS keeps a network whose +// owner is killed while the VM is attached, which blocks that subnet until the host +// restarts. Its shim is asked to stop the VM and exit first; shutdownHypervisor still +// kills it if it does not. Other guests are killed as before. +func (m *manager) stopHypervisorForDelete(ctx context.Context, inst *Instance) error { + if inst.MacOS != nil && inst.NetworkEnabled { + return m.shutdownHypervisor(ctx, inst) + } + return m.killHypervisor(ctx, inst) +} + // killHypervisor force kills the hypervisor process without graceful shutdown. // Used by delete and as stop's final fallback after graceful shutdown fails. // It returns an error when the hypervisor may still be running: neither process diff --git a/lib/instances/macos_network_test.go b/lib/instances/macos_network_test.go index 42906fe6c..002aa0ac8 100644 --- a/lib/instances/macos_network_test.go +++ b/lib/instances/macos_network_test.go @@ -1,11 +1,15 @@ package instances import ( + "context" "net/netip" "os" + "path/filepath" + "sync/atomic" "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) @@ -105,3 +109,41 @@ func TestMacOSGuestSubnet(t *testing.T) { _, err = macOSGuestSubnet("fd00::2") require.Error(t, err) } + +const deleteShutdownTestHypervisor hypervisor.Type = "macos-delete-shutdown-test" + +type shutdownRecordingHypervisor struct { + lifecycleNoopHypervisor + calls *atomic.Int32 +} + +func (h shutdownRecordingHypervisor) Shutdown(context.Context) error { + h.calls.Add(1) + return nil +} + +func (h shutdownRecordingHypervisor) Capabilities() hypervisor.Capabilities { + return hypervisor.Capabilities{SupportsGracefulVMMShutdown: true} +} + +// A killed vz-shim leaves its vmnet network behind and blocks its subnet, so delete +// asks a networked macOS guest's shim to stop the VM instead of killing it. +func TestDeleteStopsMacOSShimBeforeKilling(t *testing.T) { + var calls atomic.Int32 + hypervisor.RegisterClientFactory(deleteShutdownTestHypervisor, func(string) (hypervisor.Hypervisor, error) { + return shutdownRecordingHypervisor{lifecycleNoopHypervisor: lifecycleNoopHypervisor{state: hypervisor.StateRunning}, calls: &calls}, nil + }) + m := &manager{} + inst := func(macOS bool) *Instance { + stored := StoredMetadata{Id: "delete-shutdown", HypervisorType: deleteShutdownTestHypervisor, NetworkEnabled: true, SocketPath: filepath.Join(t.TempDir(), "vz.sock")} + if macOS { + stored.MacOS = macOSGuestMetadata(1).MacOS + } + return &Instance{StoredMetadata: stored} + } + + require.NoError(t, m.stopHypervisorForDelete(context.Background(), inst(true))) + require.EqualValues(t, 1, calls.Load(), "a macOS shim is asked to stop its VM") + require.NoError(t, m.stopHypervisorForDelete(context.Background(), inst(false))) + require.EqualValues(t, 1, calls.Load(), "other guests are killed as before") +}