diff --git a/.github/workflows/macos-image.yml b/.github/workflows/macos-image.yml new file mode 100644 index 000000000..e93560f1b --- /dev/null +++ b/.github/workflows/macos-image.yml @@ -0,0 +1,97 @@ +name: macOS image + +# Builds a macOS browser image with Packer and Tart on a stacked clone of a base, +# checks it, and publishes it to hypeman's registry over that base. See +# docs/macos-experimental.md. +on: + workflow_dispatch: + inputs: + tart_base: + description: The base as a Tart image (OCI reference) to clone stacked + required: true + base: + description: The same base in hypeman's registry, pinned by digest + required: true + name: + description: Image name; published as builds/NAME. The registry token must be minted for it. + required: true + +permissions: + contents: read + +# A Mac runs at most two VMs; builds on one runner go one at a time. +concurrency: + group: macos-image-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + # Stacked Tart VMs and the Swift vz-shim need macOS 27 and the macOS 27 SDK. + runs-on: [self-hosted, macos, arm64, tart] + timeout-minutes: 90 + env: + VM: hypeman-image-${{ github.run_id }} + TART_VERSION: 2.41.1 + TART_SHA256: e680a734febccfce7d4e910c1f7f13fc3b00f19a181c945e2d746de3214c32c2 + PACKER_VERSION: 1.16.2 + PACKER_SHA256: 3d7e4df8e7e8c21bdf495c4f63ad9189e22540dc8126dee6db791aa8519e8b3a + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + + - name: Install Tart and Packer + run: | + bin="$RUNNER_TEMP/bin" + mkdir -p "$bin" + curl -fsSL -o "$RUNNER_TEMP/tart.tar.gz" "https://github.com/cirruslabs/tart/releases/download/$TART_VERSION/tart.tar.gz" + echo "$TART_SHA256 $RUNNER_TEMP/tart.tar.gz" | shasum -a 256 -c + tar -xzf "$RUNNER_TEMP/tart.tar.gz" -C "$RUNNER_TEMP" + ln -sf "$RUNNER_TEMP/tart.app/Contents/MacOS/tart" "$bin/tart" + curl -fsSL -o "$RUNNER_TEMP/packer.zip" "https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_${PACKER_VERSION}_darwin_arm64.zip" + echo "$PACKER_SHA256 $RUNNER_TEMP/packer.zip" | shasum -a 256 -c + unzip -oq "$RUNNER_TEMP/packer.zip" -d "$bin" + echo "$bin" >> "$GITHUB_PATH" + + - name: Build import-tart + run: go build -o "$RUNNER_TEMP/bin/import-tart" ./cmd/import-tart + + - name: Clone the base stacked + env: + TART_BASE: ${{ inputs.tart_base }} + TART_REGISTRY_USERNAME: ${{ secrets.TART_REGISTRY_USERNAME }} + TART_REGISTRY_PASSWORD: ${{ secrets.TART_REGISTRY_PASSWORD }} + run: tart clone --stacked "$TART_BASE" "$VM-base" + + - name: Build and check the image + working-directory: images/macos + env: + PKR_VAR_ssh_username: ${{ vars.MACOS_IMAGE_SSH_USERNAME }} + PKR_VAR_ssh_password: ${{ secrets.MACOS_IMAGE_SSH_PASSWORD }} + run: | + packer init browser.pkr.hcl + packer build -var vm_base_name="$VM-base" -var vm_name="$VM" browser.pkr.hcl + + - name: Publish + # pipefail, so a failed publish is not hidden by tee. + shell: bash + env: + BASE: ${{ inputs.base }} + NAME: ${{ inputs.name }} + REGISTRY: ${{ vars.HYPEMAN_REGISTRY_URL }} + # Minted by import-tart --mint-token for this name and base; it can publish + # nothing else. + HYPEMAN_REGISTRY_TOKEN: ${{ secrets.HYPEMAN_REGISTRY_TOKEN }} + run: | + import-tart --vm "$HOME/.tart/vms/$VM" --registry "$REGISTRY" --name "$NAME" --base "$BASE" --guest-agent | tee "$RUNNER_TEMP/publication.json" + echo "Published \`$(jq -r .Reference "$RUNNER_TEMP/publication.json")\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Delete the build VMs + if: always() + run: | + tart delete "$VM" || true + tart delete "$VM-base" || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b398a92c3..fa11e9551 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,8 @@ jobs: # cross-compiled from the Linux release job — it builds on a macOS arm64 runner. release-darwin: needs: release - runs-on: macos-14 + # vz-shim needs the macOS 27 SDK. + runs-on: xcode-27 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c99517e68..f1edda087 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -404,6 +404,8 @@ jobs: with: go-version-file: go.mod cache: false + - name: Show macOS and SDK versions + run: sw_vers && xcrun --sdk macosx --show-sdk-version - name: Ensure Docker is running run: | if docker info >/dev/null 2>&1; then diff --git a/.gitignore b/.gitignore index 614d5996b..936c7cadd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ cloud-hypervisor cloud-hypervisor/** lib/system/exec_agent/exec-agent lib/system/guest_agent/guest-agent +lib/system/guest_agent/guest-agent-darwin lib/system/init/init lib/hypervisor/vz/vz-shim/vz-shim diff --git a/Makefile b/Makefile index 5e7a4cab0..c42f760c9 100644 --- a/Makefile +++ b/Makefile @@ -246,6 +246,12 @@ lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go @echo "Building guest-agent for Linux..." cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o guest-agent . +# Build the macOS guest-agent, carried on the read-only agent disk of macOS guests. +# Darwin AF_VSOCK needs cgo, so it builds only on macOS, the only host that embeds it. +lib/system/guest_agent/guest-agent-darwin: lib/system/guest_agent/*.go + @echo "Building guest-agent for macOS..." + cd lib/system/guest_agent && CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -o guest-agent-darwin . + # Build init binary (runs as PID 1 in guest VM) for embedding # Cross-compile for Linux since it runs inside the VM lib/system/init/init: lib/system/init/*.go @@ -254,6 +260,9 @@ lib/system/init/init: lib/system/init/*.go build-embedded: @$(MAKE) -B lib/system/guest_agent/guest-agent +ifeq ($(shell uname -s),Darwin) + @$(MAKE) -B lib/system/guest_agent/guest-agent-darwin +endif @$(MAKE) -B lib/system/init/init # Build the binary @@ -335,7 +344,7 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries # macOS tests (no sudo needed, adds e2fsprogs to PATH) # Uses 'go list' to discover compilable packages, then filters out packages # whose test files reference Linux-only symbols (devices, system/init). -DARWIN_EXCLUDE_PKGS := /lib/devices|/lib/system/init|/cmd/vz-shim +DARWIN_EXCLUDE_PKGS := /lib/devices|/lib/system/init test-darwin: build-embedded sign-vz-shim @VERBOSE_FLAG=""; \ if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ @@ -349,23 +358,6 @@ test-darwin: build-embedded sign-vz-shim PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) $$PKGS; \ fi - $(MAKE) test-vz-shim-signed VERBOSE="$(VERBOSE)" TEST="$(TEST)" TEST_TIMEOUT="$(TEST_TIMEOUT)" - -# cmd/vz-shim's tests call Virtualization.framework in-process, which requires the -# com.apple.security.virtualization entitlement. Plain `go test` binaries are -# unsigned, so compile the test binary, ad-hoc-sign it with the entitlement (as -# sign-vz-shim does for the shim), then run it. HYPEMAN_VZ_SIGNED=1 tells the -# tests not to skip for a missing entitlement. -.PHONY: test-vz-shim-signed -test-vz-shim-signed: - @set -e; \ - RUN_FLAG=""; \ - if [ -n "$(TEST)" ]; then RUN_FLAG="-test.run=$(TEST)"; fi; \ - BIN="$$(mktemp -d)/vz-shim.test"; \ - go test -tags containers_image_openpgp -c -o "$$BIN" ./cmd/vz-shim; \ - if [ ! -f "$$BIN" ]; then echo "no cmd/vz-shim test binary for this platform; skipping"; exit 0; fi; \ - codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force "$$BIN"; \ - HYPEMAN_VZ_SIGNED=1 "$$BIN" $$RUN_FLAG -test.v -test.timeout=$(TEST_TIMEOUT) # Manual-only guest memory policy integration tests (Linux hypervisors). test-guestmemory-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded @@ -408,7 +400,7 @@ clean: rm -rf lib/vmm/binaries/cloud-hypervisor/ rm -rf lib/hypervisor/firecracker/binaries/firecracker/ rm -rf lib/ingress/binaries/ - rm -f lib/system/guest_agent/guest-agent + rm -f lib/system/guest_agent/guest-agent lib/system/guest_agent/guest-agent-darwin rm -f lib/system/init/init rm -f lib/hypervisor/vz/vz-shim/vz-shim @@ -424,8 +416,13 @@ release-prep: download-ch-binaries download-firecracker-binaries build-caddy-bin # Entitlements file for macOS codesigning ENTITLEMENTS_FILE ?= vz.entitlements -# Build vz-shim (subprocess that hosts vz VMs) -# Also copies to embed directory so it gets embedded in the hypeman binary +# Build vz-shim (subprocess that hosts vz VMs) from the Swift sources in +# cmd/vz-shim. Also copies to embed directory so it gets embedded in the hypeman +# binary. It links DiskImageKit and the vmnet network API, so it needs the macOS 27 +# SDK (Xcode 27); the binary runs on macOS 26, and stacked disks need macOS 27. +VZ_SHIM_SOURCES := $(wildcard cmd/vz-shim/*.swift) +VZ_SHIM_TARGET ?= $(shell uname -m)-apple-macos26.0 + # DARWIN_LDFLAGS silences the macOS (Xcode 15+) linker's "ignoring duplicate # libraries: '-lobjc'" warning: cgo deps that link Virtualization.framework # specify -lobjc more than once, which the new linker flags. Harmless; suppress. @@ -434,7 +431,8 @@ DARWIN_LDFLAGS := -ldflags=-extldflags=-Wl,-no_warn_duplicate_libraries .PHONY: build-vz-shim build-vz-shim: | $(BIN_DIR) @echo "Building vz-shim for macOS..." - go build $(DARWIN_LDFLAGS) -o $(BIN_DIR)/vz-shim ./cmd/vz-shim + @sdk=$$(xcrun --sdk macosx --show-sdk-version); case $$sdk in 2[7-9]*|[3-9][0-9]*) ;; *) echo "vz-shim needs the macOS 27 SDK (Xcode 27); found $$sdk" >&2; exit 1;; esac + swiftc -O -target $(VZ_SHIM_TARGET) -o $(BIN_DIR)/vz-shim $(VZ_SHIM_SOURCES) mkdir -p lib/hypervisor/vz/vz-shim cp $(BIN_DIR)/vz-shim lib/hypervisor/vz/vz-shim/vz-shim @echo "Build complete: $(BIN_DIR)/vz-shim" diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index e19ca8035..fa420f011 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" @@ -26,6 +25,7 @@ const ( featureLogs = "logs" featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" + featureMacOSImages = "macos-machine-images" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -133,13 +133,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities }, Runtimes: runtimes, Network: *networkCaps, - Images: oapi.CapabilitiesImages{ - Platforms: imagePlatforms(runtime.GOARCH, emulation), - DefaultPlatform: images.HostPlatformString(), - }, + Images: hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly), Features: serverFeatures(runtime.GOOS, emulation), } + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -210,6 +208,29 @@ func imagePlatforms(goarch string, emulation bool) []string { return platforms } +func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages { + caps := oapi.CapabilitiesImages{ + Platforms: imagePlatforms(goarch, emulation), + DefaultPlatform: "linux/" + goarch, + } + if goos == "darwin" && goarch == "arm64" { + if macOSOnly { + caps.Platforms = []string{"darwin/arm64"} + caps.DefaultPlatform = "darwin/arm64" + } else { + caps.Platforms = append(caps.Platforms, "darwin/arm64") + } + } + return caps +} + +func machineImageFeatures(goos, goarch string) []string { + if goos != "darwin" || goarch != "arm64" { + return nil + } + return []string{featureMacOSImages} +} + // serverFeatures builds the server-level feature ID list: API surfaces that // are always present plus host-platform conditionals. func serverFeatures(goos string, emulation bool) []string { diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go new file mode 100644 index 000000000..220a1a812 --- /dev/null +++ b/cmd/api/api/capabilities_machine_test.go @@ -0,0 +1,33 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHostMachineImageCapabilities(t *testing.T) { + for _, tc := range []struct { + name, os, arch, defaultPlatform string + emulation, macOSOnly bool + platforms []string + }{ + {"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}}, + {"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}}, + {"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}}, + {"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}}, + {"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}}, + } { + t.Run(tc.name, func(t *testing.T) { + caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly) + require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform) + require.Equal(t, tc.platforms, caps.Platforms) + }) + } +} + +func TestMachineImageFeatureActivation(t *testing.T) { + require.Empty(t, machineImageFeatures("linux", "arm64")) + require.Empty(t, machineImageFeatures("darwin", "amd64")) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64")) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..3b9dc5e3b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,7 +63,8 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image) + platform := instances.DefaultPlatform(s.Config != nil && s.Config.MacOSOnly, derefString(request.Body.Platform)) + macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil @@ -337,7 +338,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Name: request.Body.Name, Image: request.Body.Image, Size: size, - Platform: derefString(request.Body.Platform), + Platform: platform, HotplugSize: hotplugSize, OverlaySize: overlaySize, Vcpus: vcpus, diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 5dc42cbdb..62e6652ce 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -16,11 +16,19 @@ import ( // macOSHost is true where macOS guests can exist. Tests override it. var macOSHost = runtime.GOOS == "darwin" -// isMacOSImage reports whether a create names an imported macOS image. Other -// hosts skip the lookup, so Linux creates are unchanged. An image that is not -// cached yet is not a macOS image; any other lookup error is returned. -func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) { - if !macOSHost || s.ImageManager == nil { +// isMacOSImage identifies macOS creates before Linux defaults are applied. An +// explicit Darwin platform and macOS-only servers can resolve uncached images. +func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) { + if !macOSHost { + return false, nil + } + if s.Config != nil && s.Config.MacOSOnly { + return true, nil + } + if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { + return true, nil + } + if s.ImageManager == nil { return false, nil } img, err := s.ImageManager.GetImage(ctx, name) diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index ce19cce6b..fc19ca81f 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -84,6 +84,38 @@ func TestLinuxCreateDefaultsMatchSchema(t *testing.T) { require.EqualValues(t, linuxDefaultVcpus, props["vcpus"].Value.Default) } +func TestMacOSCreateDefaultsToDarwinPlatform(t *testing.T) { + useMacOSHost(t) + darwin := "darwin/arm64" + for _, tc := range []struct { + name string + cfg *config.Config + images images.Manager + body string + platform *string + }{ + {"macOS-only server", &config.Config{MacOSOnly: true}, nil, `{"name":"macos","image":"registry.example/macos/app:latest"}`, nil}, + {"uncached darwin image", &config.Config{}, missingMacOSImageManager{}, `{"name":"macos","image":"registry.example/macos/app:latest","platform":"darwin/arm64"}`, &darwin}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, ImageManager: tc.images, Config: tc.cfg} + ctx, status := recordCreateBody(t, tc.body) + require.Equal(t, http.StatusOK, status) + size, overlay, vcpus := linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Platform: tc.platform, + Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) + }) + } +} + func TestMacOSCreateIgnoresInjectedLinuxDefaults(t *testing.T) { useMacOSHost(t) m := newCaptureCreateManager(nil) @@ -172,3 +204,9 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } + +type missingMacOSImageManager struct{ images.Manager } + +func (missingMacOSImageManager) GetImage(context.Context, string) (*images.Image, error) { + return nil, images.ErrNotFound +} diff --git a/cmd/import-tart/main.go b/cmd/import-tart/main.go new file mode 100644 index 000000000..3a0b1fa75 --- /dev/null +++ b/cmd/import-tart/main.go @@ -0,0 +1,98 @@ +//go:build darwin && arm64 + +// import-tart publishes a stopped Tart macOS VM to hypeman's registry as a machine +// image. A stacked VM built on an image already in the registry passes that image +// as --base, so only its overlay is uploaded. +// +// It authenticates with HYPEMAN_REGISTRY_TOKEN, a token from --mint-token that can +// publish only that image over that base, or with the registry's HYPEMAN_JWT_SECRET. +// --mint-token needs the secret and prints the token; CI then holds the token only. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/Code-Hex/vz/v4" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/machineimage" + "github.com/kernel/hypeman/lib/registrytoken" +) + +func main() { + vm := flag.String("vm", "", "stopped Tart VM directory, e.g. ~/.tart/vms/NAME") + registry := flag.String("registry", "", "hypeman registry URL, e.g. https://hypeman.example or http://localhost:4973") + name := flag.String("name", "", "image name; the image is published as builds/NAME") + baseRef := flag.String("base", "", "digest-pinned registry image the VM's stacked disk is built on") + guestAgent := flag.Bool("guest-agent", false, "the image runs hypeman's guest agent") + timeout := flag.Duration("timeout", 2*time.Hour, "publication timeout") + mint := flag.Bool("mint-token", false, "print a registry token that can publish --name over --base, instead of publishing") + tokenTTL := flag.Duration("token-ttl", 30*24*time.Hour, "lifetime of a token from --mint-token") + flag.Parse() + token, secret := os.Getenv("HYPEMAN_REGISTRY_TOKEN"), os.Getenv("HYPEMAN_JWT_SECRET") + if *registry == "" || *name == "" || (*mint && secret == "") || (!*mint && (*vm == "" || token == "" && secret == "")) { + fmt.Fprintln(os.Stderr, "--registry and --name are required, with --vm and HYPEMAN_REGISTRY_TOKEN or HYPEMAN_JWT_SECRET to publish, or --mint-token and HYPEMAN_JWT_SECRET") + os.Exit(2) + } + var base *machineimage.Base + if *baseRef != "" { + var err error + if base, err = machineimage.ParseBase(*registry, *baseRef); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + if *mint { + minted, err := machineimage.MintToken(registrytoken.NewGenerator(secret), *name, base, *tokenTTL) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(minted) + return + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + ctx, cancelTimeout := context.WithTimeout(ctx, *timeout) + defer cancelTimeout() + publisher := &machineimage.Publisher{RegistryURL: *registry, Token: token, NewMachineIdentifier: newMachineIdentifier} + if token == "" { + publisher.Tokens = registrytoken.NewGenerator(secret) + } + publication, err := publish(ctx, publisher, *vm, *name, base, *guestAgent) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := json.NewEncoder(os.Stdout).Encode(publication); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func publish(ctx context.Context, publisher *machineimage.Publisher, vm, name string, base *machineimage.Base, guestAgent bool) (machineimage.Publication, error) { + storage, err := machineimage.ReadTartVM(vm, guestAgent) + if err != nil { + return machineimage.Publication{}, err + } + for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { + if err := images.RequireClosedFile(ctx, path); err != nil { + return machineimage.Publication{}, fmt.Errorf("%s: %w", path, err) + } + } + return publisher.Publish(ctx, name, storage, base) +} + +func newMachineIdentifier() ([]byte, error) { + id, err := vz.NewMacMachineIdentifier() + if err != nil { + return nil, err + } + return id.DataRepresentation(), nil +} diff --git a/cmd/vz-shim/Config.swift b/cmd/vz-shim/Config.swift new file mode 100644 index 000000000..e2ad9fa7b --- /dev/null +++ b/cmd/vz-shim/Config.swift @@ -0,0 +1,100 @@ +import Foundation + +// These types mirror lib/hypervisor/vz/shimconfig. Every field is optional so a +// config written by any hypeman version decodes, and an absent field encodes as +// absent, which Go reads back as its zero value. + +struct ShimConfig: Codable { + var vcpus: Int? + var memoryBytes: Int64? + var disks: [DiskConfig]? + var networks: [NetworkConfig]? + var serialLogPath: String? + var kernelPath: String? + var initrdPath: String? + var kernelArgs: String? + var macHardwareModelData: String? + var macMachineIdentifierData: String? + var macAuxStoragePath: String? + var enableMemoryBalloon: Bool? + var requireMemoryBalloon: Bool? + var enableRosetta: Bool? + var controlSocket: String? + var vsockSocket: String? + var logPath: String? + var stopReceiptPath: String? + var machineIdentifierData: String? + var restoreMachineStatePath: String? + + enum CodingKeys: String, CodingKey { + case vcpus + case memoryBytes = "memory_bytes" + case disks, networks + case serialLogPath = "serial_log_path" + case kernelPath = "kernel_path" + case initrdPath = "initrd_path" + case kernelArgs = "kernel_args" + case macHardwareModelData = "mac_hardware_model_data" + case macMachineIdentifierData = "mac_machine_identifier_data" + case macAuxStoragePath = "mac_aux_storage_path" + case enableMemoryBalloon = "enable_memory_balloon" + case requireMemoryBalloon = "require_memory_balloon" + case enableRosetta = "enable_rosetta" + case controlSocket = "control_socket" + case vsockSocket = "vsock_socket" + case logPath = "log_path" + case stopReceiptPath = "stop_receipt_path" + case machineIdentifierData = "machine_identifier_data" + case restoreMachineStatePath = "restore_machine_state_path" + } + + var isMacGuest: Bool { + !(macHardwareModelData ?? "").isEmpty || !(macMachineIdentifierData ?? "").isEmpty || !(macAuxStoragePath ?? "").isEmpty + } +} + +struct DiskConfig: Codable { + var path: String + var readonly: Bool? + // parents, when set, are the read-only images below path, base first. path is + // then the ASIF overlay on top of them. + var parents: [String]? +} + +struct NetworkConfig: Codable { + var mac: String? + var ipv4Subnet: String? + var ipv4Address: String? + + enum CodingKeys: String, CodingKey { + case mac + case ipv4Subnet = "ipv4_subnet" + case ipv4Address = "ipv4_address" + } +} + +struct SnapshotManifest: Codable { + var hypervisor: String + var machineStateFile: String + var shimConfig: ShimConfig + + enum CodingKeys: String, CodingKey { + case hypervisor + case machineStateFile = "machine_state_file" + case shimConfig = "shim_config" + } +} + +let snapshotManifestFile = "config.json" +let snapshotMachineStateFile = "machine-state.vzm" +let rosettaMountTag = "rosetta" + +struct ShimError: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +// decodeBase64 accepts what Go's base64 decoder accepts, including line breaks. +func decodeBase64(_ s: String) -> Data? { + Data(base64Encoded: s, options: .ignoreUnknownCharacters) +} diff --git a/cmd/vz-shim/Identity.swift b/cmd/vz-shim/Identity.swift new file mode 100644 index 000000000..4611b84b7 --- /dev/null +++ b/cmd/vz-shim/Identity.swift @@ -0,0 +1,28 @@ +import CryptoKit +import Foundation + +// lockMacOSIdentity holds an exclusive lock on the Mac machine identity until the +// process exits, including while paused. The lock file is never unlinked: another +// waiter may already hold its inode open. +func lockMacOSIdentity(_ config: ShimConfig) throws -> Int32? { + guard let text = config.macMachineIdentifierData, !text.isEmpty else { return nil } + guard let data = decodeBase64(text) else { throw ShimError("decode Mac identity for admission") } + let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + // Fixed directory, not TMPDIR: servers with different environments must contend + // on the same lock. The name is predictable, so refuse symlinks and files owned + // by anyone else. + let path = "/tmp/hypeman-mac-identity-\(getuid())-\(hash).lock" + let fd = open(path, O_CREAT | O_RDWR | O_NOFOLLOW, 0o600) + guard fd >= 0 else { throw ShimError("open \(path): \(errnoString())") } + var info = stat() + guard fstat(fd, &info) == 0, info.st_mode & S_IFMT == S_IFREG, info.st_uid == getuid() else { + close(fd) + throw ShimError("Mac identity lock path is not an owned regular file") + } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + let message = errnoString() + close(fd) + throw ShimError("Mac machine identity already in use: \(message)") + } + return fd +} diff --git a/cmd/vz-shim/Log.swift b/cmd/vz-shim/Log.swift new file mode 100644 index 000000000..722ec84e6 --- /dev/null +++ b/cmd/vz-shim/Log.swift @@ -0,0 +1,45 @@ +import Foundation + +// Log writes JSON lines in the shape of Go's slog JSON handler, so the shim log +// that hypeman attaches to startup errors reads the same as before. +enum Log { + private static var handle = FileHandle.standardError + private static let lock = NSLock() + private static let formatter: ISO8601DateFormatter = { + let f = ISO8601DateFormatter() + f.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return f + }() + + static func setup(path: String?) throws { + guard let path, !path.isEmpty else { return } + try FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true) + let fd = open(path, O_CREAT | O_WRONLY | O_APPEND, 0o644) + guard fd >= 0 else { throw ShimError("open log file: \(String(cString: strerror(errno)))") } + handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + } + + static func debug(_ msg: String, _ fields: [String: Any] = [:]) { write("DEBUG", msg, fields) } + static func info(_ msg: String, _ fields: [String: Any] = [:]) { write("INFO", msg, fields) } + static func warn(_ msg: String, _ fields: [String: Any] = [:]) { write("WARN", msg, fields) } + static func error(_ msg: String, _ fields: [String: Any] = [:]) { write("ERROR", msg, fields) } + + private static func write(_ level: String, _ msg: String, _ fields: [String: Any]) { + var record = fields.mapValues { value -> Any in + if value is String || value is Int || value is Int64 || value is UInt32 || value is UInt64 || value is Bool { return value } + return String(describing: value) + } + record["time"] = formatter.string(from: Date()) + record["level"] = level + record["msg"] = msg + guard var line = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]) else { return } + line.append(0x0a) + lock.lock() + handle.write(line) + lock.unlock() + } +} + +func printError(_ message: String) { + FileHandle.standardError.write((message + "\n").data(using: .utf8)!) +} diff --git a/cmd/vz-shim/Network.swift b/cmd/vz-shim/Network.swift new file mode 100644 index 000000000..81944eb8b --- /dev/null +++ b/cmd/vz-shim/Network.swift @@ -0,0 +1,99 @@ +import Foundation +import Virtualization +import vmnet + +func makeNetworkDevices(_ networks: [NetworkConfig]) throws -> [VZNetworkDeviceConfiguration] { + // No networks configured (NetworkEnabled=false): no NIC is attached. + try networks.map { n in + let device = VZVirtioNetworkDeviceConfiguration() + if let address = n.ipv4Address, !address.isEmpty { + device.attachment = try vmnetAttachment(n) + } else { + device.attachment = VZNATNetworkDeviceAttachment() + } + device.macAddress = macAddress(n.mac) + return device + } +} + +func macAddress(_ text: String?) -> VZMACAddress { + if let text, !text.isEmpty { + if let mac = VZMACAddress(string: text) { + Log.info("using specified MAC address", ["mac": text]) + return mac + } + Log.warn("failed to parse MAC address, generating random", ["mac": text]) + return VZMACAddress.randomLocallyAdministered() + } + let mac = VZMACAddress.randomLocallyAdministered() + Log.info("generated random MAC address", ["mac": mac.string]) + return mac +} + +// vmnetAttachment attaches the guest to its own shared-mode vmnet network with the +// given subnet and a DHCP reservation for the guest's MAC, so the guest's address is +// the one hypeman assigned. +func vmnetAttachment(_ n: NetworkConfig) throws -> VZNetworkDeviceAttachment { + guard #available(macOS 26.0, *) else { + throw ShimError("configure network: vmnet networks require macOS 26 or newer") + } + guard let macText = n.mac, let mac = VZMACAddress(string: macText) else { + throw ShimError("configure network: vmnet network needs the guest MAC") + } + guard let (network, bits) = parseIPv4Prefix(n.ipv4Subnet ?? ""), bits <= 30 else { + throw ShimError("configure network: parse guest subnet \(n.ipv4Subnet ?? "")") + } + guard let address = parseIPv4(n.ipv4Address ?? "") else { + throw ShimError("configure network: parse guest address \(n.ipv4Address ?? "")") + } + let mask: UInt32 = bits == 0 ? 0 : ~UInt32(0) << UInt32(32 - bits) + let subnetText = "\(formatIPv4(network & mask))/\(bits)" + + var status = vmnet_return_t.VMNET_SUCCESS + guard let config = vmnet_network_configuration_create(.VMNET_SHARED_MODE, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: configure vmnet network \(subnetText): status \(status.rawValue)") + } + // vmnet needs the host address, the first in the subnet, to start DHCP-backed + // attachments; the network address alone leaves the guest without a lease. + var subnetAddr = in_addr(s_addr: ((network & mask) + 1).bigEndian) + var maskAddr = in_addr(s_addr: mask.bigEndian) + status = vmnet_network_configuration_set_ipv4_subnet(config, &subnetAddr, &maskAddr) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: set vmnet subnet \(subnetText): status \(status.rawValue)") + } + var client = ether_addr_t() + withUnsafeMutableBytes(of: &client.octet) { $0.copyBytes(from: mac.ethernetAddress) } + var reserved = in_addr(s_addr: address.bigEndian) + status = vmnet_network_configuration_add_dhcp_reservation(config, &client, &reserved) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: reserve \(formatIPv4(address)) on vmnet network \(subnetText): status \(status.rawValue)") + } + guard let network = vmnet_network_create(config, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: create vmnet network \(subnetText): status \(status.rawValue)") + } + Log.info("attached guest vmnet network", ["subnet": subnetText, "address": formatIPv4(address), "mac": macText]) + return VZVmnetNetworkDeviceAttachment(network: network) +} + +extension VZMACAddress { + var ethernetAddress: [UInt8] { + string.split(separator: ":").compactMap { UInt8($0, radix: 16) } + } +} + +// parseIPv4 returns an address in host byte order. +func parseIPv4(_ text: String) -> UInt32? { + var addr = in_addr() + guard inet_pton(AF_INET, text, &addr) == 1 else { return nil } + return UInt32(bigEndian: addr.s_addr) +} + +func parseIPv4Prefix(_ text: String) -> (UInt32, Int)? { + let parts = text.split(separator: "/") + guard parts.count == 2, let address = parseIPv4(String(parts[0])), let bits = Int(parts[1]), (0...32).contains(bits) else { return nil } + return (address, bits) +} + +func formatIPv4(_ address: UInt32) -> String { + "\(address >> 24 & 0xff).\(address >> 16 & 0xff).\(address >> 8 & 0xff).\(address & 0xff)" +} diff --git a/cmd/vz-shim/Platform.swift b/cmd/vz-shim/Platform.swift new file mode 100644 index 000000000..ae267df72 --- /dev/null +++ b/cmd/vz-shim/Platform.swift @@ -0,0 +1,31 @@ +import Foundation +import Virtualization + +func makeGenericPlatform(_ c: inout ShimConfig) throws -> VZGenericPlatformConfiguration { + let platform = VZGenericPlatformConfiguration() + if let text = c.machineIdentifierData, !text.isEmpty { + guard let data = decodeBase64(text), let identifier = VZGenericMachineIdentifier(dataRepresentation: data) else { + throw ShimError("configure platform: recreate machine identifier") + } + platform.machineIdentifier = identifier + } else { + let identifier = VZGenericMachineIdentifier() + platform.machineIdentifier = identifier + c.machineIdentifierData = identifier.dataRepresentation.base64EncodedString() + } + return platform +} + +func cpuCount(_ requested: Int) -> Int { + var count = requested + if count == 0 { + count = max(1, ProcessInfo.processInfo.processorCount - 1) + } + return min(max(count, VZVirtualMachineConfiguration.minimumAllowedCPUCount), VZVirtualMachineConfiguration.maximumAllowedCPUCount) +} + +func memorySize(_ requested: UInt64) -> UInt64 { + // 2 GiB safety default; the caller normally provides this. + let size = requested == 0 ? 2 << 30 : requested + return min(max(size, VZVirtualMachineConfiguration.minimumAllowedMemorySize), VZVirtualMachineConfiguration.maximumAllowedMemorySize) +} diff --git a/cmd/vz-shim/Shim.swift b/cmd/vz-shim/Shim.swift new file mode 100644 index 000000000..108ab98d2 --- /dev/null +++ b/cmd/vz-shim/Shim.swift @@ -0,0 +1,405 @@ +import Foundation +import Virtualization + +// Shim owns the VM. The VM runs on the main queue, so every VM call below happens +// there; request handlers run on their own threads and wait for it. +final class Shim: NSObject, VZVirtualMachineDelegate { + let vm: VZVirtualMachine + let vmConfig: VZVirtualMachineConfiguration + let config: ShimConfig + // Serializes control operations, as the Go shim's mutex did. + private let ops = NSLock() + // forcedStop is set, on the main queue, before the shim stops the VM itself, so + // that stop is not mistaken for the guest powering off. + private var forcedStop = false + private var listeners: [UnixListener] = [] + private var stateObservation: NSKeyValueObservation? + // Control requests still being served, which finish waits for. + private let inFlight = DispatchGroup() + private var finishing = false + + init(vm: VZVirtualMachine, vmConfig: VZVirtualMachineConfiguration, config: ShimConfig) { + self.vm = vm + self.vmConfig = vmConfig + self.config = config + super.init() + vm.delegate = self + stateObservation = vm.observe(\.state) { vm, _ in + Log.info("VM state changed", ["state": stateName(vm.state)]) + } + } + + // MARK: Lifecycle + + func listen() throws { + let control = try UnixListener(path: config.controlSocket ?? "") + let vsock = try UnixListener(path: config.vsockSocket ?? "") + listeners = [control, vsock] + control.serve { [self] fd in + inFlight.enter() + defer { inFlight.leave() } + serveHTTP(fd, route) + } + vsock.serve { [self] fd in proxyVsock(fd) } + Log.info("control API listening", ["socket": control.path]) + Log.info("vsock proxy listening", ["socket": vsock.path]) + } + + func guestDidStop(_ virtualMachine: VZVirtualMachine) { + if !forcedStop { + writeStopReceipt() + } + Log.info("VM stopped, shutting down shim") + finish() + } + + func virtualMachine(_ virtualMachine: VZVirtualMachine, didStopWithError error: Error) { + Log.error("VM stopped with error", ["error": error.localizedDescription]) + finish() + } + + // stopForSignal runs on the main queue. + func stopForSignal() { + if vm.state == .running && vm.canStop { + Log.info("stopping VM") + forcedStop = true + vm.stop { _ in self.finish() } + return + } + finish() + } + + // finish runs on the main queue. It stops accepting requests and exits once the + // control requests in flight are done, waiting at most five seconds, as the Go + // shim's server shutdown did. The wait is off the main queue, which those + // requests may need. + private func finish() { + guard !finishing else { return } + finishing = true + for listener in listeners { + listener.close() + } + DispatchQueue.global().async { [self] in + _ = inFlight.wait(timeout: .now() + 5) + Log.info("vz-shim shutdown complete") + exit(0) + } + } + + // writeStopReceipt records that the guest stopped the VM itself. The host counts + // a macOS stop as graceful only with this receipt. + private func writeStopReceipt() { + guard let path = config.stopReceiptPath, !path.isEmpty else { return } + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + let data = Data((formatter.string(from: Date()) + "\n").utf8) + let fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, 0o600) + guard fd >= 0, writeAll(fd, data) else { + Log.error("failed to write stop receipt", ["error": errnoString(), "path": path]) + if fd >= 0 { close(fd) } + return + } + close(fd) + } + + // MARK: Main-queue helpers + + private func onMain(_ body: () throws -> T) rethrows -> T { + try DispatchQueue.main.sync(execute: body) + } + + // awaitMain starts an asynchronous VM operation on the main queue and waits for + // its completion handler. + private func awaitMain(_ start: @escaping (@escaping (Error?) -> Void) -> Void) -> Error? { + let done = DispatchSemaphore(value: 0) + var result: Error? + DispatchQueue.main.async { + start { error in + result = error + done.signal() + } + } + done.wait() + return result + } + + // MARK: Control API + + // Endpoint is a control API route: a method and a path. + private enum Endpoint: String, CaseIterable { + case ping = "GET /api/v1/vmm.ping" + case vmmShutdown = "PUT /api/v1/vmm.shutdown" + case info = "GET /api/v1/vm.info" + case pause = "PUT /api/v1/vm.pause" + case resume = "PUT /api/v1/vm.resume" + case shutdown = "PUT /api/v1/vm.shutdown" + case snapshotSupport = "GET /api/v1/vm.snapshot-support" + case snapshot = "PUT /api/v1/vm.snapshot" + case getBalloon = "GET /api/v1/vm.balloon" + case setBalloon = "PUT /api/v1/vm.balloon" + + var path: Substring { rawValue.split(separator: " ")[1] } + } + + private static let paths = Set(Endpoint.allCases.map(\.path)) + + private func route(_ request: HTTPRequest) -> HTTPResponse { + guard let endpoint = Endpoint(rawValue: "\(request.method) \(request.path)") else { + return Shim.paths.contains(Substring(request.path)) ? .text(405, "Method Not Allowed") : .text(404, "404 page not found") + } + switch endpoint { + case .ping: return HTTPResponse(status: 200, body: Data("OK".utf8)) + case .vmmShutdown: return vmmShutdown() + default: break + } + ops.lock() + defer { ops.unlock() } + switch endpoint { + case .ping, .vmmShutdown: fatalError("served above") + case .info: return info() + case .pause: return pause() + case .resume: return resume() + case .shutdown: return requestStop() + case .snapshotSupport: return snapshotSupport() + case .snapshot: return snapshot(request.body) + case .getBalloon: return getBalloon() + case .setBalloon: return setBalloon(request.body) + } + } + + private struct VMInfo: Encodable { + let state: String + let memoryBalloonDevices: Int? + } + + private struct SnapshotSupport: Encodable { + let supported: Bool + let reason: String? + } + + private struct SnapshotRequest: Decodable { + let destinationPath: String + } + + private struct Balloon: Codable { + let targetGuestMemoryBytes: Int64 + } + + private func info() -> HTTPResponse { + let (state, balloons) = onMain { (stateName(vm.state), vm.memoryBalloonDevices.count) } + return .json(VMInfo(state: state, memoryBalloonDevices: balloons > 0 ? balloons : nil)) + } + + private func pause() -> HTTPResponse { + guard onMain({ vm.canPause }) else { return .text(400, "cannot pause VM") } + if let error = awaitMain({ done in self.vm.pause { done($0.error) } }) { + Log.error("failed to pause VM", ["error": error.localizedDescription]) + return .text(500, "pause failed: \(error.localizedDescription)") + } + Log.info("VM paused") + return .noContent() + } + + private func resume() -> HTTPResponse { + guard onMain({ vm.canResume }) else { return .text(400, "cannot resume VM") } + if let error = awaitMain({ done in self.vm.resume { done($0.error) } }) { + Log.error("failed to resume VM", ["error": error.localizedDescription]) + return .text(500, "resume failed: \(error.localizedDescription)") + } + Log.info("VM resumed") + return .noContent() + } + + // requestStop sends the guest a power-button event. The instance manager handles + // the timeout and forced stop if the guest does not shut down. + private func requestStop() -> HTTPResponse { + do { + try onMain { try vm.requestStop() } + } catch { + Log.error("RequestStop failed", ["error": error.localizedDescription]) + return .text(500, "shutdown failed: \(error.localizedDescription)") + } + Log.info("VM graceful shutdown requested (ACPI power button)") + return .noContent() + } + + private func snapshotSupport() -> HTTPResponse { + let reason = saveRestoreUnsupportedReason() + return .json(SnapshotSupport(supported: reason == nil, reason: reason)) + } + + func saveRestoreUnsupportedReason() -> String? { + do { + try vmConfig.validateSaveRestoreSupport() + return nil + } catch { + return error.localizedDescription + } + } + + private func snapshot(_ body: Data) -> HTTPResponse { + let destination: String + do { + destination = try snakeCaseDecoder.decode(SnapshotRequest.self, from: body).destinationPath + } catch { + return .text(400, "invalid snapshot request: \(error.localizedDescription)") + } + guard !destination.isEmpty else { return .text(400, "destination_path is required") } + guard onMain({ vm.state == .paused }) else { return .text(400, "vm must be paused before snapshot") } + if let reason = saveRestoreUnsupportedReason() { + return .text(400, "save/restore not supported: \(reason)") + } + let fm = FileManager.default + do { + try fm.createDirectory(atPath: destination, withIntermediateDirectories: true) + } catch { + return .text(500, "create snapshot dir failed: \(error.localizedDescription)") + } + var complete = false + defer { + if !complete { try? fm.removeItem(atPath: destination) } + } + let statePath = (destination as NSString).appendingPathComponent(snapshotMachineStateFile) + try? fm.removeItem(atPath: statePath) + if let error = awaitMain({ done in self.vm.saveMachineStateTo(url: URL(fileURLWithPath: statePath), completionHandler: done) }) { + return .text(500, "save machine state failed: \(error.localizedDescription)") + } + var shimConfig = config + // Runtime-only: the restore path is filled in by the caller on restore. + shimConfig.restoreMachineStatePath = nil + let manifestPath = (destination as NSString).appendingPathComponent(snapshotManifestFile) + do { + let manifest = SnapshotManifest(hypervisor: "vz", machineStateFile: snapshotMachineStateFile, shimConfig: shimConfig) + try JSONEncoder().encode(manifest).write(to: URL(fileURLWithPath: manifestPath + ".tmp")) + } catch { + return .text(500, "write manifest failed: \(error.localizedDescription)") + } + // rename(2) replaces the manifest atomically, as the Go shim's os.Rename did. + guard rename(manifestPath + ".tmp", manifestPath) == 0 else { + return .text(500, "write manifest failed: \(errnoString())") + } + complete = true + Log.info("VM snapshot saved", ["destination": destination, "machine_state": statePath]) + return .noContent() + } + + private func balloon() -> VZVirtioTraditionalMemoryBalloonDevice? { + vm.memoryBalloonDevices.compactMap { $0 as? VZVirtioTraditionalMemoryBalloonDevice }.first + } + + private func getBalloon() -> HTTPResponse { + guard let target = onMain({ balloon()?.targetVirtualMachineMemorySize }) else { + return .text(404, "no memory balloon device configured") + } + return .json(Balloon(targetGuestMemoryBytes: Int64(target))) + } + + private func setBalloon(_ body: Data) -> HTTPResponse { + guard onMain({ balloon() != nil }) else { return .text(404, "no memory balloon device configured") } + guard let target = try? snakeCaseDecoder.decode(Balloon.self, from: body).targetGuestMemoryBytes else { + return .text(400, "invalid balloon request: target_guest_memory_bytes is required") + } + guard target >= 0 else { return .text(400, "target_guest_memory_bytes must be non-negative") } + onMain { balloon()?.targetVirtualMachineMemorySize = UInt64(target) } + return .noContent() + } + + private func vmmShutdown() -> HTTPResponse { + Log.info("VMM shutdown requested") + var response = HTTPResponse.noContent() + response.after = { [self] in + // The process exits once the VM stops. + DispatchQueue.main.async { + guard self.vm.canStop else { return } + self.forcedStop = true + self.vm.stop { _ in self.finish() } + } + } + return response + } + + // MARK: Vsock proxy + + // proxyVsock speaks Cloud Hypervisor's vsock protocol: the client sends + // "CONNECT {port}\n", the shim answers "OK {port}\n", then proxies bytes. + private func proxyVsock(_ client: Int32) { + defer { close(client) } + guard let line = readLine(client, limit: 64) else { + Log.error("failed to read vsock handshake") + return + } + let fields = line.trimmingCharacters(in: .newlines).split(separator: " ") + guard fields.count == 2, fields[0] == "CONNECT", let port = UInt32(fields[1]) else { + Log.error("invalid vsock handshake", ["cmd": line]) + _ = writeAll(client, Data("ERR invalid command: \(line)".utf8)) + return + } + guard let device = onMain({ vm.socketDevices.first as? VZVirtioSocketDevice }) else { + Log.error("no vsock device configured") + _ = writeAll(client, Data("ERR no vsock device\n".utf8)) + return + } + var connection: VZVirtioSocketConnection? + let connectError = awaitMain { done in + device.connect(toPort: port) { result in + switch result { + case .success(let c): + connection = c + done(nil) + case .failure(let error): + done(error) + } + } + } + guard let connection else { + let message = connectError?.localizedDescription ?? "unknown error" + Log.error("failed to connect to guest vsock", ["port": port, "error": message]) + _ = writeAll(client, Data("ERR connect failed: \(message)\n".utf8)) + return + } + defer { connection.close() } + guard writeAll(client, Data("OK \(port)\n".utf8)) else { + Log.error("failed to send OK response") + return + } + Log.debug("vsock connection established", ["port": port]) + let guest = connection.fileDescriptor + // When either direction ends, shut both sockets down to wake the other copy, + // then wait for it before the descriptors are closed. + let finished = DispatchSemaphore(value: 0) + for (src, dst) in [(client, guest), (guest, client)] { + Thread.detachNewThread { + pump(from: src, to: dst) + shutdown(client, SHUT_RDWR) + shutdown(guest, SHUT_RDWR) + finished.signal() + } + } + finished.wait() + finished.wait() + } +} + +extension Result { + var error: Failure? { + if case .failure(let error) = self { return error } + return nil + } +} + +// stateName matches the Cloud Hypervisor state names the client expects. +func stateName(_ state: VZVirtualMachine.State) -> String { + switch state { + case .stopped: return "Shutdown" + case .running: return "Running" + case .paused: return "Paused" + case .error: return "Error" + case .starting: return "Starting" + case .pausing: return "Pausing" + case .resuming: return "Resuming" + case .stopping: return "Stopping" + case .saving: return "Saving" + case .restoring: return "Restoring" + @unknown default: return "Unknown" + } +} diff --git a/cmd/vz-shim/Socket.swift b/cmd/vz-shim/Socket.swift new file mode 100644 index 000000000..5c6b84e9f --- /dev/null +++ b/cmd/vz-shim/Socket.swift @@ -0,0 +1,194 @@ +import Foundation + +// UnixListener accepts stream connections on a Unix socket, each handled on its own +// thread. Closing it unlinks the socket path, as Go's listener does. +final class UnixListener { + let path: String + private let fd: Int32 + + init(path: String) throws { + unlink(path) + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + let capacity = MemoryLayout.size(ofValue: addr.sun_path) + guard path.utf8.count < capacity else { throw ShimError("socket path too long: \(path)") } + withUnsafeMutableBytes(of: &addr.sun_path) { buffer in + buffer.copyBytes(from: path.utf8) + buffer[path.utf8.count] = 0 + } + addr.sun_len = UInt8(MemoryLayout.size) + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { throw ShimError("socket: \(errnoString())") } + let bound = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, socklen_t(MemoryLayout.size)) } + } + guard bound == 0, listen(fd, 64) == 0 else { + let message = errnoString() + Darwin.close(fd) + throw ShimError("listen on \(path): \(message)") + } + self.path = path + self.fd = fd + } + + func serve(_ handle: @escaping (Int32) -> Void) { + Thread.detachNewThread { [fd] in + while true { + let conn = accept(fd, nil, nil) + if conn < 0 { + if errno == EINTR { continue } + return + } + var on: Int32 = 1 + setsockopt(conn, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout.size)) + Thread.detachNewThread { handle(conn) } + } + } + } + + func close() { + Darwin.close(fd) + unlink(path) + } +} + +func errnoString() -> String { + String(cString: strerror(errno)) +} + +func writeAll(_ fd: Int32, _ data: Data) -> Bool { + data.withUnsafeBytes { buffer -> Bool in + var offset = 0 + while offset < buffer.count { + let n = write(fd, buffer.baseAddress! + offset, buffer.count - offset) + if n < 0 { + if errno == EINTR { continue } + return false + } + offset += n + } + return true + } +} + +// readLine reads one newline-terminated line a byte at a time, so nothing after the +// newline is consumed. +func readLine(_ fd: Int32, limit: Int) -> String? { + var bytes: [UInt8] = [] + var byte: UInt8 = 0 + while bytes.count < limit { + let n = read(fd, &byte, 1) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + bytes.append(byte) + if byte == 0x0a { return String(decoding: bytes, as: UTF8.self) } + } + return nil +} + +// pump copies src to dst until either side fails or reaches EOF. +func pump(from src: Int32, to dst: Int32) { + var buffer = [UInt8](repeating: 0, count: 64 << 10) + while true { + let n = read(src, &buffer, buffer.count) + if n < 0, errno == EINTR { continue } + if n <= 0 { return } + if !writeAll(dst, Data(buffer[0.. Void)? + + static func noContent() -> HTTPResponse { HTTPResponse(status: 204) } + + static func text(_ status: Int, _ message: String) -> HTTPResponse { + HTTPResponse(status: status, body: (message + "\n").data(using: .utf8)!) + } + + static func json(_ value: T) -> HTTPResponse { + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + encoder.outputFormatting = .sortedKeys + let data = (try? encoder.encode(value)) ?? Data("{}".utf8) + return HTTPResponse(status: 200, contentType: "application/json", body: data + Data([0x0a])) + } +} + +// snakeCaseDecoder decodes control API request bodies, which use snake_case keys. +let snakeCaseDecoder: JSONDecoder = { + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + return decoder +}() + +private let reasons = [200: "OK", 204: "No Content", 400: "Bad Request", 404: "Not Found", 405: "Method Not Allowed", 500: "Internal Server Error"] + +func serveHTTP(_ fd: Int32, _ handle: (HTTPRequest) -> HTTPResponse) { + defer { close(fd) } + guard let request = readHTTPRequest(fd) else { + _ = writeHTTPResponse(fd, .text(400, "malformed request")) + return + } + let response = handle(request) + _ = writeHTTPResponse(fd, response) + response.after?() +} + +private func readHTTPRequest(_ fd: Int32) -> HTTPRequest? { + var data = Data() + var buffer = [UInt8](repeating: 0, count: 4096) + let separator = Data("\r\n\r\n".utf8) + var headerEnd: Range? + while headerEnd == nil { + guard data.count < 64 << 10 else { return nil } + let n = read(fd, &buffer, buffer.count) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + data.append(contentsOf: buffer[0..= 2 else { return nil } + var contentLength = 0 + for line in lines { + let parts = line.split(separator: ":", maxSplits: 1) + if parts.count == 2, parts[0].lowercased() == "content-length" { + contentLength = Int(parts[1].trimmingCharacters(in: .whitespaces)) ?? -1 + } + } + guard (0...(1 << 20)).contains(contentLength) else { return nil } + var body = Data(data[headerEnd!.upperBound...]) + while body.count < contentLength { + let n = read(fd, &buffer, min(buffer.count, contentLength - body.count)) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + body.append(contentsOf: buffer[0.. Bool { + var head = "HTTP/1.1 \(response.status) \(reasons[response.status] ?? "Status")\r\n" + if response.status != 204 { + head += "Content-Type: \(response.contentType)\r\nContent-Length: \(response.body.count)\r\n" + } + head += "Connection: close\r\n\r\n" + return writeAll(fd, Data(head.utf8) + response.body) +} diff --git a/cmd/vz-shim/Storage.swift b/cmd/vz-shim/Storage.swift new file mode 100644 index 000000000..81a79ac04 --- /dev/null +++ b/cmd/vz-shim/Storage.swift @@ -0,0 +1,56 @@ +import DiskImageKit +import Foundation +import Virtualization + +func makeStorageDevices(_ disks: [DiskConfig]) throws -> [VZStorageDeviceConfiguration] { + try disks.map { disk in + guard FileManager.default.fileExists(atPath: disk.path) else { + throw ShimError("configure storage: disk image not found: \(disk.path)") + } + if disk.path.hasSuffix(".qcow2") { + throw ShimError("configure storage: qcow2 not supported by vz, use raw format: \(disk.path)") + } + let readOnly = disk.readonly ?? false + let attachment: VZStorageDeviceAttachment + do { + if let parents = disk.parents, !parents.isEmpty { + attachment = try stackedAttachment(parents: parents, overlay: disk.path, readOnly: readOnly) + } else { + attachment = try VZDiskImageStorageDeviceAttachment(url: URL(fileURLWithPath: disk.path), readOnly: readOnly) + } + } catch let error as ShimError { + throw error + } catch { + throw ShimError("configure storage: create disk attachment for \(disk.path): \(error.localizedDescription)") + } + return VZVirtioBlockDeviceConfiguration(attachment: attachment) + } +} + +// stackedAttachment attaches an ASIF overlay above its read-only parents, base +// first. Each image must name the one below it as its parent; hypeman creates them. +func stackedAttachment(parents: [String], overlay: String, readOnly: Bool) throws -> VZStorageDeviceAttachment { + guard #available(macOS 27.0, *) else { + throw ShimError("configure storage: stacked disks require macOS 27 or newer") + } + for path in parents where !FileManager.default.fileExists(atPath: path) { + throw ShimError("configure storage: disk parent not found: \(path)") + } + var stacked = try openASIF(parents[0], mode: .readOnly) + for path in parents.dropFirst() { + stacked = try stacked.appending(try openASIF(path, mode: .readOnly)) + } + stacked = try stacked.appending(try openASIF(overlay, mode: readOnly ? .readOnly : .readWrite)) + Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": parents.count + 1]) + return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) +} + +// openASIF opens one image of a stack, which must be ASIF: the base and every overlay. +@available(macOS 27.0, *) +func openASIF(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { + let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) + guard image.format == .asif else { + throw ShimError("configure storage: stacked disk images must use ASIF format: \(path)") + } + return image +} diff --git a/cmd/vz-shim/VM.swift b/cmd/vz-shim/VM.swift new file mode 100644 index 000000000..4429f9a1f --- /dev/null +++ b/cmd/vz-shim/VM.swift @@ -0,0 +1,171 @@ +import Foundation +import Virtualization + +// makeConfiguration builds a validated VM configuration. A Linux guest without a +// machine identifier gets a new one, recorded in config so snapshots keep it. +func makeConfiguration(_ config: inout ShimConfig) throws -> VZVirtualMachineConfiguration { + if config.isMacGuest { + return try makeMacConfiguration(config) + } + return try makeLinuxConfiguration(&config) +} + +// makeMacConfiguration boots an imported macOS disk with its platform identity. +// It has no serial console, entropy device, Rosetta share or memory balloon. +func makeMacConfiguration(_ c: ShimConfig) throws -> VZVirtualMachineConfiguration { + #if arch(arm64) + if c.enableRosetta == true || c.enableMemoryBalloon == true || c.requireMemoryBalloon == true { + throw ShimError("macOS guests do not support Linux Rosetta or memory ballooning") + } + guard let modelText = c.macHardwareModelData, !modelText.isEmpty, + let idText = c.macMachineIdentifierData, !idText.isEmpty, + let auxPath = c.macAuxStoragePath, !auxPath.isEmpty + else { + throw ShimError("create boot loader: macOS boot requires hardware model, machine identifier, and auxiliary storage") + } + guard let modelData = decodeBase64(modelText) else { + throw ShimError("configure Mac platform: decode Mac hardware model") + } + guard let model = VZMacHardwareModel(dataRepresentation: modelData) else { + throw ShimError("configure Mac platform: invalid Mac hardware model") + } + guard model.isSupported else { + throw ShimError("configure Mac platform: Mac hardware model is unsupported on this host") + } + guard let idData = decodeBase64(idText) else { + throw ShimError("configure Mac platform: decode Mac machine identifier") + } + guard let identifier = VZMacMachineIdentifier(dataRepresentation: idData) else { + throw ShimError("configure Mac platform: invalid Mac machine identifier") + } + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = model + platform.machineIdentifier = identifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: URL(fileURLWithPath: auxPath)) + + let vc = VZVirtualMachineConfiguration() + vc.bootLoader = VZMacOSBootLoader() + vc.cpuCount = c.vcpus ?? 0 + vc.memorySize = UInt64(max(0, c.memoryBytes ?? 0)) + vc.platform = platform + Log.debug("macOS VM config", ["vcpus": vc.cpuCount, "memory_bytes": vc.memorySize]) + vc.networkDevices = try makeNetworkDevices(c.networks ?? []) + vc.storageDevices = try makeStorageDevices(c.disks ?? []) + + let graphics = VZMacGraphicsDeviceConfiguration() + // 3024x1964 pixels at 2x: 1512x982 points, the logical size of a 14-inch MacBook Pro. + graphics.displays = [VZMacGraphicsDisplayConfiguration(widthInPixels: 3024, heightInPixels: 1964, pixelsPerInch: 160)] + vc.graphicsDevices = [graphics] + vc.keyboards = [VZMacKeyboardConfiguration()] + vc.pointingDevices = [VZMacTrackpadConfiguration()] + vc.audioDevices = [makeMacAudio()] + vc.socketDevices = [VZVirtioSocketDeviceConfiguration()] + do { + try vc.validate() + } catch { + throw ShimError("invalid vm configuration: \(error.localizedDescription)") + } + return vc + #else + throw ShimError("macOS guests require Apple silicon") + #endif +} + +// makeMacAudio gives the guest an audio device. Virtualization attaches audio only +// through a sound device, so without one the guest boots with no CoreAudio devices: +// `system_profiler SPAudioDataType` lists none and `afplay` fails with +// `AudioQueueStart failed (-66680)`. +// +// The output stream has no host sink. The samples are discarded on the host, which +// is what a headless browser VM wants -- nothing should play on the machine running +// the guests -- while the guest still has a real output device, so applications can +// render audio and ScreenCaptureKit has a mix to capture for the live view. +func makeMacAudio() -> VZAudioDeviceConfiguration { + let sound = VZVirtioSoundDeviceConfiguration() + sound.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()] + return sound +} + +func makeLinuxConfiguration(_ c: inout ShimConfig) throws -> VZVirtualMachineConfiguration { + // vz uses hvc0 for the serial console. + var kernelArgs = c.kernelArgs ?? "" + if kernelArgs.isEmpty { + kernelArgs = "console=hvc0 root=/dev/vda" + } else { + kernelArgs = kernelArgs.replacingOccurrences(of: "console=ttyS0", with: "console=hvc0") + } + let bootLoader = VZLinuxBootLoader(kernelURL: URL(fileURLWithPath: c.kernelPath ?? "")) + bootLoader.commandLine = kernelArgs + if let initrd = c.initrdPath, !initrd.isEmpty { + bootLoader.initialRamdiskURL = URL(fileURLWithPath: initrd) + } + + let vc = VZVirtualMachineConfiguration() + vc.bootLoader = bootLoader + vc.cpuCount = cpuCount(c.vcpus ?? 0) + vc.memorySize = memorySize(UInt64(max(0, c.memoryBytes ?? 0))) + Log.debug("VM config", ["vcpus": vc.cpuCount, "memory_bytes": vc.memorySize, "kernel": c.kernelPath ?? "", "initrd": c.initrdPath ?? ""]) + + vc.serialPorts = [try makeSerialPort(c.serialLogPath)] + vc.networkDevices = try makeNetworkDevices(c.networks ?? []) + vc.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + vc.storageDevices = try makeStorageDevices(c.disks ?? []) + vc.directorySharingDevices = try makeRosettaShare(c) + vc.platform = try makeGenericPlatform(&c) + vc.socketDevices = [VZVirtioSocketDeviceConfiguration()] + if c.enableMemoryBalloon == true { + vc.memoryBalloonDevices = [VZVirtioTraditionalMemoryBalloonDeviceConfiguration()] + } + do { + try vc.validate() + } catch { + throw ShimError("invalid vm configuration: \(error.localizedDescription)") + } + return vc +} + +func makeSerialPort(_ logPath: String?) throws -> VZSerialPortConfiguration { + guard let reader = FileHandle(forReadingAtPath: "/dev/null") else { + throw ShimError("configure serial: open /dev/null for reading") + } + let writer: FileHandle + if let logPath, !logPath.isEmpty { + let fd = open(logPath, O_CREAT | O_WRONLY | O_APPEND, 0o644) + guard fd >= 0 else { throw ShimError("configure serial: open serial log file: \(String(cString: strerror(errno)))") } + writer = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + } else { + guard let null = FileHandle(forWritingAtPath: "/dev/null") else { + throw ShimError("configure serial: open /dev/null for writing") + } + writer = null + } + let port = VZVirtioConsoleDeviceSerialPortConfiguration() + port.attachment = VZFileHandleSerialPortAttachment(fileHandleForReading: reader, fileHandleForWriting: writer) + return port +} + +// makeRosettaShare attaches the Linux Rosetta virtio-fs share when requested. The +// host must have Rosetta installed; anything else is a startup error. +func makeRosettaShare(_ c: ShimConfig) throws -> [VZDirectorySharingDeviceConfiguration] { + guard c.enableRosetta == true else { return [] } + #if arch(arm64) + switch VZLinuxRosettaDirectoryShare.availability { + case .installed: + break + case .notInstalled: + throw ShimError("configure directory sharing: rosetta requested but not installed (run: softwareupdate --install-rosetta)") + default: + throw ShimError("configure directory sharing: rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") + } + let device = VZVirtioFileSystemDeviceConfiguration(tag: rosettaMountTag) + do { + device.share = try VZLinuxRosettaDirectoryShare() + } catch { + throw ShimError("configure directory sharing: create rosetta directory share: \(error.localizedDescription)") + } + Log.info("attached rosetta directory share", ["tag": rosettaMountTag]) + return [device] + #else + throw ShimError("configure directory sharing: rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") + #endif +} diff --git a/cmd/vz-shim/macos_arm64.go b/cmd/vz-shim/macos_arm64.go deleted file mode 100644 index e2887abad..000000000 --- a/cmd/vz-shim/macos_arm64.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "encoding/base64" - "fmt" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func newMacBootLoader(c *shimconfig.ShimConfig) (vz.BootLoader, error) { - if c.MacHardwareModelData == "" || c.MacMachineIdentifierData == "" || c.MacAuxStoragePath == "" { - return nil, fmt.Errorf("macOS boot requires hardware model, machine identifier, and auxiliary storage") - } - return vz.NewMacOSBootLoader() -} -func configureMacPlatform(vc *vz.VirtualMachineConfiguration, c *shimconfig.ShimConfig) error { - modelData, e := base64.StdEncoding.DecodeString(c.MacHardwareModelData) - if e != nil { - return fmt.Errorf("decode Mac hardware model: %w", e) - } - model, e := vz.NewMacHardwareModelWithData(modelData) - if e != nil { - return e - } - if !model.Supported() { - return fmt.Errorf("Mac hardware model is unsupported on this host") - } - idData, e := base64.StdEncoding.DecodeString(c.MacMachineIdentifierData) - if e != nil { - return fmt.Errorf("decode Mac machine identifier: %w", e) - } - id, e := vz.NewMacMachineIdentifierWithData(idData) - if e != nil { - return e - } - aux, e := vz.NewMacAuxiliaryStorage(c.MacAuxStoragePath) - if e != nil { - return e - } - platform, e := vz.NewMacPlatformConfiguration(vz.WithMacHardwareModel(model), vz.WithMacMachineIdentifier(id), vz.WithMacAuxiliaryStorage(aux)) - if e != nil { - return e - } - vc.SetPlatformVirtualMachineConfiguration(platform) - graphics, e := vz.NewMacGraphicsDeviceConfiguration() - if e != nil { - return e - } - // 3024x1964 pixels at 2x: 1512x982 points, the logical size of a 14-inch MacBook Pro. - display, e := vz.NewMacGraphicsDisplayConfiguration(3024, 1964, 160) - if e != nil { - return e - } - graphics.SetDisplays(display) - vc.SetGraphicsDevicesVirtualMachineConfiguration([]vz.GraphicsDeviceConfiguration{graphics}) - keyboard, e := vz.NewMacKeyboardConfiguration() - if e != nil { - return e - } - trackpad, e := vz.NewMacTrackpadConfiguration() - if e != nil { - return e - } - vc.SetKeyboardsVirtualMachineConfiguration([]vz.KeyboardConfiguration{keyboard}) - vc.SetPointingDevicesVirtualMachineConfiguration([]vz.PointingDeviceConfiguration{trackpad}) - return nil -} diff --git a/cmd/vz-shim/macos_arm64_test.go b/cmd/vz-shim/macos_arm64_test.go deleted file mode 100644 index 073c5387e..000000000 --- a/cmd/vz-shim/macos_arm64_test.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/require" - "testing" -) - -func TestMacBootRequiresCompleteIdentity(t *testing.T) { - for _, c := range []shimconfig.ShimConfig{ - {MacHardwareModelData: "AA=="}, - {MacMachineIdentifierData: "AA=="}, - {MacAuxStoragePath: "/missing"}, - } { - _, _, err := createVM(&c) - require.ErrorContains(t, err, "macOS boot requires") - } -} -func TestMacPlatformRejectsMalformedModel(t *testing.T) { - err := configureMacPlatform(nil, &shimconfig.ShimConfig{MacHardwareModelData: "not base64!"}) - require.ErrorContains(t, err, "decode Mac hardware model") -} diff --git a/cmd/vz-shim/macos_identity.go b/cmd/vz-shim/macos_identity.go deleted file mode 100644 index 43fd1dfdf..000000000 --- a/cmd/vz-shim/macos_identity.go +++ /dev/null @@ -1,48 +0,0 @@ -//go:build darwin - -package main - -import ( - "crypto/sha256" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "syscall" - - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// Preserve the identity lock until subprocess exit, including while paused. -// Never unlink a lock file: another waiter may already hold its inode open. -func lockMacOSIdentity(c *shimconfig.ShimConfig) (*os.File, error) { - if c.MacMachineIdentifierData == "" { - return nil, nil - } - data, err := base64.StdEncoding.DecodeString(c.MacMachineIdentifierData) - if err != nil { - return nil, fmt.Errorf("decode Mac identity for admission: %w", err) - } - hash := sha256.Sum256(data) - // Fixed directory, not TMPDIR: servers with different environments must contend on the same lock. - path := filepath.Join("/tmp", fmt.Sprintf("hypeman-mac-identity-%d-%x.lock", os.Getuid(), hash)) - // The shared /tmp name is predictable, so refuse symlinks and foreign-owned files. - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600) - if err != nil { - return nil, err - } - info, err := f.Stat() - if err != nil { - f.Close() - return nil, err - } - if st, ok := info.Sys().(*syscall.Stat_t); !ok || !info.Mode().IsRegular() || int(st.Uid) != os.Getuid() { - f.Close() - return nil, fmt.Errorf("Mac identity lock path is not an owned regular file") - } - if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - f.Close() - return nil, fmt.Errorf("Mac machine identity already in use: %w", err) - } - return f, nil -} diff --git a/cmd/vz-shim/macos_identity_test.go b/cmd/vz-shim/macos_identity_test.go deleted file mode 100644 index af56eb4c4..000000000 --- a/cmd/vz-shim/macos_identity_test.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build darwin - -package main - -import ( - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/require" - "testing" -) - -func TestMacOSIdentityLifetimeLock(t *testing.T) { - c := &shimconfig.ShimConfig{MacMachineIdentifierData: "aWRlbnRpdHk="} - first, err := lockMacOSIdentity(c) - require.NoError(t, err) - defer first.Close() - equivalent := *c - equivalent.MacMachineIdentifierData += "\n" - second, err := lockMacOSIdentity(&equivalent) - require.Error(t, err) - require.Nil(t, second) - require.NoError(t, first.Close()) - second, err = lockMacOSIdentity(c) - require.NoError(t, err) - second.Close() -} diff --git a/cmd/vz-shim/macos_other.go b/cmd/vz-shim/macos_other.go deleted file mode 100644 index ed256b850..000000000 --- a/cmd/vz-shim/macos_other.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func newMacBootLoader(*shimconfig.ShimConfig) (vz.BootLoader, error) { - return nil, fmt.Errorf("macOS guests require Apple silicon") -} -func configureMacPlatform(*vz.VirtualMachineConfiguration, *shimconfig.ShimConfig) error { - return fmt.Errorf("macOS guests require Apple silicon") -} diff --git a/cmd/vz-shim/macos_vm.go b/cmd/vz-shim/macos_vm.go deleted file mode 100644 index 269e9af82..000000000 --- a/cmd/vz-shim/macos_vm.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build darwin - -package main - -import ( - "fmt" - "log/slog" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func isMacGuest(config *shimconfig.ShimConfig) bool { - return config.MacHardwareModelData != "" || config.MacMachineIdentifierData != "" || config.MacAuxStoragePath != "" -} - -// createMacVM boots an imported macOS disk with its preserved platform identity. -// It has no serial console, entropy device, Rosetta share or memory balloon. -func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { - if config.EnableRosetta || config.EnableMemoryBalloon || config.RequireMemoryBalloon { - return nil, nil, fmt.Errorf("macOS guests do not support Linux Rosetta or memory ballooning") - } - bootLoader, err := newMacBootLoader(config) - if err != nil { - return nil, nil, fmt.Errorf("create boot loader: %w", err) - } - - vcpus := uint(config.VCPUs) - memoryBytes := uint64(config.MemoryBytes) - slog.Debug("macOS VM config", "vcpus", vcpus, "memory_bytes", memoryBytes) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) - if err != nil { - return nil, nil, fmt.Errorf("create vm configuration: %w", err) - } - if err := configureNetwork(vmConfig, config.Networks); err != nil { - return nil, nil, fmt.Errorf("configure network: %w", err) - } - if err := configureStorage(vmConfig, config.Disks); err != nil { - return nil, nil, fmt.Errorf("configure storage: %w", err) - } - if err := configureDirectorySharing(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure directory sharing: %w", err) - } - if err := configureMacPlatform(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure Mac platform: %w", err) - } - - vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create vsock device: %w", err) - } - vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) - - validated, err := vmConfig.Validate() - if err != nil { - return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) - } - if !validated { - return nil, nil, fmt.Errorf("invalid vm configuration") - } - vm, err := vz.NewVirtualMachine(vmConfig) - if err != nil { - return nil, nil, fmt.Errorf("create virtual machine: %w", err) - } - return vm, vmConfig, nil -} diff --git a/cmd/vz-shim/main.go b/cmd/vz-shim/main.go deleted file mode 100644 index 950602184..000000000 --- a/cmd/vz-shim/main.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build darwin - -// Package main implements hypeman-vz-shim, a subprocess that hosts vz VMs. -// This allows VMs to survive hypeman restarts by running in a separate process. -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log/slog" - "net" - "net/http" - "os" - "os/signal" - "path/filepath" - "syscall" - "time" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -const controlIdleTimeout = 30 * time.Second - -func main() { - configJSON := flag.String("config", "", "VM configuration as JSON") - flag.Parse() - - if *configJSON == "" { - fmt.Fprintln(os.Stderr, "error: -config is required") - os.Exit(1) - } - - var config shimconfig.ShimConfig - if err := json.Unmarshal([]byte(*configJSON), &config); err != nil { - fmt.Fprintf(os.Stderr, "error: invalid config JSON: %v\n", err) - os.Exit(1) - } - - // Setup logging to file - if err := setupLogging(config.LogPath); err != nil { - fmt.Fprintf(os.Stderr, "error: setup logging: %v\n", err) - os.Exit(1) - } - - slog.Info("vz-shim starting", "control_socket", config.ControlSocket, "vsock_socket", config.VsockSocket) - - identityLock, err := lockMacOSIdentity(&config) - if err != nil { - fmt.Fprintf(os.Stderr, "macOS identity admission: %v\n", err) - os.Exit(1) - } - if identityLock != nil { - defer identityLock.Close() - } - - // Create the VM - vm, vmConfig, err := createVM(&config) - if err != nil { - slog.Error("failed to create VM", "error", err) - fmt.Fprintf(os.Stderr, "failed to create VM: %v\n", err) - os.Exit(1) - } - - if config.RestoreMachineStatePath != "" { - if err := validateSaveRestoreSupport(vmConfig); err != nil { - slog.Error("save/restore not supported for VM config", "error", err) - fmt.Fprintf(os.Stderr, "save/restore not supported for VM config: %v\n", err) - os.Exit(1) - } - if err := restoreMachineState(vm, config.RestoreMachineStatePath); err != nil { - slog.Error("failed to restore VM machine state", "error", err, "path", config.RestoreMachineStatePath) - fmt.Fprintf(os.Stderr, "failed to restore VM machine state: %v\n", err) - os.Exit(1) - } - slog.Info("VM restored from machine state", "path", config.RestoreMachineStatePath, "state", vm.State()) - } else { - if err := vm.Start(); err != nil { - slog.Error("failed to start VM", "error", err) - fmt.Fprintf(os.Stderr, "failed to start VM: %v\n", err) - os.Exit(1) - } - slog.Info("VM started", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024) - } - - // Create the shim server - server := NewShimServer(vm, vmConfig, config) - - // Start control socket listener (remove stale socket from previous run) - os.Remove(config.ControlSocket) - controlListener, err := net.Listen("unix", config.ControlSocket) - if err != nil { - slog.Error("failed to listen on control socket", "error", err, "path", config.ControlSocket) - os.Exit(1) - } - defer controlListener.Close() - - // Start vsock proxy listener (remove stale socket from previous run) - os.Remove(config.VsockSocket) - vsockListener, err := net.Listen("unix", config.VsockSocket) - if err != nil { - slog.Error("failed to listen on vsock socket", "error", err, "path", config.VsockSocket) - os.Exit(1) - } - defer vsockListener.Close() - - // Start HTTP server for control API - httpServer := &http.Server{ - Handler: server.Handler(), - IdleTimeout: controlIdleTimeout, - } - go func() { - slog.Info("control API listening", "socket", config.ControlSocket) - if err := httpServer.Serve(controlListener); err != nil && err != http.ErrServerClosed { - slog.Error("control API server error", "error", err) - } - }() - - // Start vsock proxy - go func() { - slog.Info("vsock proxy listening", "socket", config.VsockSocket) - server.ServeVsock(vsockListener) - }() - - // Wait for shutdown signal or VM stop - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - - // Monitor VM state. Notifications can be dropped and their values can be stale, - // so each one, and the start, re-reads the current state. - stateChanged := make(chan vz.VirtualMachineState, 8) - stopNotify, err := vm.StateChangedNotify(stateChanged) - if err != nil { - slog.Error("failed to watch VM state", "error", err) - os.Exit(1) - } - defer stopNotify() - go func() { - for { - newState := vm.State() - if newState != vz.VirtualMachineStateStopped && newState != vz.VirtualMachineStateError { - select { - case <-ctx.Done(): - return - case <-stateChanged: - } - newState = vm.State() - slog.Info("VM state changed", "state", newState) - } - if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { - if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() { - writeStopReceipt(config.StopReceiptPath) - } - slog.Info("VM stopped, shutting down shim") - cancel() - return - } - } - }() - - select { - case sig := <-sigChan: - slog.Info("received signal, shutting down", "signal", sig) - case <-ctx.Done(): - slog.Info("context cancelled, shutting down") - } - - // Graceful shutdown - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer shutdownCancel() - - httpServer.Shutdown(shutdownCtx) - - if vm.State() == vz.VirtualMachineStateRunning { - slog.Info("stopping VM") - if vm.CanStop() { - server.forcedStop.Store(true) - vm.Stop() - } - } - - slog.Info("vz-shim shutdown complete") -} - -// writeStopReceipt records that the guest stopped the VM itself. The host counts a -// macOS stop as graceful only with this receipt. -func writeStopReceipt(path string) { - if path == "" { - return - } - if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339Nano)+"\n"), 0600); err != nil { - slog.Error("failed to write stop receipt", "error", err, "path", path) - } -} - -func setupLogging(logPath string) error { - if logPath == "" { - // Log to stderr if no path specified - slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) - return nil - } - - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return fmt.Errorf("create log directory: %w", err) - } - - file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return fmt.Errorf("open log file: %w", err) - } - - slog.SetDefault(slog.New(slog.NewJSONHandler(file, &slog.HandlerOptions{Level: slog.LevelDebug}))) - return nil -} diff --git a/cmd/vz-shim/main.swift b/cmd/vz-shim/main.swift new file mode 100644 index 000000000..a14ef4500 --- /dev/null +++ b/cmd/vz-shim/main.swift @@ -0,0 +1,110 @@ +// hypeman-vz-shim is a subprocess that hosts one vz VM, so VMs survive hypeman +// restarts. Hypeman starts it with -config and talks to it over +// the control and vsock sockets named in that config. +import Foundation +import Virtualization + +func configArgument() -> String? { + let args = CommandLine.arguments.dropFirst() + var iterator = args.makeIterator() + while let arg = iterator.next() { + if arg == "-config" || arg == "--config" { + return iterator.next() + } + for prefix in ["-config=", "--config="] where arg.hasPrefix(prefix) { + return String(arg.dropFirst(prefix.count)) + } + } + return nil +} + +signal(SIGPIPE, SIG_IGN) + +guard let configJSON = configArgument(), !configJSON.isEmpty else { + printError("error: -config is required") + exit(1) +} +var config: ShimConfig +do { + config = try JSONDecoder().decode(ShimConfig.self, from: Data(configJSON.utf8)) +} catch { + printError("error: invalid config JSON: \(error)") + exit(1) +} +do { + try Log.setup(path: config.logPath) +} catch { + printError("error: setup logging: \(error)") + exit(1) +} +Log.info("vz-shim starting", ["control_socket": config.controlSocket ?? "", "vsock_socket": config.vsockSocket ?? ""]) + +let identityLock: Int32? +do { + identityLock = try lockMacOSIdentity(config) +} catch { + printError("macOS identity admission: \(error)") + exit(1) +} + +let vmConfig: VZVirtualMachineConfiguration +do { + vmConfig = try makeConfiguration(&config) +} catch { + Log.error("failed to create VM", ["error": "\(error)"]) + printError("failed to create VM: \(error)") + exit(1) +} +let shim = Shim(vm: VZVirtualMachine(configuration: vmConfig, queue: .main), vmConfig: vmConfig, config: config) + +func serveOrExit() { + do { + try shim.listen() + } catch { + Log.error("failed to listen", ["error": "\(error)"]) + exit(1) + } +} + +if let restorePath = config.restoreMachineStatePath, !restorePath.isEmpty { + if let reason = shim.saveRestoreUnsupportedReason() { + Log.error("save/restore not supported for VM config", ["error": reason]) + printError("save/restore not supported for VM config: \(reason)") + exit(1) + } + shim.vm.restoreMachineStateFrom(url: URL(fileURLWithPath: restorePath)) { error in + if let error { + Log.error("failed to restore VM machine state", ["error": error.localizedDescription, "path": restorePath]) + printError("failed to restore VM machine state: \(error.localizedDescription)") + exit(1) + } + Log.info("VM restored from machine state", ["path": restorePath, "state": stateName(shim.vm.state)]) + serveOrExit() + } +} else { + shim.vm.start { result in + if let error = result.error { + Log.error("failed to start VM", ["error": error.localizedDescription]) + printError("failed to start VM: \(error.localizedDescription)") + exit(1) + } + Log.info("VM started", ["vcpus": config.vcpus ?? 0, "memory_mb": (config.memoryBytes ?? 0) / 1024 / 1024]) + serveOrExit() + } +} + +var signalSources: [DispatchSourceSignal] = [] +for number in [SIGTERM, SIGINT] { + signal(number, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: number, queue: .main) + source.setEventHandler { + Log.info("received signal, shutting down", ["signal": number]) + shim.stopForSignal() + } + source.resume() + signalSources.append(source) +} + +withExtendedLifetime(identityLock) { + dispatchMain() +} diff --git a/cmd/vz-shim/rosetta_arm64.go b/cmd/vz-shim/rosetta_arm64.go deleted file mode 100644 index 5495f7731..000000000 --- a/cmd/vz-shim/rosetta_arm64.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "fmt" - "log/slog" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// configureDirectorySharing attaches the Linux Rosetta virtio-fs share when -// EnableRosetta is set. The host must have Rosetta installed; NotInstalled and -// NotSupported are hard errors surfaced back through the shim-failure path. -func configureDirectorySharing(vmConfig *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - if !config.EnableRosetta { - return nil - } - - switch vz.LinuxRosettaDirectoryShareAvailability() { - case vz.LinuxRosettaAvailabilityInstalled: - case vz.LinuxRosettaAvailabilityNotInstalled: - return fmt.Errorf("rosetta requested but not installed (run: softwareupdate --install-rosetta)") - default: - return fmt.Errorf("rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") - } - - fsDevice, err := vz.NewVirtioFileSystemDeviceConfiguration(shimconfig.RosettaMountTag) - if err != nil { - return fmt.Errorf("create rosetta virtio-fs device (tag %q): %w", shimconfig.RosettaMountTag, err) - } - - share, err := vz.NewLinuxRosettaDirectoryShare() - if err != nil { - return fmt.Errorf("create rosetta directory share: %w", err) - } - fsDevice.SetDirectoryShare(share) - - vmConfig.SetDirectorySharingDevicesVirtualMachineConfiguration( - []vz.DirectorySharingDeviceConfiguration{fsDevice}, - ) - slog.Info("attached rosetta directory share", "tag", shimconfig.RosettaMountTag) - return nil -} diff --git a/cmd/vz-shim/rosetta_arm64_test.go b/cmd/vz-shim/rosetta_arm64_test.go deleted file mode 100644 index 25f3b5fad..000000000 --- a/cmd/vz-shim/rosetta_arm64_test.go +++ /dev/null @@ -1,81 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRosettaConfigureDirectorySharingDisabled(t *testing.T) { - // When Rosetta is not requested the call is a no-op and never touches the - // VM configuration, so a nil config is safe. - err := configureDirectorySharing(nil, &shimconfig.ShimConfig{EnableRosetta: false}) - assert.NoError(t, err) -} - -// TestRosettaConfigureDirectorySharingEnabled exercises the error paths -// unconditionally. The attach+validate path requires Rosetta to be installed on -// the runner; it is covered by TestRosettaConfigureDirectorySharingInstalled, -// which skips (visibly, under -v) when Rosetta is unavailable rather than -// silently asserting nothing. -func TestRosettaConfigureDirectorySharingEnabled(t *testing.T) { - cfg := &shimconfig.ShimConfig{EnableRosetta: true} - - switch vz.LinuxRosettaDirectoryShareAvailability() { - case vz.LinuxRosettaAvailabilityNotInstalled: - err := configureDirectorySharing(nil, cfg) - require.Error(t, err) - assert.Contains(t, err.Error(), "not installed") - case vz.LinuxRosettaAvailabilityNotSupported: - err := configureDirectorySharing(nil, cfg) - require.Error(t, err) - assert.Contains(t, err.Error(), "not supported") - } -} - -// TestRosettaConfigureDirectorySharingInstalled asserts that, when Rosetta is -// installed, configureDirectorySharing attaches the virtio-fs share and the -// resulting VM configuration validates. It skips otherwise so the log records -// whether the attach path actually ran on this runner. -func TestRosettaConfigureDirectorySharingInstalled(t *testing.T) { - if vz.LinuxRosettaDirectoryShareAvailability() != vz.LinuxRosettaAvailabilityInstalled { - t.Skip("rosetta not installed on this host; skipping attach+validate coverage") - } - - vmConfig := newTestVMConfiguration(t) - require.NoError(t, configureDirectorySharing(vmConfig, &shimconfig.ShimConfig{EnableRosetta: true})) - ok, err := vmConfig.Validate() - if err != nil && strings.Contains(err.Error(), "entitlement") { - if os.Getenv("HYPEMAN_VZ_SIGNED") != "" { - t.Fatalf("test binary should be signed with the virtualization entitlement (run via `make test-vz-shim-signed`): %v", err) - } - t.Skip("unsigned test binary lacks the virtualization entitlement; run via `make test-vz-shim-signed`") - } - require.NoError(t, err) - assert.True(t, ok) -} - -// newTestVMConfiguration builds a minimal valid VM configuration for tests that -// need to attach a device. The kernel file only needs to exist for the boot -// loader constructor. -func newTestVMConfiguration(t *testing.T) *vz.VirtualMachineConfiguration { - t.Helper() - - kernelPath := filepath.Join(t.TempDir(), "vmlinuz") - require.NoError(t, os.WriteFile(kernelPath, []byte("kernel"), 0o644)) - - bootLoader, err := vz.NewLinuxBootLoader(kernelPath, vz.WithCommandLine("console=hvc0")) - require.NoError(t, err) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, 1, 256*1024*1024) - require.NoError(t, err) - return vmConfig -} diff --git a/cmd/vz-shim/rosetta_other.go b/cmd/vz-shim/rosetta_other.go deleted file mode 100644 index 5f45abb06..000000000 --- a/cmd/vz-shim/rosetta_other.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// configureDirectorySharing is the non-arm64 stub: Rosetta is Apple-silicon-only, -// so it errors when requested and is otherwise a no-op. cmd/vz-shim is arm64-only -// in practice (Code-Hex/vz v3.7.1 doesn't compile for darwin/amd64); this only -// satisfies the !arm64 type check, like save_restore_unsupported.go. -func configureDirectorySharing(_ *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - if config.EnableRosetta { - return fmt.Errorf("rosetta is only available on Apple silicon") - } - return nil -} diff --git a/cmd/vz-shim/save_restore_arm64.go b/cmd/vz-shim/save_restore_arm64.go deleted file mode 100644 index a5d522216..000000000 --- a/cmd/vz-shim/save_restore_arm64.go +++ /dev/null @@ -1,29 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "fmt" - - "github.com/Code-Hex/vz/v4" -) - -func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { - ok, err := vmConfig.ValidateSaveRestoreSupport() - if err != nil { - return err - } - if !ok { - return fmt.Errorf("virtual machine configuration does not support save/restore") - } - return nil -} - -func saveMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return vm.SaveMachineStateToPath(snapshotPath) -} - -func restoreMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - // The vz wrapper accepts a filesystem path and constructs a file URL internally. - return vm.RestoreMachineStateFromURL(snapshotPath) -} diff --git a/cmd/vz-shim/save_restore_unsupported.go b/cmd/vz-shim/save_restore_unsupported.go deleted file mode 100644 index 47ce10fd0..000000000 --- a/cmd/vz-shim/save_restore_unsupported.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - "runtime" - - "github.com/Code-Hex/vz/v4" -) - -func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} - -func saveMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} - -func restoreMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go deleted file mode 100644 index 100e69276..000000000 --- a/cmd/vz-shim/server.go +++ /dev/null @@ -1,422 +0,0 @@ -//go:build darwin - -package main - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "log/slog" - "net" - "net/http" - "os" - "path/filepath" - "sync" - "sync/atomic" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// ShimServer handles control API and vsock proxy for a vz VM. -type ShimServer struct { - vm *vz.VirtualMachine - vmConfig *vz.VirtualMachineConfiguration - shimConfig shimconfig.ShimConfig - mu sync.RWMutex - // forcedStop is set before the shim stops the VM itself, so that stop is not - // mistaken for the guest powering off. - forcedStop atomic.Bool -} - -// NewShimServer creates a new shim server. -func NewShimServer(vm *vz.VirtualMachine, vmConfig *vz.VirtualMachineConfiguration, config shimconfig.ShimConfig) *ShimServer { - return &ShimServer{ - vm: vm, - vmConfig: vmConfig, - shimConfig: config, - } -} - -// VMInfoResponse matches the cloud-hypervisor VmInfo structure. -type VMInfoResponse struct { - State string `json:"state"` - MemoryBalloonDevices int `json:"memory_balloon_devices,omitempty"` -} - -type snapshotRequest struct { - DestinationPath string `json:"destination_path"` -} - -type balloonRequest struct { - TargetGuestMemoryBytes int64 `json:"target_guest_memory_bytes"` -} - -// Handler returns the HTTP handler for the control API. -func (s *ShimServer) Handler() http.Handler { - mux := http.NewServeMux() - - // Match cloud-hypervisor API patterns - mux.HandleFunc("GET /api/v1/vm.info", s.handleVMInfo) - mux.HandleFunc("PUT /api/v1/vm.pause", s.handlePause) - mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume) - mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown) - mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot) - mux.HandleFunc("GET /api/v1/vm.snapshot-support", s.handleSnapshotSupport) - mux.HandleFunc("GET /api/v1/vm.balloon", s.handleGetBalloon) - mux.HandleFunc("PUT /api/v1/vm.balloon", s.handleSetBalloon) - mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton) - mux.HandleFunc("GET /api/v1/vmm.ping", s.handlePing) - mux.HandleFunc("PUT /api/v1/vmm.shutdown", s.handleVMMShutdown) - - return mux -} - -func (s *ShimServer) handleVMInfo(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - - state := vzStateToString(s.vm.State()) - resp := VMInfoResponse{ - State: state, - MemoryBalloonDevices: len(s.vm.MemoryBalloonDevices()), - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) -} - -func (s *ShimServer) handlePause(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.vm.CanPause() { - http.Error(w, "cannot pause VM", http.StatusBadRequest) - return - } - - if err := s.vm.Pause(); err != nil { - slog.Error("failed to pause VM", "error", err) - http.Error(w, fmt.Sprintf("pause failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM paused") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleResume(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.vm.CanResume() { - http.Error(w, "cannot resume VM", http.StatusBadRequest) - return - } - - if err := s.vm.Resume(); err != nil { - slog.Error("failed to resume VM", "error", err) - http.Error(w, fmt.Sprintf("resume failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM resumed") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - // Send ACPI power button (graceful shutdown signal to guest). - // The caller (instance manager) handles timeout/force-kill if the guest - // doesn't shut down in time. Force-kill is in handleVMMShutdown / killHypervisor. - success, err := s.vm.RequestStop() - if err != nil || !success { - slog.Error("RequestStop failed", "error", err, "success", success) - http.Error(w, fmt.Sprintf("shutdown failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM graceful shutdown requested (ACPI power button)") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleSnapshotSupport(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - response := struct { - Supported bool `json:"supported"` - Reason string `json:"reason,omitempty"` - }{Supported: true} - if err := validateSaveRestoreSupport(s.vmConfig); err != nil { - response.Supported = false - response.Reason = err.Error() - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) -} - -func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - var req snapshotRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, fmt.Sprintf("invalid snapshot request: %v", err), http.StatusBadRequest) - return - } - if req.DestinationPath == "" { - http.Error(w, "destination_path is required", http.StatusBadRequest) - return - } - if s.vm.State() != vz.VirtualMachineStatePaused { - http.Error(w, "vm must be paused before snapshot", http.StatusBadRequest) - return - } - if err := validateSaveRestoreSupport(s.vmConfig); err != nil { - http.Error(w, fmt.Sprintf("save/restore not supported: %v", err), http.StatusBadRequest) - return - } - - if err := os.MkdirAll(req.DestinationPath, 0755); err != nil { - http.Error(w, fmt.Sprintf("create snapshot dir failed: %v", err), http.StatusInternalServerError) - return - } - snapshotComplete := false - defer func() { - if !snapshotComplete { - _ = os.RemoveAll(req.DestinationPath) - } - }() - - machineStatePath := filepath.Join(req.DestinationPath, shimconfig.SnapshotMachineStateFile) - if err := os.RemoveAll(machineStatePath); err != nil { - http.Error(w, fmt.Sprintf("prepare machine state path failed: %v", err), http.StatusInternalServerError) - return - } - if err := saveMachineState(s.vm, machineStatePath); err != nil { - http.Error(w, fmt.Sprintf("save machine state failed: %v", err), http.StatusInternalServerError) - return - } - - manifestPath := filepath.Join(req.DestinationPath, shimconfig.SnapshotManifestFile) - tmpManifestPath := manifestPath + ".tmp" - manifest := shimconfig.SnapshotManifest{ - Hypervisor: "vz", - MachineStateFile: shimconfig.SnapshotMachineStateFile, - ShimConfig: s.shimConfig, - } - // This field is runtime-only; restore path is populated by the caller on restore. - manifest.ShimConfig.RestoreMachineStatePath = "" - manifestBytes, err := json.Marshal(manifest) - if err != nil { - http.Error(w, fmt.Sprintf("marshal manifest failed: %v", err), http.StatusInternalServerError) - return - } - if err := os.WriteFile(tmpManifestPath, manifestBytes, 0644); err != nil { - http.Error(w, fmt.Sprintf("write manifest failed: %v", err), http.StatusInternalServerError) - return - } - if err := os.Rename(tmpManifestPath, manifestPath); err != nil { - http.Error(w, fmt.Sprintf("finalize manifest failed: %v", err), http.StatusInternalServerError) - return - } - - snapshotComplete = true - slog.Info("VM snapshot saved", "destination", req.DestinationPath, "machine_state", machineStatePath) - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleGetBalloon(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - - device, err := s.getTraditionalBalloonDevice() - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(balloonRequest{ - TargetGuestMemoryBytes: int64(device.GetTargetVirtualMachineMemorySize()), - }) -} - -func (s *ShimServer) handleSetBalloon(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - device, err := s.getTraditionalBalloonDevice() - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - var req balloonRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, fmt.Sprintf("invalid balloon request: %v", err), http.StatusBadRequest) - return - } - if req.TargetGuestMemoryBytes < 0 { - http.Error(w, "target_guest_memory_bytes must be non-negative", http.StatusBadRequest) - return - } - - device.SetTargetVirtualMachineMemorySize(uint64(req.TargetGuestMemoryBytes)) - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handlePowerButton(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - // RequestStop sends an ACPI power button event - success, err := s.vm.RequestStop() - if err != nil || !success { - slog.Error("failed to send power button", "error", err, "success", success) - http.Error(w, fmt.Sprintf("power button failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("power button sent") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) getTraditionalBalloonDevice() (*vz.VirtioTraditionalMemoryBalloonDevice, error) { - for _, device := range s.vm.MemoryBalloonDevices() { - if traditional := vz.AsVirtioTraditionalMemoryBalloonDevice(device); traditional != nil { - return traditional, nil - } - } - return nil, fmt.Errorf("no memory balloon device configured") -} - -func (s *ShimServer) handlePing(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("OK")) -} - -func (s *ShimServer) handleVMMShutdown(w http.ResponseWriter, r *http.Request) { - slog.Info("VMM shutdown requested") - w.WriteHeader(http.StatusNoContent) - - // Stop the VM and exit - go func() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.vm.CanStop() { - s.forcedStop.Store(true) - s.vm.Stop() - } - // Process will exit when VM stops (monitored in main) - }() -} - -func vzStateToString(state vz.VirtualMachineState) string { - switch state { - case vz.VirtualMachineStateStopped: - return "Shutdown" - case vz.VirtualMachineStateRunning: - return "Running" - case vz.VirtualMachineStatePaused: - return "Paused" - case vz.VirtualMachineStateError: - return "Error" - case vz.VirtualMachineStateStarting: - return "Starting" - case vz.VirtualMachineStatePausing: - return "Pausing" - case vz.VirtualMachineStateResuming: - return "Resuming" - case vz.VirtualMachineStateStopping: - return "Stopping" - case vz.VirtualMachineStateSaving: - return "Saving" - case vz.VirtualMachineStateRestoring: - return "Restoring" - default: - return "Unknown" - } -} - -// ServeVsock handles vsock proxy connections using the Cloud Hypervisor protocol. -// Protocol: Client sends "CONNECT {port}\n", server responds "OK {port}\n", then proxies. -func (s *ShimServer) ServeVsock(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil { - slog.Debug("vsock listener closed", "error", err) - return - } - go s.handleVsockConnection(conn) - } -} - -func (s *ShimServer) handleVsockConnection(conn net.Conn) { - defer conn.Close() - - // Read the CONNECT command - reader := bufio.NewReader(conn) - cmd, err := reader.ReadString('\n') - if err != nil { - slog.Error("failed to read vsock handshake", "error", err) - return - } - - // Parse "CONNECT {port}\n" - var port uint32 - if _, err := fmt.Sscanf(cmd, "CONNECT %d\n", &port); err != nil { - slog.Error("invalid vsock handshake", "cmd", cmd, "error", err) - conn.Write([]byte(fmt.Sprintf("ERR invalid command: %s", cmd))) - return - } - - slog.Debug("vsock connect request", "port", port) - - // Get vsock device and connect to guest - s.mu.RLock() - socketDevices := s.vm.SocketDevices() - s.mu.RUnlock() - - if len(socketDevices) == 0 { - slog.Error("no vsock device configured") - conn.Write([]byte("ERR no vsock device\n")) - return - } - - guestConn, err := socketDevices[0].Connect(port) - if err != nil { - slog.Error("failed to connect to guest vsock", "port", port, "error", err) - conn.Write([]byte(fmt.Sprintf("ERR connect failed: %v\n", err))) - return - } - defer guestConn.Close() - - // Send OK response (matching CH protocol) - if _, err := conn.Write([]byte(fmt.Sprintf("OK %d\n", port))); err != nil { - slog.Error("failed to send OK response", "error", err) - return - } - - slog.Debug("vsock connection established", "port", port) - - // Proxy data bidirectionally - done := make(chan struct{}, 2) - - go func() { - io.Copy(guestConn, reader) - done <- struct{}{} - }() - - go func() { - io.Copy(conn, guestConn) - done <- struct{}{} - }() - - // Wait for one direction to close - <-done -} diff --git a/cmd/vz-shim/vm.go b/cmd/vz-shim/vm.go deleted file mode 100644 index 08b373e5f..000000000 --- a/cmd/vz-shim/vm.go +++ /dev/null @@ -1,377 +0,0 @@ -//go:build darwin - -package main - -import ( - "encoding/base64" - "fmt" - "log/slog" - "net" - "net/netip" - "os" - "runtime" - "strings" - - "github.com/Code-Hex/vz/v4" - "github.com/Code-Hex/vz/v4/vmnet" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// createVM creates and configures a vz.VirtualMachine from ShimConfig. -func createVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { - if isMacGuest(config) { - return createMacVM(config) - } - - // Prepare kernel command line (vz uses hvc0 for serial console) - kernelArgs := config.KernelArgs - if kernelArgs == "" { - kernelArgs = "console=hvc0 root=/dev/vda" - } else { - kernelArgs = strings.ReplaceAll(kernelArgs, "console=ttyS0", "console=hvc0") - } - - bootLoader, err := vz.NewLinuxBootLoader( - config.KernelPath, - vz.WithCommandLine(kernelArgs), - vz.WithInitrd(config.InitrdPath), - ) - if err != nil { - return nil, nil, fmt.Errorf("create boot loader: %w", err) - } - - vcpus := computeCPUCount(config.VCPUs) - memoryBytes := computeMemorySize(uint64(config.MemoryBytes)) - - slog.Debug("VM config", "vcpus", vcpus, "memory_bytes", memoryBytes, "kernel", config.KernelPath, "initrd", config.InitrdPath) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) - if err != nil { - return nil, nil, fmt.Errorf("create vm configuration: %w", err) - } - - if err := configureSerialConsole(vmConfig, config.SerialLogPath); err != nil { - return nil, nil, fmt.Errorf("configure serial: %w", err) - } - - if err := configureNetwork(vmConfig, config.Networks); err != nil { - return nil, nil, fmt.Errorf("configure network: %w", err) - } - - entropyConfig, err := vz.NewVirtioEntropyDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create entropy device: %w", err) - } - vmConfig.SetEntropyDevicesVirtualMachineConfiguration([]*vz.VirtioEntropyDeviceConfiguration{entropyConfig}) - - if err := configureStorage(vmConfig, config.Disks); err != nil { - return nil, nil, fmt.Errorf("configure storage: %w", err) - } - - if err := configureDirectorySharing(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure directory sharing: %w", err) - } - - if err := configurePlatform(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure platform: %w", err) - } - - vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create vsock device: %w", err) - } - vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) - - if config.EnableMemoryBalloon { - balloonConfig, err := vz.NewVirtioTraditionalMemoryBalloonDeviceConfiguration() - if err != nil { - if config.RequireMemoryBalloon { - return nil, nil, fmt.Errorf("create memory balloon device: %w", err) - } - slog.Warn("memory balloon unavailable, continuing without balloon", "error", err) - } else { - vmConfig.SetMemoryBalloonDevicesVirtualMachineConfiguration([]vz.MemoryBalloonDeviceConfiguration{ - balloonConfig, - }) - } - } - - if validated, err := vmConfig.Validate(); !validated || err != nil { - return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) - } - - vm, err := vz.NewVirtualMachine(vmConfig) - if err != nil { - return nil, nil, fmt.Errorf("create virtual machine: %w", err) - } - - return vm, vmConfig, nil -} - -func configurePlatform(vmConfig *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - var machineID *vz.GenericMachineIdentifier - var err error - - if config.MachineIdentifierData != "" { - b, decodeErr := base64.StdEncoding.DecodeString(config.MachineIdentifierData) - if decodeErr != nil { - return fmt.Errorf("decode machine identifier data: %w", decodeErr) - } - machineID, err = vz.NewGenericMachineIdentifierWithData(b) - if err != nil { - return fmt.Errorf("recreate machine identifier: %w", err) - } - } else { - machineID, err = vz.NewGenericMachineIdentifier() - if err != nil { - return fmt.Errorf("create machine identifier: %w", err) - } - config.MachineIdentifierData = base64.StdEncoding.EncodeToString(machineID.DataRepresentation()) - } - - platformConfig, err := vz.NewGenericPlatformConfiguration( - vz.WithGenericMachineIdentifier(machineID), - ) - if err != nil { - return fmt.Errorf("create generic platform config: %w", err) - } - vmConfig.SetPlatformVirtualMachineConfiguration(platformConfig) - return nil -} - -func configureSerialConsole(vmConfig *vz.VirtualMachineConfiguration, logPath string) error { - var serialAttachment *vz.FileHandleSerialPortAttachment - - nullRead, err := os.OpenFile("/dev/null", os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("open /dev/null for reading: %w", err) - } - - if logPath != "" { - file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - nullRead.Close() - return fmt.Errorf("open serial log file: %w", err) - } - serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, file) - if err != nil { - nullRead.Close() - file.Close() - return fmt.Errorf("create serial attachment: %w", err) - } - } else { - nullWrite, err := os.OpenFile("/dev/null", os.O_WRONLY, 0) - if err != nil { - nullRead.Close() - return fmt.Errorf("open /dev/null for writing: %w", err) - } - serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, nullWrite) - if err != nil { - nullRead.Close() - nullWrite.Close() - return fmt.Errorf("create serial attachment: %w", err) - } - } - - consoleConfig, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(serialAttachment) - if err != nil { - return fmt.Errorf("create console config: %w", err) - } - vmConfig.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{ - consoleConfig, - }) - - return nil -} - -func configureNetwork(vmConfig *vz.VirtualMachineConfiguration, networks []shimconfig.NetworkConfig) error { - if len(networks) == 0 { - // No networks configured (NetworkEnabled=false) — do not attach any NIC. - return nil - } - - var devices []*vz.VirtioNetworkDeviceConfiguration - for _, netConfig := range networks { - var dev *vz.VirtioNetworkDeviceConfiguration - var err error - if netConfig.IPv4Address != "" { - dev, err = createVmnetNetworkDevice(netConfig) - } else { - dev, err = createNATNetworkDevice(netConfig.MAC) - } - if err != nil { - return err - } - devices = append(devices, dev) - } - vmConfig.SetNetworkDevicesVirtualMachineConfiguration(devices) - return nil -} - -func createNATNetworkDevice(macAddr string) (*vz.VirtioNetworkDeviceConfiguration, error) { - natAttachment, err := vz.NewNATNetworkDeviceAttachment() - if err != nil { - return nil, fmt.Errorf("create NAT attachment: %w", err) - } - - networkConfig, err := vz.NewVirtioNetworkDeviceConfiguration(natAttachment) - if err != nil { - return nil, fmt.Errorf("create network config: %w", err) - } - - mac, err := assignMACAddress(macAddr) - if err != nil { - return nil, err - } - networkConfig.SetMACAddress(mac) - - return networkConfig, nil -} - -// createVmnetNetworkDevice attaches the guest to its own shared-mode vmnet network -// with the given subnet and a DHCP reservation for the guest's MAC, so the guest's -// address is the one hypeman assigned. Saved state records this configuration, so a -// guest saved on the shared NAT (no address in its config) is restored on NAT. -func createVmnetNetworkDevice(netConfig shimconfig.NetworkConfig) (*vz.VirtioNetworkDeviceConfiguration, error) { - hw, err := net.ParseMAC(netConfig.MAC) - if err != nil { - return nil, fmt.Errorf("vmnet network needs the guest MAC: %w", err) - } - subnet, err := netip.ParsePrefix(netConfig.IPv4Subnet) - if err != nil { - return nil, fmt.Errorf("parse guest subnet: %w", err) - } - address, err := netip.ParseAddr(netConfig.IPv4Address) - if err != nil { - return nil, fmt.Errorf("parse guest address: %w", err) - } - cfg, err := vmnet.NewNetworkConfiguration(vmnet.ModeShared, vmnet.WithIPv4Subnet(subnet), vmnet.WithDHCPReservation(hw, address)) - if err != nil { - return nil, fmt.Errorf("configure vmnet network %s: %w", subnet, err) - } - network, err := vmnet.NewNetwork(cfg) - if err != nil { - return nil, fmt.Errorf("create vmnet network %s: %w", subnet, err) - } - attachment, err := vz.NewVMNetNetworkDeviceAttachment(network) - if err != nil { - return nil, fmt.Errorf("create vmnet attachment: %w", err) - } - dev, err := vz.NewVirtioNetworkDeviceConfiguration(attachment) - if err != nil { - return nil, fmt.Errorf("create network config: %w", err) - } - mac, err := assignMACAddress(netConfig.MAC) - if err != nil { - return nil, err - } - dev.SetMACAddress(mac) - slog.Info("attached guest vmnet network", "subnet", subnet.String(), "address", address.String(), "mac", netConfig.MAC) - return dev, nil -} - -func assignMACAddress(macAddr string) (*vz.MACAddress, error) { - if macAddr == "" { - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - slog.Info("generated random MAC address", "mac", mac.String()) - return mac, nil - } - - hwAddr, err := net.ParseMAC(macAddr) - if err != nil { - slog.Warn("failed to parse MAC address, generating random", "mac", macAddr, "error", err) - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - return mac, nil - } - - mac, err := vz.NewMACAddress(hwAddr) - if err != nil { - slog.Warn("failed to create MAC from parsed address, generating random", "mac", macAddr, "error", err) - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - return mac, nil - } - - slog.Info("using specified MAC address", "mac", macAddr) - return mac, nil -} - -func configureStorage(vmConfig *vz.VirtualMachineConfiguration, disks []shimconfig.DiskConfig) error { - var storageDevices []vz.StorageDeviceConfiguration - - for _, disk := range disks { - if _, err := os.Stat(disk.Path); os.IsNotExist(err) { - return fmt.Errorf("disk image not found: %s", disk.Path) - } - - if strings.HasSuffix(disk.Path, ".qcow2") { - return fmt.Errorf("qcow2 not supported by vz, use raw format: %s", disk.Path) - } - - attachment, err := vz.NewDiskImageStorageDeviceAttachment(disk.Path, disk.Readonly) - if err != nil { - return fmt.Errorf("create disk attachment for %s: %w", disk.Path, err) - } - - blockConfig, err := vz.NewVirtioBlockDeviceConfiguration(attachment) - if err != nil { - return fmt.Errorf("create block device config: %w", err) - } - - storageDevices = append(storageDevices, blockConfig) - } - - if len(storageDevices) > 0 { - vmConfig.SetStorageDevicesVirtualMachineConfiguration(storageDevices) - } - - return nil -} - -func computeCPUCount(requested int) uint { - virtualCPUCount := uint(requested) - if virtualCPUCount == 0 { - virtualCPUCount = uint(runtime.NumCPU() - 1) - if virtualCPUCount < 1 { - virtualCPUCount = 1 - } - } - - maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() - minAllowed := vz.VirtualMachineConfigurationMinimumAllowedCPUCount() - - if virtualCPUCount > maxAllowed { - virtualCPUCount = maxAllowed - } - if virtualCPUCount < minAllowed { - virtualCPUCount = minAllowed - } - - return virtualCPUCount -} - -func computeMemorySize(requested uint64) uint64 { - if requested == 0 { - requested = 2 * 1024 * 1024 * 1024 // 2GB safety default (caller normally provides this) - } - - maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedMemorySize() - minAllowed := vz.VirtualMachineConfigurationMinimumAllowedMemorySize() - - if requested > maxAllowed { - requested = maxAllowed - } - if requested < minAllowed { - requested = minAllowed - } - - return requested -} diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index e7745fb1a..362b3d18d 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,6 +27,9 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" +# Native macOS guests are opt-in; this default retains Linux guests on VZ. +macos_only: false + # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) # ============================================================================= diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 585453ee6..00606cfc5 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -6,11 +6,12 @@ Linux guests retain the existing defaults and boot path. ## Import -Use a stopped macOS bundle (`config.json`, `disk.img`, `aux.img`) on -Darwin/arm64. Configure its user, SSH, FileVault and desktop before import. -The importer verifies that storage is not open, clones disk and auxiliary -storage, hashes their contents and identity/configuration into a digest, and -publishes a local ready image. Import is an administrator CLI operation, not +Use a stopped macOS bundle (`config.json`, `aux.img`, and `disk.asif` or a raw +`disk.img`) on Darwin/arm64. Configure its user, SSH, FileVault and desktop before +import. The importer verifies that storage is not open, clones an ASIF disk or +converts a raw one to ASIF (all-zero 1 MiB chunks stay unallocated; a 100 GB disk +took about 10 minutes on the QA host), clones auxiliary storage, hashes their +contents and identity/configuration into a digest, and publishes a local ready image. Import is an administrator CLI operation, not an HTTP endpoint accepting arbitrary host paths. ```sh @@ -20,7 +21,8 @@ go run -tags containers_image_openpgp ./cmd/import-macos \ --name localhost/macos:spike ``` -The local importer needs APFS clonefile support. The data directory path must be +The local importer needs APFS clonefile support, and ASIF disks need macOS 26 or +later; the VZ shim needs macOS 27. The data directory path must be short enough that each guest's sockets fit the 103-byte macOS limit; a boot that would exceed it is refused. Images are `darwin/arm64` machine images, not OCI macOS containers. Registry pulls also support experimental complete-machine @@ -35,30 +37,110 @@ labels describe a complete installed machine: | Label | Value | |---|---| -| `io.hypeman.machine-image.version` | `1` | +| `io.hypeman.machine-image.version` | `2` | | `io.hypeman.machine-image.kind` | `macos-image` | -| `io.hypeman.machine-image.disk-format` | `raw` | -| `io.hypeman.machine-image.disk-path` | Relative installed boot disk path | +| `io.hypeman.machine-image.disk-format` | `asif` | +| `io.hypeman.machine-image.disk-path` | Relative ASIF disk path, in every layer | | `io.hypeman.machine-image.aux-path` | Relative auxiliary storage path | | `io.hypeman.machine-image.platform-path` | Relative macOS platform configuration JSON path | +| `io.hypeman.machine-image.disk-digests` | Optional: sha256 of each layer's disk file, base first, comma-separated | -All three files must be distinct, nonempty regular files inside the unpacked +Each layer carries one ASIF disk, and each layer's disk is an overlay on the +previous layer's, so the layers form the boot disk's chain, base first. The +auxiliary storage and platform come from the top layer; lower layers' copies are +ignored. A base image is one layer; an image built on it adds one layer holding +only the build's writes. A pull clones any lower layer whose diff ID a ready image +on the host already has instead of unpacking it, so images built on one base share +its disk. All files must be distinct, nonempty regular files inside the unpacked image. Platform configuration uses the local bundle's `MacOSImage` schema and is limited to 64 KiB. Absolute paths, traversal and symlink escapes are rejected. The installed disk is materialized directly rather than converted to a Linux root filesystem. Machine layers are unpacked sparsely (zero chunks become holes), accept only regular files (not GNU sparse tar entries) and directories, and are -checked against their diff IDs, so a mostly empty boot disk occupies only its -data. A bundle's layers may decompress to at most 1 TiB in total. The boot disk -is stored as `rootfs.raw`, as for a local import; machine images pulled before -that naming must be deleted and pulled again. The matching auxiliary storage and platform -identity are retained. Do not package live mutable guest storage. +checked against their diff IDs. A bundle's layers may decompress to at most 1 +TiB in total. The chain is stored as `disk.{n}.asif`, as for a local import, and +must open as an ASIF stack. Machine images from before ASIF (version `1`, `raw`) +are refused and must be rebuilt or reimported. The matching auxiliary storage and +platform identity are retained. Do not package live mutable guest storage. This schema is experimental pending reconciliation with other machine-image -platforms. Complete bundles only: no base/delta chain, identity rebinding, or -concurrent-template guarantee. Ordinary OCI transport does not imply sparse +platforms. There is no identity rebinding on pull or concurrent-template guarantee. Ordinary OCI transport does not imply sparse distribution efficiency, boot portability, or container execution semantics. +## Building images with Packer and Tart + +hypeman does not build macOS images itself. Build them with Packer's Tart builder +on a macOS 27 host, as stacked Tart VMs, then publish the result with +`import-tart`. A stacked VM shares a read-only base disk and writes only to its own +`overlay.asif`, so a build on a base already in the registry uploads only that +overlay. + +1. Publish the base once from a standalone Tart VM whose `disk.img` is ASIF + (`diskFormat: asif`). Every layer is uploaded. +2. Get the base as a Tart image (`tart push` to any OCI registry) and create the + build VM with `tart clone --stacked`. The Packer Tart plugin cannot clone + stacked itself, so point its `vm_base_name` at that local clone. +3. Build with `images/macos/browser.pkr.hcl`. It installs Chrome (from Google's + stable DMG, checked by its code signature and Gatekeeper), runs any + `extra_scripts`, installs hypeman's agent launcher (hypeman supplies the agent + on its own disk; see [macos-guest-agent.md](macos-guest-agent.md#agent-disk)), + checks the image with `images/macos/smoke-test.sh` (launcher present, no baked + agent, headless Chrome answers CDP), regenerates the SSH host keys and shuts the + guest down from inside. It keeps the recovery partition; Packer's default + deletes it and rewrites the whole disk. The base must have an SSH user with sudo + for Packer. + + ```sh + cd images/macos && packer init browser.pkr.hcl + packer build -var vm_base_name=base-clone -var vm_name=my-build \ + -var ssh_username=admin -var ssh_password=... browser.pkr.hcl + ``` + +4. Publish the stopped VM over the base: + +```sh +HYPEMAN_REGISTRY_TOKEN=... go run ./cmd/import-tart \ + --vm ~/.tart/vms/my-build \ + --registry https://hypeman.example \ + --name my-build \ + --base hypeman.example/builds/my-base@sha256:... \ + --guest-agent +``` + +The `macOS image` workflow (`.github/workflows/macos-image.yml`, run by hand) does +steps 2 to 4 on a self-hosted macOS 27 runner labelled `tart`. It needs the +`HYPEMAN_REGISTRY_URL` and `MACOS_IMAGE_SSH_USERNAME` variables and the +`HYPEMAN_REGISTRY_TOKEN`, `MACOS_IMAGE_SSH_PASSWORD`, `TART_REGISTRY_USERNAME` and +`TART_REGISTRY_PASSWORD` secrets. + +`HYPEMAN_REGISTRY_TOKEN` can push only `builds/` and pull only the base. +Someone with the registry's JWT secret mints it once, and mints a new one when the +name, the base or its expiry changes: + +```sh +HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart --mint-token \ + --registry https://hypeman.example --name my-build \ + --base hypeman.example/builds/my-base@sha256:... --token-ttl 720h +``` + +`import-tart` also accepts `HYPEMAN_JWT_SECRET` in place of a token, minting one for +each publication. + +The image is published as `builds/`. `import-tart` reads the VM's layers from +Tart's content store (under `TART_HOME`, or `~/.tart`) in the order its +`manifest.json` lists them, refuses a suspended or running VM, and checks that the +overlay stacks on those layers. Each published image records its layers' disk +digests. With `--base`, the VM's layer digests must equal the ones that image +records, and only the config and overlay are uploaded; a base without them is +refused. Each image gets a fresh machine identifier and MAC, since +Tart keeps its source's; the SSH host keys are the template's job. The registry +imports a pushed image on its host, as `builds/@sha256:...`. Other hosts pull +it by digest with registry credentials. + +Building on a build needs that build as a Tart image too: `tart clone --stacked` +only stacks on images pulled from a registry, and a local clone copies the +overlay instead of stacking on it. + ## API Use the usual JWT-authenticated API with a VZ-capable, signed build: @@ -73,8 +155,10 @@ Use the usual JWT-authenticated API with a VZ-capable, signed build: ``` CPU and memory default to the imported template (minimum 2 CPUs and 4GB). -Each instance gets its own writable complete boot disk (`boot.raw`) and auxiliary -storage; instances created before that naming must be deleted and recreated. No +Each instance gets clones of the image's ASIF layers (`disk.{n}.asif`), its own +auxiliary storage, and an empty writable ASIF overlay (`boot.asif`) above the +layers; the guest's writes only grow the overlay. Instances created before ASIF +must be deleted and recreated. No Linux kernel, initrd, configuration disk, overlay filesystem or balloon is attached. A networked guest gets its own vmnet network (macOS 26 and later): hypeman assigns it a /29 from `network.macos_subnet_cidr` (default diff --git a/docs/macos-guest-agent.md b/docs/macos-guest-agent.md index fda435897..63c40f53e 100644 --- a/docs/macos-guest-agent.md +++ b/docs/macos-guest-agent.md @@ -9,7 +9,9 @@ is unchanged. ## Build -Build on a macOS development machine with the macOS SDK and cgo enabled: +`make build-embedded` on macOS builds the agent that hypeman embeds for the agent +disk. To build it by hand on a macOS development machine with the macOS SDK and +cgo enabled: ```sh CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build \ @@ -23,6 +25,27 @@ listener and cgo-disabled build path. The Darwin listener accepts only host CID 2, marks descriptors close-on-exec, and provides net.Conn deadlines for gRPC. Do not start the executable on the development host to test guest services. +## Agent disk + +hypeman carries the macOS agent and attaches it to each macOS guest whose image +declares `guest_agent: true`, on a read-only disk second after the boot disk. The +disk is an APFS volume named `HypemanAgent` holding `hypeman-guest-agent` and a +`VERSION` file, built on the host with `hdiutil` the first time a guest needs it, +under `system/macos-agent/`. It is named by the agent's hash, so a new hypeman +builds a new disk. + +Each guest keeps its own copy (`agent.img`). Creating a guest and every cold start +copy the host's current disk, so an updated agent takes effect at the next cold +start. Standby, restore and snapshots keep the copy the guest was saved with, so +saved state restores against the same disk. + +The image runs the agent from the disk with a small launcher, installed once in the +base by `lib/system/guest_agent/macos/install-launcher.sh` (as root, for example +from a Packer provisioner). It removes an agent installed in the image itself and +adds a LaunchDaemon that waits for the volume, mounts it read-only if macOS has +not, and runs the agent from it. An image with its own agent and no launcher still +works; the disk is attached and unused. + ## Provisioning boundary Install the binary **inside a stopped-template provisioning guest**, not on the diff --git a/docs/macos-support.md b/docs/macos-support.md index 2ee42b466..704b3d40b 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Unsupported | Build with Packer and Tart, then publish with `import-tart`. See [macos-experimental.md](macos-experimental.md#building-images-with-packer-and-tart). Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/images/macos/browser.pkr.hcl b/images/macos/browser.pkr.hcl new file mode 100644 index 000000000..aa3545ea6 --- /dev/null +++ b/images/macos/browser.pkr.hcl @@ -0,0 +1,97 @@ +# Builds a macOS image for hypeman on a stacked Tart VM. Clone the base with +# `tart clone --stacked` first and pass that VM as vm_base_name: the build writes +# only to the clone's overlay, which import-tart publishes over the base. +# See docs/macos-experimental.md. +packer { + required_plugins { + tart = { + version = "= 1.21.0" + source = "github.com/cirruslabs/tart" + } + } +} + +variable "vm_base_name" { + type = string + description = "Stopped Tart VM to build on, made with tart clone --stacked." +} + +variable "vm_name" { + type = string + description = "Tart VM the build writes, which import-tart publishes." +} + +variable "ssh_username" { + type = string +} + +variable "ssh_password" { + type = string + sensitive = true +} + +variable "extra_scripts" { + type = list(string) + default = [] + description = "Scripts run as root, in order, after Chrome is installed." +} + +locals { + # The sudo password is read from a file uploaded first, so it is never quoted + # into a command line. The build user's home is not readable by other users. + sudo_password = "/Users/${var.ssh_username}/.packer-sudo-password" + sudo = "sudo -S -p '' sh -eux '{{ .Path }}' < ${local.sudo_password}" +} + +source "tart-cli" "macos" { + vm_base_name = var.vm_base_name + vm_name = var.vm_name + headless = true + # The default, "delete", removes the recovery partition and grows APFS, which + # rewrites the whole disk instead of the build's changes. + recovery_partition = "keep" + ssh_username = var.ssh_username + ssh_password = var.ssh_password + ssh_timeout = "300s" +} + +build { + sources = ["source.tart-cli.macos"] + + provisioner "file" { + content = var.ssh_password + destination = local.sudo_password + } + + provisioner "shell" { + execute_command = local.sudo + scripts = concat(["${path.root}/install-chrome.sh"], var.extra_scripts) + } + + # hypeman supplies its guest agent on a read-only disk; the image only carries + # the launcher that runs it. + provisioner "shell" { + execute_command = local.sudo + script = "${path.root}/../../lib/system/guest_agent/macos/install-launcher.sh" + } + + provisioner "shell" { + script = "${path.root}/smoke-test.sh" + } + + # Every image gets its own SSH host keys; Tart keeps its source's. The guest + # shuts itself down: tart stop is a hard power-off. + provisioner "shell" { + execute_command = local.sudo + inline = [ + "rm -f ${local.sudo_password}", + "rm -f /etc/ssh/ssh_host_*", + "ssh-keygen -A", + "shutdown -h now", + ] + expect_disconnect = true + # Without this, Packer retries removing its script over the dead connection + # for five minutes. + skip_clean = true + } +} diff --git a/images/macos/install-chrome.sh b/images/macos/install-chrome.sh new file mode 100755 index 000000000..f01f3dcb6 --- /dev/null +++ b/images/macos/install-chrome.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Install Google Chrome from Google's stable DMG. The download URL is not +# versioned, so the copy to be installed is checked by its code signature: it must +# be signed by Google's Developer ID team and pass Gatekeeper. (codesign --strict +# rejects Chrome's own Alerts helper, so the deep check is not strict.) +set -eu +team=EQHXZ8M8AV +work=$(mktemp -d) +mount="$work/mount" +trap 'hdiutil detach -quiet "$mount" 2>/dev/null || true; rm -rf "$work"' EXIT +curl -fsSL -o "$work/chrome.dmg" https://dl.google.com/chrome/mac/universal/stable/GGRO/googlechrome.dmg +hdiutil attach -quiet -nobrowse -readonly -mountpoint "$mount" "$work/chrome.dmg" +app="$work/Google Chrome.app" +ditto "$mount/Google Chrome.app" "$app" +codesign --verify --deep "$app" +codesign -dv "$app" 2>&1 | grep -qx "TeamIdentifier=$team" +spctl --assess --type execute "$app" +rm -rf "/Applications/Google Chrome.app" +mv "$app" "/Applications/Google Chrome.app" +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version diff --git a/images/macos/smoke-test.sh b/images/macos/smoke-test.sh new file mode 100755 index 000000000..a54909440 --- /dev/null +++ b/images/macos/smoke-test.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Checks the image before it is published: hypeman's agent launcher is installed, +# no agent is baked into the image, and Chrome starts headless and answers CDP. +# Runs as the build user, leaving no browser profile behind. +set -eu +test -x /usr/local/libexec/hypeman-agent-launcher +test -f /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist +test ! -e /usr/local/sbin/hypeman-guest-agent + +chrome="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" +test -x "$chrome" +profile=$(mktemp -d) +"$chrome" --headless=new --user-data-dir="$profile" --remote-debugging-port=9222 --no-first-run about:blank >/dev/null 2>&1 & +pid=$! +trap 'kill $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$profile"' EXIT +for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9222/json/version; then + echo + exit 0 + fi + sleep 1 +done +echo "Chrome did not answer on its debugging port" >&2 +exit 1 diff --git a/lib/builds/registry_token.go b/lib/builds/registry_token.go index 3a2f8abfe..0aaa34d03 100644 --- a/lib/builds/registry_token.go +++ b/lib/builds/registry_token.go @@ -1,220 +1,13 @@ -// Package builds implements registry token generation for secure builder VM authentication. package builds -import ( - "fmt" - "time" +import "github.com/kernel/hypeman/lib/registrytoken" - "github.com/golang-jwt/jwt/v5" +// Registry tokens are minted by lib/registrytoken, which image publication uses +// without the rest of this package. +type ( + RepoPermission = registrytoken.RepoPermission + RegistryTokenClaims = registrytoken.Claims + RegistryTokenGenerator = registrytoken.Generator ) -// RepoPermission defines access permissions for a specific repository. -// This enables fine-grained control where different repos can have different scopes. -type RepoPermission struct { - // Repo is the repository path (e.g., "builds/abc123", "cache/tenant-x", "cache/global/node") - Repo string `json:"repo"` - - // Scope is the access scope for this repo: "pull" for read-only, "push" for read+write - Scope string `json:"scope"` -} - -// RegistryTokenClaims contains the claims for a scoped registry access token. -// These tokens are issued to builder VMs to grant limited access to specific repositories. -type RegistryTokenClaims struct { - jwt.RegisteredClaims - - // BuildID is the build job identifier for audit purposes - BuildID string `json:"build_id"` - - // RepoAccess defines per-repository access permissions (new two-tier format) - // If present, this takes precedence over the legacy Repositories/Scope fields - RepoAccess []RepoPermission `json:"repo_access,omitempty"` - - // Repositories is the list of allowed repository paths (legacy format, kept for backward compat) - // Deprecated: Use RepoAccess for new tokens - Repositories []string `json:"repos,omitempty"` - - // Scope is the access scope (legacy format, kept for backward compat) - // Deprecated: Use RepoAccess for new tokens - Scope string `json:"scope,omitempty"` -} - -// RegistryTokenGenerator creates scoped registry access tokens -type RegistryTokenGenerator struct { - secret []byte -} - -// NewRegistryTokenGenerator creates a new token generator with the given secret -func NewRegistryTokenGenerator(secret string) *RegistryTokenGenerator { - return &RegistryTokenGenerator{ - secret: []byte(secret), - } -} - -// GeneratePushToken creates a short-lived token granting push access to specific repositories. -// The token expires after the specified duration (typically matching the build timeout). -// Deprecated: Use GenerateToken for new code that needs per-repo scopes. -func (g *RegistryTokenGenerator) GeneratePushToken(buildID string, repos []string, ttl time.Duration) (string, error) { - if buildID == "" { - return "", fmt.Errorf("build ID is required") - } - if len(repos) == 0 { - return "", fmt.Errorf("at least one repository is required") - } - - now := time.Now() - claims := RegistryTokenClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Subject: "builder-" + buildID, - IssuedAt: jwt.NewNumericDate(now), - ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), - Issuer: "hypeman", - }, - BuildID: buildID, - Repositories: repos, - Scope: "push", - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString(g.secret) -} - -// GenerateToken creates a short-lived token with per-repository access permissions. -// This supports the two-tier cache model where different repos can have different scopes. -// For example: pull on cache/global/*, push on cache/{tenant} -func (g *RegistryTokenGenerator) GenerateToken(buildID string, repoAccess []RepoPermission, ttl time.Duration) (string, error) { - if buildID == "" { - return "", fmt.Errorf("build ID is required") - } - if len(repoAccess) == 0 { - return "", fmt.Errorf("at least one repository permission is required") - } - - now := time.Now() - claims := RegistryTokenClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Subject: "builder-" + buildID, - IssuedAt: jwt.NewNumericDate(now), - ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), - Issuer: "hypeman", - }, - BuildID: buildID, - RepoAccess: repoAccess, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString(g.secret) -} - -// ValidateToken parses and validates a registry token, returning the claims if valid. -func (g *RegistryTokenGenerator) ValidateToken(tokenString string) (*RegistryTokenClaims, error) { - token, err := jwt.ParseWithClaims(tokenString, &RegistryTokenClaims{}, func(token *jwt.Token) (interface{}, error) { - // Validate signing method - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) - } - return g.secret, nil - }) - - if err != nil { - return nil, fmt.Errorf("parse token: %w", err) - } - - claims, ok := token.Claims.(*RegistryTokenClaims) - if !ok || !token.Valid { - return nil, fmt.Errorf("invalid token") - } - - return claims, nil -} - -// IsRepositoryAllowed checks if the given repository path is allowed by the token claims. -func (c *RegistryTokenClaims) IsRepositoryAllowed(repo string) bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Repo == repo { - return true - } - } - return false - } - - // Fall back to legacy format - for _, allowed := range c.Repositories { - if allowed == repo { - return true - } - } - return false -} - -// GetRepoScope returns the scope for a specific repository. -// Returns empty string if the repository is not allowed. -func (c *RegistryTokenClaims) GetRepoScope(repo string) string { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Repo == repo { - return perm.Scope - } - } - return "" - } - - // Fall back to legacy format - all repos have the same scope - for _, allowed := range c.Repositories { - if allowed == repo { - return c.Scope - } - } - return "" -} - -// IsPushAllowedForRepo returns true if the token grants push (write) access to the given repo. -func (c *RegistryTokenClaims) IsPushAllowedForRepo(repo string) bool { - scope := c.GetRepoScope(repo) - return scope == "push" -} - -// IsPullAllowedForRepo returns true if the token grants pull (read) access to the given repo. -// Push scope also implicitly grants pull access. -func (c *RegistryTokenClaims) IsPullAllowedForRepo(repo string) bool { - scope := c.GetRepoScope(repo) - return scope == "push" || scope == "pull" -} - -// IsPushAllowed returns true if the token grants push (write) access to any repo. -// Deprecated: Use IsPushAllowedForRepo for per-repo scope checking. -func (c *RegistryTokenClaims) IsPushAllowed() bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Scope == "push" { - return true - } - } - return false - } - - // Fall back to legacy format - return c.Scope == "push" -} - -// IsPullAllowed returns true if the token grants pull (read) access. -// Push tokens also implicitly grant pull access. -// Deprecated: Use IsPullAllowedForRepo for per-repo scope checking. -func (c *RegistryTokenClaims) IsPullAllowed() bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Scope == "push" || perm.Scope == "pull" { - return true - } - } - return false - } - - // Fall back to legacy format - return c.Scope == "push" || c.Scope == "pull" -} +var NewRegistryTokenGenerator = registrytoken.NewGenerator diff --git a/lib/diskutilization/diskutilization.go b/lib/diskutilization/diskutilization.go index 996e81642..3da7e4667 100644 --- a/lib/diskutilization/diskutilization.go +++ b/lib/diskutilization/diskutilization.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "syscall" "github.com/kernel/hypeman/lib/paths" @@ -56,7 +57,7 @@ func Collect(p *paths.Paths) (Breakdown, error) { return false } name := entry.Name() - return name == "rootfs.erofs" || name == "rootfs.ext4" || name == "rootfs.raw" + return name == "rootfs.erofs" || name == "rootfs.ext4" || (strings.HasPrefix(name, "disk.") && strings.HasSuffix(name, ".asif")) }) if err != nil { return Breakdown{}, err @@ -88,7 +89,10 @@ func Collect(p *paths.Paths) (Breakdown, error) { instanceID := guest.Name() - breakdown.RootfsOverlays += allocatedBytesForPath(p.InstanceOverlay(instanceID)) + allocatedBytesForPath(p.InstanceMacOSBootDisk(instanceID)) + // A macOS guest's own writes and files. Its disk.N.asif layers are clones of + // its image's, which Images already counts. + breakdown.RootfsOverlays += allocatedBytesForPath(p.InstanceOverlay(instanceID)) + allocatedBytesForPath(p.InstanceMacOSBootDisk(instanceID)) + + allocatedBytesForPath(p.InstanceMacOSAgentDisk(instanceID)) + allocatedBytesForPath(filepath.Join(p.InstanceDir(instanceID), paths.MacOSAuxStorageFile)) volumeOverlays, err := sumMatchingFilesAllocatedBytes(p.InstanceVolumeOverlaysDir(instanceID), func(path string, entry fs.DirEntry) bool { return !entry.IsDir() && filepath.Ext(entry.Name()) == ".raw" diff --git a/lib/diskutilization/diskutilization_test.go b/lib/diskutilization/diskutilization_test.go index fab0a54bc..c84fabb9e 100644 --- a/lib/diskutilization/diskutilization_test.go +++ b/lib/diskutilization/diskutilization_test.go @@ -22,7 +22,7 @@ func TestCollect_UsesAllocatedBytesAndClassifiesSnapshots(t *testing.T) { require.NoError(t, createSparseTestFile(imagePath, 32*1024*1024, []sparseWrite{ {offset: 0, data: bytes.Repeat([]byte("i"), 4096)}, })) - machineImagePath := filepath.Join(p.ImagesDir(), "content", "def", "rootfs.raw") + machineImagePath := filepath.Join(p.ImagesDir(), "content", "def", "disk.0.asif") require.NoError(t, createSparseTestFile(machineImagePath, 32*1024*1024, []sparseWrite{ {offset: 0, data: bytes.Repeat([]byte("m"), 4096)}, })) diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index da8c41daa..4aadd5f89 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -104,6 +104,8 @@ type DiskFormat string const ( DiskFormatRaw DiskFormat = "raw" DiskFormatQCOW2 DiskFormat = "qcow2" + // DiskFormatASIF is an ASIF overlay on its Parents, only for macOS boot disks. + DiskFormatASIF DiskFormat = "asif" ) // EffectiveFormat preserves raw disks for existing callers. @@ -119,8 +121,9 @@ type DiskConfig struct { Path string Format DiskFormat Readonly bool - IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) - IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) + Parents []string // vz only: read-only images below Path, base first; Path is their ASIF overlay + IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) + IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) } // NetworkConfig represents a network interface attached to the VM diff --git a/lib/hypervisor/config_validation.go b/lib/hypervisor/config_validation.go index 26a23065f..7de9bbd68 100644 --- a/lib/hypervisor/config_validation.go +++ b/lib/hypervisor/config_validation.go @@ -42,6 +42,10 @@ func ValidateBootConfig(cfg VMConfig) error { for i, disk := range cfg.Disks { switch disk.EffectiveFormat() { case DiskFormatRaw, DiskFormatQCOW2: + case DiskFormatASIF: + if cfg.EffectiveBootMode() != BootModeMacOS { + return fmt.Errorf("disk %d: ASIF disks are only for macOS boot", i) + } default: return fmt.Errorf("disk %d has unsupported format %q", i, disk.Format) } diff --git a/lib/hypervisor/vz/fork.go b/lib/hypervisor/vz/fork.go index 87c4acede..0ee47810c 100644 --- a/lib/hypervisor/vz/fork.go +++ b/lib/hypervisor/vz/fork.go @@ -94,9 +94,17 @@ func rewriteShimConfigPaths(cfg shimconfig.ShimConfig, sourceDir, targetDir stri cfg.VsockSocket = replace(cfg.VsockSocket) cfg.LogPath = replace(cfg.LogPath) cfg.RestoreMachineStatePath = replace(cfg.RestoreMachineStatePath) + cfg.MacAuxStoragePath = replace(cfg.MacAuxStoragePath) for i := range cfg.Disks { cfg.Disks[i].Path = replace(cfg.Disks[i].Path) + parents := make([]string, len(cfg.Disks[i].Parents)) + for j, parent := range cfg.Disks[i].Parents { + parents[j] = replace(parent) + } + if len(parents) > 0 { + cfg.Disks[i].Parents = parents + } } return cfg diff --git a/lib/hypervisor/vz/fork_test.go b/lib/hypervisor/vz/fork_test.go index 240ddb1f9..fa683ae8e 100644 --- a/lib/hypervisor/vz/fork_test.go +++ b/lib/hypervisor/vz/fork_test.go @@ -42,7 +42,10 @@ func TestPrepareFork_RewritesSnapshotManifest(t *testing.T) { Disks: []shimconfig.DiskConfig{ {Path: sourceDir + "/overlay.raw"}, {Path: "/volumes/shared.raw"}, + {Path: sourceDir + "/boot.asif", Parents: []string{sourceDir + "/disk.0.asif", "/images/shared.asif"}}, + {Path: sourceDir + "/agent.img", Readonly: true}, }, + MacAuxStoragePath: sourceDir + "/mac-aux.img", Networks: []shimconfig.NetworkConfig{ {MAC: "02:00:00:00:00:01"}, }, @@ -77,9 +80,13 @@ func TestPrepareFork_RewritesSnapshotManifest(t *testing.T) { assert.Equal(t, targetDir+"/kernel/initrd", updated.ShimConfig.InitrdPath) assert.Equal(t, targetDir+"/vz.sock", updated.ShimConfig.ControlSocket) assert.Equal(t, targetDir+"/logs/vz-shim.log", updated.ShimConfig.LogPath) - require.Len(t, updated.ShimConfig.Disks, 2) + require.Len(t, updated.ShimConfig.Disks, 4) assert.Equal(t, targetDir+"/overlay.raw", updated.ShimConfig.Disks[0].Path) assert.Equal(t, "/volumes/shared.raw", updated.ShimConfig.Disks[1].Path) + assert.Equal(t, targetDir+"/boot.asif", updated.ShimConfig.Disks[2].Path) + assert.Equal(t, []string{targetDir + "/disk.0.asif", "/images/shared.asif"}, updated.ShimConfig.Disks[2].Parents) + assert.Equal(t, shimconfig.DiskConfig{Path: targetDir + "/agent.img", Readonly: true}, updated.ShimConfig.Disks[3]) + assert.Equal(t, targetDir+"/mac-aux.img", updated.ShimConfig.MacAuxStoragePath) require.Len(t, updated.ShimConfig.Networks, 1) assert.Equal(t, "02:00:00:00:00:01", updated.ShimConfig.Networks[0].MAC) } diff --git a/lib/hypervisor/vz/macos_config_test.go b/lib/hypervisor/vz/macos_config_test.go index 38fbac0b0..ecc276b11 100644 --- a/lib/hypervisor/vz/macos_config_test.go +++ b/lib/hypervisor/vz/macos_config_test.go @@ -11,7 +11,8 @@ import ( ) func TestMacOSStarterConfig(t *testing.T) { - cfg := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: 2, MemoryBytes: 4 << 30, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: []byte{1}, MachineIdentifierData: []byte{2}, AuxStoragePath: "/vm/aux.img"}, Disks: []hypervisor.DiskConfig{{Path: "/vm/root.raw"}}} + bootDisk := hypervisor.DiskConfig{Path: "/vm/boot.asif", Format: hypervisor.DiskFormatASIF, Parents: []string{"/vm/disk.0.asif"}} + cfg := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: 2, MemoryBytes: 4 << 30, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: []byte{1}, MachineIdentifierData: []byte{2}, AuxStoragePath: "/vm/aux.img"}, Disks: []hypervisor.DiskConfig{bootDisk}} require.NoError(t, NewStarter().ValidateConfig(cfg)) shim := buildShimConfigFromVMConfig(cfg, "/vm/vz.sock") require.Equal(t, base64.StdEncoding.EncodeToString([]byte{1}), shim.MacHardwareModelData) @@ -40,6 +41,15 @@ func TestMacOSStarterConfig(t *testing.T) { bad = cfg bad.Disks = []hypervisor.DiskConfig{{Readonly: true}} require.Error(t, NewStarter().ValidateConfig(bad)) + agent := cfg + agent.Disks = []hypervisor.DiskConfig{bootDisk, {Path: "/vm/agent.img", Readonly: true}} + require.NoError(t, NewStarter().ValidateConfig(agent), "a read-only disk may follow the boot disk") + bad = cfg + bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/agent.img", Readonly: true}, bootDisk} + require.Error(t, NewStarter().ValidateConfig(bad), "the boot disk comes first") + bad = cfg + bad.Disks = []hypervisor.DiskConfig{bootDisk, {Path: "/vm/data.img"}} + require.Error(t, NewStarter().ValidateConfig(bad), "later disks are read-only") bad = cfg bad.MacOS = nil require.Error(t, NewStarter().ValidateConfig(bad)) diff --git a/lib/hypervisor/vz/shim_darwin_test.go b/lib/hypervisor/vz/shim_darwin_test.go new file mode 100644 index 000000000..a9bef337b --- /dev/null +++ b/lib/hypervisor/vz/shim_darwin_test.go @@ -0,0 +1,93 @@ +//go:build darwin + +package vz + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/stretchr/testify/require" +) + +// runShim runs the embedded shim with args and returns its stderr and exit code. +func runShim(t *testing.T, args ...string) (string, int) { + t.Helper() + bin, err := extractShim() + require.NoError(t, err) + var stderr bytes.Buffer + cmd := exec.Command(bin, args...) + cmd.Stderr = &stderr + err = cmd.Run() + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return stderr.String(), exitErr.ExitCode() + } + require.NoError(t, err) + return stderr.String(), 0 +} + +func runShimConfig(t *testing.T, config shimconfig.ShimConfig) (string, int) { + t.Helper() + config.LogPath = filepath.Join(t.TempDir(), "vz-shim.log") + data, err := json.Marshal(config) + require.NoError(t, err) + return runShim(t, "-config", string(data)) +} + +func TestShimRejectsMissingOrInvalidConfig(t *testing.T) { + stderr, code := runShim(t) + require.Equal(t, 1, code) + require.Contains(t, stderr, "-config is required") + stderr, code = runShim(t, "-config", "{") + require.Equal(t, 1, code) + require.Contains(t, stderr, "invalid config JSON") +} + +func TestShimMacBootRequiresCompleteIdentity(t *testing.T) { + for _, c := range []shimconfig.ShimConfig{ + {MacHardwareModelData: "AA=="}, + {MacMachineIdentifierData: "AA=="}, + {MacAuxStoragePath: "/missing"}, + } { + stderr, code := runShimConfig(t, c) + require.Equal(t, 1, code) + require.Contains(t, stderr, "macOS boot requires") + } +} + +func TestShimMacPlatformRejectsMalformedModel(t *testing.T) { + stderr, code := runShimConfig(t, shimconfig.ShimConfig{MacHardwareModelData: "not base64!", MacMachineIdentifierData: "AA==", MacAuxStoragePath: "/missing"}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "decode Mac hardware model") +} + +// The shim holds a lock named for the identity's bytes, so encodings of the same +// identity contend for it. +func TestShimMacIdentityLockExcludesEquivalentIdentity(t *testing.T) { + identity := []byte("identity") + path := filepath.Join("/tmp", fmt.Sprintf("hypeman-mac-identity-%d-%x.lock", os.Getuid(), sha256.Sum256(identity))) + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, err) + defer f.Close() + require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)) + + encoded := base64.StdEncoding.EncodeToString(identity) + "\n" + stderr, code := runShimConfig(t, shimconfig.ShimConfig{MacMachineIdentifierData: encoded}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "Mac machine identity already in use") + + require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_UN)) + stderr, code = runShimConfig(t, shimconfig.ShimConfig{MacMachineIdentifierData: encoded}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "macOS boot requires", "with the lock free the shim gets past admission") +} diff --git a/lib/hypervisor/vz/shimconfig/config.go b/lib/hypervisor/vz/shimconfig/config.go index 16f413876..90b4b9e71 100644 --- a/lib/hypervisor/vz/shimconfig/config.go +++ b/lib/hypervisor/vz/shimconfig/config.go @@ -71,6 +71,9 @@ type ShimConfig struct { type DiskConfig struct { Path string `json:"path"` Readonly bool `json:"readonly"` + // Parents, when set, are the read-only images below Path, base first; Path is + // then the ASIF overlay on top of them. + Parents []string `json:"parents,omitempty"` } // NetworkConfig represents a network interface. diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 321f832dd..15479b941 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -119,8 +119,16 @@ func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { if config.HotplugBytes != 0 || config.EnableRosetta || config.GuestMemory.EnableBalloon || config.GuestMemory.RequireBalloon || len(config.PCIDevices) != 0 || config.VGPUDevicePath != "" { return fmt.Errorf("macOS does not support hotplug, ballooning, Rosetta shares or PCI passthrough") } - if len(config.Disks) != 1 || config.Disks[0].Readonly || config.Disks[0].EffectiveFormat() != hypervisor.DiskFormatRaw { - return fmt.Errorf("macOS requires one writable raw boot disk") + // The boot disk comes first; VZ does not boot macOS from a later disk. It is a + // writable ASIF overlay on its image's layers. Any other disk, such as the guest + // agent disk, is read-only and raw. + if boot := config.Disks; len(boot) == 0 || boot[0].Readonly || boot[0].EffectiveFormat() != hypervisor.DiskFormatASIF || len(boot[0].Parents) == 0 { + return fmt.Errorf("macOS requires a writable ASIF boot disk with its image layers first") + } + for _, disk := range config.Disks[1:] { + if !disk.Readonly || len(disk.Parents) != 0 || disk.EffectiveFormat() != hypervisor.DiskFormatRaw { + return fmt.Errorf("macOS disks after the boot disk must be read-only raw disks") + } } if config.VCPUs < 2 || uint(config.VCPUs) > vz.VirtualMachineConfigurationMaximumAllowedCPUCount() { return fmt.Errorf("macOS CPU count is outside the host's supported range") @@ -232,6 +240,7 @@ func buildShimConfigFromVMConfig(config hypervisor.VMConfig, socketPath string) cfg.Disks = append(cfg.Disks, shimconfig.DiskConfig{ Path: disk.Path, Readonly: disk.Readonly, + Parents: disk.Parents, }) } for _, n := range config.Networks { diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index ca72c1994..494365288 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -26,6 +26,7 @@ func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) + seenMacOSLayers := make(map[string]struct{}) layersDir := filepath.Join(imagesDir, "layers") err := walkWithContext(ctx, imagesDir, func(path string, info os.FileInfo, err error) error { @@ -70,6 +71,10 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir } return fmt.Errorf("unmarshal image metadata %s: %w", path, err) } + if meta.Status == StatusReady && meta.MacOS != nil { + total += macOSImageAllocatedBytes(filepath.Dir(path), &meta, seenMacOSLayers) + return nil + } if meta.Status == StatusReady { rootfsPaths, globErr := filepath.Glob(filepath.Join(filepath.Dir(path), "rootfs.*")) if globErr != nil { @@ -273,6 +278,45 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin return total, nil } +// macOSImageAllocatedBytes returns the disk a macOS image holds: its chain files' +// allocated blocks, not their logical size, and its auxiliary storage. A layer seen +// in an earlier image is not counted again: images that share a base clone its +// disk, keyed by diff ID, or by inode for a local import, which has none. +func macOSImageAllocatedBytes(dir string, meta *imageMetadata, seen map[string]struct{}) int64 { + var total int64 + for i := range meta.MacOSDiskLayers { + path := filepath.Join(dir, MacOSDiskLayerFile(i)) + info, err := os.Stat(path) + if err != nil { + continue + } + key := path + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + key = fmt.Sprintf("inode:%d:%d", stat.Dev, stat.Ino) + } + if i < len(meta.MacOSDiskDiffIDs) { + key = meta.MacOSDiskDiffIDs[i] + } + if _, counted := seen[key]; counted { + continue + } + seen[key] = struct{}{} + total += fileAllocatedBytes(info) + } + if info, err := os.Stat(filepath.Join(dir, MacOSMachineAuxFile)); err == nil { + total += fileAllocatedBytes(info) + } + return total +} + +// fileAllocatedBytes is a file's allocated blocks, or its size where those are unknown. +func fileAllocatedBytes(info os.FileInfo) int64 { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return stat.Blocks * 512 + } + return info.Size() +} + type rootfsIdentity struct { dev uint64 ino uint64 diff --git a/lib/images/disk_usage_macos_test.go b/lib/images/disk_usage_macos_test.go new file mode 100644 index 000000000..23e60b72c --- /dev/null +++ b/lib/images/disk_usage_macos_test.go @@ -0,0 +1,43 @@ +package images + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// writeMacOSImageForUsage writes a ready macOS image whose chain files hold 1 MiB +// of data in a 64 MiB logical disk each, with diffIDs for its layers. +func writeMacOSImageForUsage(t *testing.T, imagesDir, digest string, diffIDs []string) { + t.Helper() + dir := filepath.Join(imagesDir, "content", digest) + require.NoError(t, os.MkdirAll(dir, 0700)) + for i := range diffIDs { + file, err := os.Create(filepath.Join(dir, MacOSDiskLayerFile(i))) + require.NoError(t, err) + _, err = file.Write(make([]byte, 1<<20)) + require.NoError(t, err) + require.NoError(t, file.Truncate(64<<20)) + require.NoError(t, file.Close()) + } + require.NoError(t, os.WriteFile(filepath.Join(dir, MacOSMachineAuxFile), []byte("aux"), 0600)) + data, err := json.Marshal(imageMetadata{Name: "builds/" + digest, Digest: "sha256:" + digest, Status: StatusReady, MacOS: &MacOSImage{}, MacOSDiskLayers: len(diffIDs), MacOSDiskDiffIDs: diffIDs, SizeBytes: 64 << 20}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0600)) +} + +func TestMacOSImageDiskUsageCountsAllocatedBytesAndSharedLayersOnce(t *testing.T) { + imagesDir := t.TempDir() + writeMacOSImageForUsage(t, imagesDir, "aaaa", []string{"sha256:base"}) + writeMacOSImageForUsage(t, imagesDir, "bbbb", []string{"sha256:base", "sha256:build"}) + + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) + require.NoError(t, err) + // Two distinct layers of about 1 MiB each, not three 64 MiB logical disks. + require.Greater(t, total, int64(2<<20)-1) + require.Less(t, total, int64(4<<20)) +} diff --git a/lib/images/finalize.go b/lib/images/finalize.go index c16396702..2d77cffff 100644 --- a/lib/images/finalize.go +++ b/lib/images/finalize.go @@ -21,6 +21,9 @@ type stagedImageFiles struct { func (s stagedImageFiles) remove() { os.Remove(s.disk) if s.macos != nil { + for _, layer := range s.macos.layers { + os.Remove(layer) + } os.Remove(s.macos.aux) } } diff --git a/lib/images/macos_asif.go b/lib/images/macos_asif.go new file mode 100644 index 000000000..5098a7f6a --- /dev/null +++ b/lib/images/macos_asif.go @@ -0,0 +1,128 @@ +package images + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/paths" +) + +// MacOSDiskLayerFile names the ith file of a macOS boot disk chain, in an image or +// an instance. The chain is ASIF: disk.0.asif is the base and each later file is an +// overlay on the one before it. A pulled image has one file per OCI layer; an +// imported image has one. +func MacOSDiskLayerFile(i int) string { + return fmt.Sprintf("disk.%d.asif", i) +} + +// MacOSDiskChain returns the boot disk chain of a ready macOS image, base first. +func MacOSDiskChain(p *paths.Paths, imageName, digest string) ([]string, error) { + ref, err := ParseNormalizedRef(imageName) + if err != nil { + return nil, fmt.Errorf("parse image name: %w", err) + } + layout := resolveImageLayout(p, ref.Repository(), strings.TrimPrefix(digest, "sha256:")) + meta, err := readMetadataAt(layout) + if err != nil { + return nil, err + } + if meta.MacOS == nil || meta.MacOSDiskLayers < 1 { + return nil, fmt.Errorf("%w: image is not a macOS disk chain", ErrInvalidPlatform) + } + chain := make([]string, meta.MacOSDiskLayers) + for i := range chain { + chain[i] = filepath.Join(layout.dir, MacOSDiskLayerFile(i)) + } + return chain, nil +} + +// OpenMacOSDiskChain opens an ASIF chain read-only, base first. Each layer must name +// the one below it as its parent. +func OpenMacOSDiskChain(chain []string) (*diskimagekit.DiskImage, error) { + if len(chain) == 0 { + return nil, fmt.Errorf("empty macOS disk chain") + } + img, err := diskimagekit.Open(chain[0], diskimagekit.WithOpenMode(diskimagekit.ReadOnly)) + if err != nil { + return nil, fmt.Errorf("open %s: %w", filepath.Base(chain[0]), err) + } + for _, layer := range chain[1:] { + if err := img.Append(layer, diskimagekit.WithOpenMode(diskimagekit.ReadOnly)); err != nil { + img.Close() + return nil, fmt.Errorf("stack %s: %w", filepath.Base(layer), err) + } + } + for i, info := range img.Layers() { + if info.Format != diskimagekit.ASIF { + img.Close() + return nil, fmt.Errorf("macOS disk layer %d is not ASIF", i) + } + } + return img, nil +} + +// macOSChainSize validates a chain and returns its logical size in bytes. +func macOSChainSize(chain []string) (int64, error) { + img, err := OpenMacOSDiskChain(chain) + if err != nil { + return 0, err + } + defer img.Close() + return img.Size(), nil +} + +// ConvertRawToASIF writes a raw disk into a new ASIF image at dst with the same +// logical size. All-zero chunks are skipped, so they stay unallocated. +func ConvertRawToASIF(ctx context.Context, src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + const block = 512 + if info.Size() == 0 || info.Size()%block != 0 { + return fmt.Errorf("raw disk size %d is not a positive multiple of %d", info.Size(), block) + } + out, err := diskimagekit.CreateASIF(dst, info.Size()/block) + if err != nil { + return err + } + const chunk = 1 << 20 + buf, zero := make([]byte, chunk), make([]byte, chunk) + for offset := int64(0); offset < info.Size(); offset += chunk { + if err = ctx.Err(); err != nil { + break + } + n, readErr := io.ReadFull(io.NewSectionReader(in, offset, chunk), buf) + if readErr != nil && readErr != io.ErrUnexpectedEOF { + err = readErr + break + } + if bytes.Equal(buf[:n], zero[:n]) { + continue + } + if _, err = out.WriteAt(buf[:n], offset); err != nil { + break + } + } + if err == nil { + err = out.Sync() + } + if closeErr := out.Close(); err == nil { + err = closeErr + } + if err != nil { + os.Remove(dst) + } + return err +} diff --git a/lib/images/macos_asif_test.go b/lib/images/macos_asif_test.go new file mode 100644 index 000000000..2f21e4bd5 --- /dev/null +++ b/lib/images/macos_asif_test.go @@ -0,0 +1,57 @@ +package images + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConvertRawToASIF(t *testing.T) { + dir := t.TempDir() + raw := filepath.Join(dir, "disk.img") + data := make([]byte, 256<<20) + copy(data, "first chunk") + copy(data[100<<20+123:], "middle") + copy(data[len(data)-4:], "tail") + require.NoError(t, os.WriteFile(raw, data, 0600)) + asif := filepath.Join(dir, "disk.asif") + require.NoError(t, ConvertRawToASIF(context.Background(), raw, asif)) + + img, err := OpenMacOSDiskChain([]string{asif}) + require.NoError(t, err) + defer img.Close() + require.Equal(t, int64(len(data)), img.Size()) + got := make([]byte, len(data)) + _, err = img.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, data, got) + info, err := os.Stat(asif) + require.NoError(t, err) + require.Less(t, info.Size(), int64(len(data))/4, "zero chunks are not allocated") +} + +func TestConvertRawToASIFRejectsPartialBlocksAndCancellation(t *testing.T) { + dir := t.TempDir() + raw := filepath.Join(dir, "disk.img") + require.NoError(t, os.WriteFile(raw, make([]byte, 1000), 0600)) + require.Error(t, ConvertRawToASIF(context.Background(), raw, filepath.Join(dir, "a.asif"))) + + require.NoError(t, os.WriteFile(raw, make([]byte, 4<<20), 0600)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := filepath.Join(dir, "b.asif") + require.ErrorIs(t, ConvertRawToASIF(ctx, raw, out), context.Canceled) + require.NoFileExists(t, out, "a failed conversion leaves no image") +} + +func TestOpenMacOSDiskChainRejectsUnrelatedOverlay(t *testing.T) { + _, overlay := macOSFixtureChain(t, t.TempDir()) + otherBase, _ := macOSFixtureChain(t, t.TempDir()) + _, err := OpenMacOSDiskChain([]string{otherBase, overlay}) + require.Error(t, err) + _, err = OpenMacOSDiskChain(nil) + require.Error(t, err) +} diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 5e41bb8e7..5a6f3fde6 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -2,34 +2,42 @@ package images import ( "bytes" + "context" "encoding/json" + "errors" "fmt" "io" "os" + "os/exec" "path/filepath" + "strconv" ) const maxMacOSMetadataBytes = 64 << 10 -// macOSMachinePayload is a complete cold-boot bundle, not a container rootfs. -// Admission of live storage and export sanitation belong to the caller. +// macOSMachinePayload is a complete cold-boot bundle, not a container rootfs: an +// ASIF boot disk chain, base first, with the auxiliary storage and platform of its +// top layer. Admission of live storage and export sanitation belong to the caller. type macOSMachinePayload struct { - Disk string + Disks []string Aux string Platform *MacOSImage } -// ValidateMacOSBundle validates a fixed-layout export, rejecting unknown metadata -// fields so an accidental credential field cannot become part of an image. -func ValidateMacOSBundle(root string) (*MacOSImage, error) { - bundle, err := readMacOSMachineBundle(root, "disk.img", "aux.img", "config.json", true) - if err != nil { - return nil, err +// RequireClosedFile fails unless no process holds path open. lsof exits 1 when +// nothing holds the file; any other failure leaves the storage unverified. +func RequireClosedFile(ctx context.Context, path string) error { + err := exec.CommandContext(ctx, "/usr/sbin/lsof", "-t", path).Run() + if err == nil { + return fmt.Errorf("storage is open: stop its VM first") + } + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + return fmt.Errorf("cannot verify storage is closed: %w", err) } - return bundle.Platform, nil + return nil } -func readMacOSMachineBundle(root, diskName, auxName, configName string, strict bool) (*macOSMachinePayload, error) { +func readMacOSMachineBundle(root, diskName, auxName, configName string) (*macOSMachinePayload, error) { disk, err := machineBundleFile(root, diskName) if err != nil { return nil, err @@ -42,10 +50,60 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b if err != nil { return nil, err } - // Compare inodes: different hardlink names do not establish distinctness. - if sameFile(disk, aux) || sameFile(disk, config) || sameFile(aux, config) { - return nil, fmt.Errorf("machine bundle files must be distinct") + if err := CheckMachineFiles(disk, aux, config); err != nil { + return nil, err + } + platform, err := readMacOSPlatform(config) + if err != nil { + return nil, err + } + return &macOSMachinePayload{Disks: []string{disk}, Aux: aux, Platform: platform}, nil +} + +// readMacOSMachineLayers reads a pulled image unpacked one layer per numbered +// directory. Every layer holds a disk; the auxiliary storage and platform come from +// the highest layer that has them. +func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSMachinePayload, error) { + var payload macOSMachinePayload + var top string + for i := 0; ; i++ { + dir := filepath.Join(root, strconv.Itoa(i)) + if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { + break + } + disk, err := machineBundleFile(dir, diskName) + if err != nil { + return nil, fmt.Errorf("machine layer %d: %w", i, err) + } + payload.Disks = append(payload.Disks, disk) + top = dir + } + if len(payload.Disks) == 0 { + return nil, fmt.Errorf("machine image needs a disk in every layer") + } + // Only the top layer's auxiliary storage and platform count, so a lower layer + // can be reused from another image without unpacking it. + aux, err := machineBundleFile(top, auxName) + if err != nil { + return nil, fmt.Errorf("machine image top layer: %w", err) } + payload.Aux = aux + config, err := machineBundleFile(top, configName) + if err != nil { + return nil, fmt.Errorf("machine image top layer: %w", err) + } + if err := CheckMachineFiles(append([]string{aux, config}, payload.Disks...)...); err != nil { + return nil, err + } + platform, err := readMacOSPlatform(config) + if err != nil { + return nil, err + } + payload.Platform = platform + return &payload, nil +} + +func readMacOSPlatform(config string) (*MacOSImage, error) { file, err := os.Open(config) if err != nil { return nil, err @@ -59,9 +117,6 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b return nil, fmt.Errorf("machine platform metadata is too large") } decoder := json.NewDecoder(bytes.NewReader(data)) - if strict { - decoder.DisallowUnknownFields() - } var platform MacOSImage if err := decoder.Decode(&platform); err != nil { return nil, err @@ -72,7 +127,7 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b if err := platform.Validate(); err != nil { return nil, err } - return &macOSMachinePayload{Disk: disk, Aux: aux, Platform: &platform}, nil + return &platform, nil } func machineBundleFile(root, relative string) (string, error) { @@ -101,8 +156,25 @@ func machineBundleFile(root, relative string) (string, error) { return path, nil } -func sameFile(a, b string) bool { - ai, errA := os.Stat(a) - bi, errB := os.Stat(b) - return errA == nil && errB == nil && os.SameFile(ai, bi) +// CheckMachineFiles requires each path to be a nonempty regular file, not a +// symlink, and no two to be the same file. Inodes are compared: different hardlink +// names do not establish distinctness. +func CheckMachineFiles(paths ...string) error { + seen := make([]os.FileInfo, 0, len(paths)) + for _, path := range paths { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + for _, other := range seen { + if os.SameFile(other, info) { + return fmt.Errorf("machine storage files must be distinct") + } + } + seen = append(seen, info) + } + return nil } diff --git a/lib/images/macos_bundle_test.go b/lib/images/macos_bundle_test.go index 0a5d883a0..b35675643 100644 --- a/lib/images/macos_bundle_test.go +++ b/lib/images/macos_bundle_test.go @@ -17,20 +17,20 @@ func TestMacOSBundleValidation(t *testing.T) { for _, name := range []string{"valid", "empty-disk", "hardlink", "escape", "oversized", "truncated", "trailing", "unknown-field", "invalid-mac"} { t.Run(name, func(t *testing.T) { root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "disk.img"), []byte("disk"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "disk.asif"), []byte("disk"), 0600)) require.NoError(t, os.WriteFile(filepath.Join(root, "aux.img"), []byte("aux"), 0600)) config := append([]byte(nil), data...) switch name { case "empty-disk": - require.NoError(t, os.WriteFile(filepath.Join(root, "disk.img"), nil, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "disk.asif"), nil, 0600)) case "hardlink": require.NoError(t, os.Remove(filepath.Join(root, "aux.img"))) - require.NoError(t, os.Link(filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"))) + require.NoError(t, os.Link(filepath.Join(root, "disk.asif"), filepath.Join(root, "aux.img"))) case "escape": outside := filepath.Join(t.TempDir(), "disk") require.NoError(t, os.WriteFile(outside, []byte("outside"), 0600)) - require.NoError(t, os.Remove(filepath.Join(root, "disk.img"))) - require.NoError(t, os.Symlink(outside, filepath.Join(root, "disk.img"))) + require.NoError(t, os.Remove(filepath.Join(root, "disk.asif"))) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "disk.asif"))) case "oversized": config = []byte(strings.Repeat(" ", maxMacOSMetadataBytes+1)) case "truncated": @@ -43,18 +43,13 @@ func TestMacOSBundleValidation(t *testing.T) { config = []byte(strings.Replace(string(config), platform.MAC, "invalid", 1)) } require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), config, 0600)) - got, err := ValidateMacOSBundle(root) - if name == "valid" { + bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json") + if name == "valid" || name == "unknown-field" { require.NoError(t, err) - require.Equal(t, &platform, got) + require.Equal(t, &platform, bundle.Platform) } else { require.Error(t, err) } - if name == "unknown-field" { - bundle, err := readMacOSMachineBundle(root, "disk.img", "aux.img", "config.json", false) - require.NoError(t, err, "import/OCI compatibility permits unknown non-export metadata") - require.Equal(t, &platform, bundle.Platform) - } }) } } diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index d0de8ee4d..2b1a8b16d 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "time" @@ -19,9 +18,9 @@ import ( ) // ImportMacOSImage is an offline, administrator-only import of a STOPPED macvm -// bundle. It intentionally is not exposed as an HTTP host-filesystem read API. -// Imports preserve identity: instances from this image may run sequentially, -// not concurrently, until rekeying is validated. +// bundle. It intentionally is not exposed as an HTTP host-filesystem read API. The +// bundle's boot disk is disk.asif, cloned as is, or a raw disk.img, converted to +// ASIF. The image is a one-layer chain. func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) (*Image, error) { ref, err := ParseNormalizedRef(name) if err != nil { @@ -30,12 +29,16 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if ref.IsDigest() { return nil, fmt.Errorf("%w: import requires a tagged name", ErrInvalidName) } - bundle, err := readMacOSMachineBundle(source, "disk.img", "aux.img", "config.json", false) + diskName := MacOSMachineDiskFile + if _, err := os.Lstat(filepath.Join(source, diskName)); os.IsNotExist(err) { + diskName = "disk.img" + } + bundle, err := readMacOSMachineBundle(source, diskName, MacOSMachineAuxFile, MacOSMachinePlatformFile) if err != nil { return nil, err } mac := bundle.Platform - for _, file := range []string{"disk.img", "aux.img"} { + for _, file := range []string{diskName, MacOSMachineAuxFile} { file = filepath.Join(source, file) info, e := os.Lstat(file) if e != nil { @@ -44,11 +47,8 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if !info.Mode().IsRegular() { return nil, fmt.Errorf("bundle file must be regular: %s", file) } - if e = exec.CommandContext(ctx, "lsof", "-t", file).Run(); e == nil { - return nil, fmt.Errorf("bundle storage is open: stop its VM first") - } - if exit, ok := e.(*exec.ExitError); !ok || exit.ExitCode() != 1 { - return nil, fmt.Errorf("cannot verify storage is closed: %v", e) + if e = RequireClosedFile(ctx, file); e != nil { + return nil, e } } root := p.ImageRepositoryDir(ref.Repository()) @@ -62,21 +62,32 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, err } defer os.RemoveAll(stage) - for src, dst := range map[string]string{bundle.Disk: "rootfs.raw", bundle.Aux: "aux.img"} { + clones := map[string]string{bundle.Aux: MacOSMachineAuxFile} + if diskName == MacOSMachineDiskFile { + clones[bundle.Disks[0]] = machineDiskFile + } else if err = ConvertRawToASIF(ctx, bundle.Disks[0], filepath.Join(stage, machineDiskFile)); err != nil { + return nil, fmt.Errorf("convert boot disk to ASIF: %w", err) + } + for src, dst := range clones { if err = unix.Clonefile(src, filepath.Join(stage, dst), unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil { return nil, fmt.Errorf("clone bundle: %w", err) } - if err = os.Chmod(filepath.Join(stage, dst), 0600); err != nil { + } + for _, name := range []string{machineDiskFile, MacOSMachineAuxFile} { + if err = os.Chmod(filepath.Join(stage, name), 0600); err != nil { return nil, err } } + logicalSize, err := macOSChainSize([]string{filepath.Join(stage, machineDiskFile)}) + if err != nil { + return nil, fmt.Errorf("imported boot disk: %w", err) + } // Hash cloned content, not the caller's mutable paths. Per-file hashes and // logical sizes make the digest unambiguous, including sparse zero ranges. h := sha256.New() canonical, _ := json.Marshal(mac) h.Write(canonical) - var logicalSize int64 - for _, name := range []string{"rootfs.raw", "aux.img"} { + for _, name := range []string{machineDiskFile, MacOSMachineAuxFile} { f, e := os.Open(filepath.Join(stage, name)) if e != nil { return nil, e @@ -88,13 +99,10 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, e } fmt.Fprintf(h, "\n%s:%d:%x", name, n, fh.Sum(nil)) - if name == "rootfs.raw" { - logicalSize = n - } } digestHex := hex.EncodeToString(h.Sum(nil)) digest := "sha256:" + digestHex - meta := &imageMetadata{Name: ref.Repository() + "@" + digest, Digest: digest, Platform: "darwin/arm64", MacOS: mac, Status: StatusReady, SizeBytes: logicalSize, CreatedAt: time.Now().UTC()} + meta := &imageMetadata{Name: ref.Repository() + "@" + digest, Digest: digest, Platform: "darwin/arm64", MacOS: mac, MacOSDiskLayers: 1, Status: StatusReady, SizeBytes: logicalSize, CreatedAt: time.Now().UTC()} if err = writeMetadataFile(filepath.Join(stage, "metadata.json"), meta); err != nil { return nil, err } diff --git a/lib/images/macos_import_darwin_test.go b/lib/images/macos_import_darwin_test.go index f8cb4b660..ca04ca228 100644 --- a/lib/images/macos_import_darwin_test.go +++ b/lib/images/macos_import_darwin_test.go @@ -18,7 +18,9 @@ func TestMacOSOfflineImport(t *testing.T) { c := MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30} b, _ := json.Marshal(c) require.NoError(t, os.WriteFile(filepath.Join(source, "config.json"), b, 0600)) - require.NoError(t, os.WriteFile(filepath.Join(source, "disk.img"), []byte("boot disk"), 0600)) + raw := make([]byte, 4<<20) + copy(raw[1<<20:], "boot disk") + require.NoError(t, os.WriteFile(filepath.Join(source, "disk.img"), raw, 0600)) require.NoError(t, os.WriteFile(filepath.Join(source, "aux.img"), []byte("aux storage"), 0600)) img, err := ImportMacOSImage(context.Background(), p, "localhost/macos:test", source) require.NoError(t, err) @@ -28,9 +30,18 @@ func TestMacOSOfflineImport(t *testing.T) { loaded, err := m.GetImage(context.Background(), img.Name) require.NoError(t, err) require.NotNil(t, loaded.MacOS) - disk, err := GetDiskPath(p, img.Name, img.Digest) + chain, err := MacOSDiskChain(p, img.Name, img.Digest) require.NoError(t, err) - got, err := os.ReadFile(filepath.Join(filepath.Dir(disk), "aux.img")) + require.Len(t, chain, 1, "a raw disk imports as a one-layer ASIF chain") + disk, err := OpenMacOSDiskChain(chain) + require.NoError(t, err) + require.Equal(t, int64(len(raw)), disk.Size()) + buf := make([]byte, len("boot disk")) + _, err = disk.ReadAt(buf, 1<<20) + require.NoError(t, err) + require.Equal(t, "boot disk", string(buf)) + require.NoError(t, disk.Close()) + got, err := os.ReadFile(filepath.Join(filepath.Dir(chain[0]), "aux.img")) require.NoError(t, err) require.Equal(t, "aux storage", string(got)) _, err = m.TagImage(context.Background(), img.Name, "localhost/alias:tag") diff --git a/lib/images/macos_machine.go b/lib/images/macos_machine.go index 8ce357019..5396b2136 100644 --- a/lib/images/macos_machine.go +++ b/lib/images/macos_machine.go @@ -3,6 +3,7 @@ package images import ( "context" "fmt" + "log/slog" "os" "path/filepath" "runtime" @@ -20,12 +21,37 @@ const ( MacOSMachineFormatLabel = "io.hypeman.machine-image.disk-format" MacOSMachineAuxLabel = "io.hypeman.machine-image.aux-path" MacOSMachinePlatformLabel = "io.hypeman.machine-image.platform-path" + // MacOSMachineDiskDigestsLabel lists the sha256 of each layer's disk file, base + // first, comma-separated. A publication over a base checks its layers against it. + MacOSMachineDiskDigestsLabel = "io.hypeman.machine-image.disk-digests" ) -// stagedMacOS is the part of a staged macOS machine image beyond its boot disk. +// The fixed label values of a macOS machine image, and the file names a published +// image uses in each layer. +const ( + MacOSMachineVersion = "2" + MacOSMachineKind = "macos-image" + MacOSMachineFormat = "asif" + MacOSMachineDiskFile = "disk.asif" + MacOSMachineAuxFile = "aux.img" + MacOSMachinePlatformFile = "config.json" +) + +// MacOSMachineLabels returns the labels of a macOS machine image that uses the +// standard file names. +func MacOSMachineLabels() map[string]string { + return map[string]string{ + MacOSMachineVersionLabel: MacOSMachineVersion, MacOSMachineKindLabel: MacOSMachineKind, MacOSMachineFormatLabel: MacOSMachineFormat, + MacOSMachineDiskLabel: MacOSMachineDiskFile, MacOSMachineAuxLabel: MacOSMachineAuxFile, MacOSMachinePlatformLabel: MacOSMachinePlatformFile, + } +} + +// stagedMacOS is the part of a staged macOS machine image beyond its base disk. type stagedMacOS struct { - aux string // staged auxiliary storage, beside its final path + layers []string // staged overlays above the base, in order, beside their final paths + aux string // staged auxiliary storage, beside its final path platform *MacOSImage + diffIDs []string // OCI diff ID of each chain layer, base first } // finalDiskPath is where finalization installs the staged disk: the layout's rootfs @@ -37,9 +63,9 @@ func (s stagedImageFiles) finalDiskPath(layout imageLayout) string { return layout.disk } -// installMacOSMachine installs a macOS machine image's auxiliary storage and records -// its platform on meta. It returns the paths installed, for rollback. Rootfs images -// install nothing here. +// installMacOSMachine installs a macOS machine image's overlays and auxiliary +// storage and records its platform and chain length on meta. It returns the paths +// installed, for rollback. Rootfs images install nothing here. func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imageMetadata, actualPlatform Platform) ([]string, error) { if actualPlatform.OS == "darwin" && staged.macos == nil { return nil, fmt.Errorf("macOS image requires a validated machine bundle") @@ -47,12 +73,22 @@ func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imag if staged.macos == nil { return nil, nil } - auxPath := filepath.Join(layout.dir, "aux.img") + var installed []string + for i, layer := range staged.macos.layers { + path := filepath.Join(layout.dir, MacOSDiskLayerFile(i+1)) + if err := os.Rename(layer, path); err != nil { + return installed, err + } + installed = append(installed, path) + } + auxPath := filepath.Join(layout.dir, MacOSMachineAuxFile) if err := os.Rename(staged.macos.aux, auxPath); err != nil { - return nil, err + return installed, err } meta.MacOS = staged.macos.platform - return []string{auxPath}, nil + meta.MacOSDiskLayers = len(staged.macos.layers) + 1 + meta.MacOSDiskDiffIDs = staged.macos.diffIDs + return append(installed, auxPath), nil } // checkCachedMachinePlatform refuses to reuse a record whose platform does not match @@ -84,63 +120,106 @@ func checkCachedMachinePlatform(p *paths.Paths, ref *ResolvedRef, requested Plat return nil } -// stageMacOSOutputs stages a macOS machine image's raw boot disk and auxiliary +// reusableMacOSLayers returns, for each lower layer of a chain with diffIDs, the +// disk of a ready image whose chain has the same layers up to and including it, or +// "". Matching the layers beneath keeps each reused overlay on the chain it was +// built on. The top layer is never reused, since it carries the image's own files. +func (m *manager) reusableMacOSLayers(diffIDs []string) []string { + reuse := make([]string, len(diffIDs)) + metas, err := listAllMetadata(m.paths) + if err != nil { + slog.Warn("list images for macOS layer reuse", "error", err) + return reuse + } + for _, meta := range metas { + if meta.Status != StatusReady || meta.MacOS == nil || len(meta.MacOSDiskDiffIDs) != meta.MacOSDiskLayers { + continue + } + shared := 0 + for shared < len(meta.MacOSDiskDiffIDs) && shared < len(diffIDs)-1 && meta.MacOSDiskDiffIDs[shared] == diffIDs[shared] { + shared++ + } + if shared == 0 { + continue + } + chain, err := MacOSDiskChain(m.paths, meta.Name, meta.Digest) + if err != nil { + slog.Warn("read macOS image chain for layer reuse", "image", meta.Name, "error", err) + continue + } + for i := range shared { + if reuse[i] == "" { + reuse[i] = chain[i] + } + } + } + return reuse +} + +// stageMacOSOutputs stages a macOS machine image's boot disk chain and auxiliary // storage in layoutDir, outside the manager lock. For other images it stages nothing // and returns diskTemp for the rootfs export. -func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, meta *containerMetadata, layoutDir, diskTemp, buildID string) (stagedImageFiles, error) { +func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, result *pullResult, layoutDir, diskTemp, buildID string) (stagedImageFiles, error) { staged := stagedImageFiles{disk: diskTemp} - if !isMacOSMachine(meta) { + if !isMacOSMachine(result.Metadata) { return staged, nil } start := time.Now() staged.disk = filepath.Join(layoutDir, machineDiskFile+".tmp-"+buildID) - err := stageMacOSMachine(&staged, root, meta, filepath.Join(layoutDir, "aux.img.tmp-"+buildID)) + err := stageMacOSMachine(&staged, root, result.Metadata, layoutDir, buildID) + if err == nil && result.Manifest != nil { + staged.macos.diffIDs = result.Manifest.Config.DiffIDs + } m.recordImageBuildPhase(ctx, digest, "filesystem_export", time.Since(start), phaseStatus(err), "not_applicable") return staged, err } -// readMacOSMachine reads the unpacked bundle that validated labels describe. +// readMacOSMachine reads the unpacked layers that validated labels describe. func readMacOSMachine(root string, meta *containerMetadata) (*macOSMachinePayload, error) { labels := meta.Labels - return readMacOSMachineBundle(root, labels[MacOSMachineDiskLabel], labels[MacOSMachineAuxLabel], labels[MacOSMachinePlatformLabel], false) + return readMacOSMachineLayers(root, labels[MacOSMachineDiskLabel], labels[MacOSMachineAuxLabel], labels[MacOSMachinePlatformLabel]) } -// stageMacOSMachine copies an unpacked bundle's files to the staging paths. The -// recorded size is the boot disk's logical size, as for a local import. -func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMetadata, auxTemp string) error { +// stageMacOSMachine copies an unpacked image's disk chain and auxiliary storage to +// staging paths in layoutDir and checks that the chain stacks. The recorded size is +// the chain's logical size, as for a local import. +func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMetadata, layoutDir, buildID string) error { payload, err := readMacOSMachine(root, meta) if err != nil { return err } - if staged.sizeBytes, err = stageMachineFile(payload.Disk, staged.disk); err != nil { - return fmt.Errorf("stage boot disk: %w", err) + staged.macos = &stagedMacOS{aux: filepath.Join(layoutDir, "aux.img.tmp-"+buildID), platform: payload.Platform} + chain := []string{staged.disk} + for i, disk := range payload.Disks { + if i > 0 { + layer := filepath.Join(layoutDir, MacOSDiskLayerFile(i)+".tmp-"+buildID) + staged.macos.layers = append(staged.macos.layers, layer) + chain = append(chain, layer) + } + if err := stageMachineFile(disk, chain[i]); err != nil { + return fmt.Errorf("stage boot disk layer %d: %w", i, err) + } + } + if staged.sizeBytes, err = macOSChainSize(chain); err != nil { + return fmt.Errorf("machine boot disk: %w", err) } - staged.macos = &stagedMacOS{aux: auxTemp, platform: payload.Platform} - if _, err := stageMachineFile(payload.Aux, auxTemp); err != nil { + if err := stageMachineFile(payload.Aux, staged.macos.aux); err != nil { return fmt.Errorf("stage auxiliary storage: %w", err) } return nil } -func stageMachineFile(src, dst string) (int64, error) { +func stageMachineFile(src, dst string) error { // Copy helpers create the destination with the registry-supplied source mode, // and a multi-GB copy would expose contents before any later chmod. The copy // happens in installAtomically's fresh 0700 directory and is made 0600 before // it appears at the canonical staging path. - err := installAtomically(dst, func(path string) error { + return installAtomically(dst, func(path string) error { if err := forkvm.CopyRegularFile(src, path); err != nil { return err } return os.Chmod(path, 0600) }) - if err != nil { - return 0, err - } - info, err := os.Stat(dst) - if err != nil { - return 0, err - } - return info.Size(), nil } // macOSMachineHost is true where macOS machine images can boot. Other hosts @@ -188,9 +267,9 @@ func validateMacOSMachine(meta *containerMetadata) error { return fmt.Errorf("macOS machine requires darwin/arm64, not %s", normalized) } labels := meta.Labels - for _, want := range [][2]string{{MacOSMachineVersionLabel, "1"}, {MacOSMachineKindLabel, "macos-image"}, {MacOSMachineFormatLabel, "raw"}} { - if label := want[0]; labels[label] != want[1] { - return fmt.Errorf("darwin artifact requires %s=%q, got %q; ordinary container images are not macOS machines", label, want[1], labels[label]) + for label, want := range map[string]string{MacOSMachineVersionLabel: MacOSMachineVersion, MacOSMachineKindLabel: MacOSMachineKind, MacOSMachineFormatLabel: MacOSMachineFormat} { + if labels[label] != want { + return fmt.Errorf("darwin artifact requires %s=%q, got %q; ordinary container images are not macOS machines", label, want, labels[label]) } } pathLabels := []string{MacOSMachineDiskLabel, MacOSMachineAuxLabel, MacOSMachinePlatformLabel} diff --git a/lib/images/macos_machine_test.go b/lib/images/macos_machine_test.go index f2671a818..7f393797e 100644 --- a/lib/images/macos_machine_test.go +++ b/lib/images/macos_machine_test.go @@ -4,22 +4,23 @@ import ( "archive/tar" "compress/gzip" "context" - "crypto/sha256" "encoding/json" - "fmt" "io" "maps" "net/http" "net/http/httptest" "os" "path/filepath" + "slices" "strings" "sync/atomic" "testing" "time" + "github.com/Code-Hex/vz/v4/diskimagekit" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" + gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" @@ -30,26 +31,48 @@ import ( "github.com/stretchr/testify/require" ) +// macOSFixtureChain writes a small ASIF chain to dir: a base with data at two +// offsets and an overlay that replaces the second. It returns their paths. +func macOSFixtureChain(t *testing.T, dir string) (string, string) { + t.Helper() + base, overlay := filepath.Join(dir, "base.asif"), filepath.Join(dir, "overlay.asif") + img, err := diskimagekit.CreateASIF(base, (64<<20)/512) + require.NoError(t, err) + _, err = img.WriteAt([]byte("base data"), 0) + require.NoError(t, err) + _, err = img.WriteAt([]byte("replaced in overlay"), 32<<20) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(overlay, diskimagekit.Overlay)) + _, err = img.WriteAt([]byte("overlay data"), 32<<20) + require.NoError(t, err) + require.NoError(t, img.Close()) + return base, overlay +} + +func macOSFixturePlatform(t *testing.T) []byte { + t.Helper() + b, err := json.Marshal(MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30}) + require.NoError(t, err) + return b +} + +// macOSFixture writes an unpacked one-layer image: root/0 holds the disk, the +// auxiliary storage and the platform. func macOSFixture(t *testing.T) string { t.Helper() root := t.TempDir() - c := MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30} - b, err := json.Marshal(c) - require.NoError(t, err) - // A mostly empty disk, like a real one, so transport can be checked for sparseness. - disk := make([]byte, 64<<20) - copy(disk, "not a real boot disk: synthetic OCI transport fixture") - copy(disk[len(disk)-4:], "tail") - for f, data := range map[string][]byte{"config.json": b, "disk.img": disk, "aux.img": []byte("synthetic aux")} { - require.NoError(t, os.WriteFile(filepath.Join(root, f), data, 0600)) + layer := filepath.Join(root, "0") + require.NoError(t, os.MkdirAll(layer, 0700)) + for f, data := range map[string][]byte{"config.json": macOSFixturePlatform(t), "disk.asif": []byte("disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(layer, f), data, 0600)) } return root } func macOSFixtureMetadata() *containerMetadata { return &containerMetadata{OS: "darwin", Architecture: "arm64", Labels: map[string]string{ - MacOSMachineVersionLabel: "1", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "raw", - MacOSMachineDiskLabel: "disk.img", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "config.json", + MacOSMachineVersionLabel: "2", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "asif", + MacOSMachineDiskLabel: "disk.asif", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "config.json", }} } @@ -69,10 +92,10 @@ func TestMacOSMachineValidation(t *testing.T) { require.NoError(t, err) require.NotNil(t, payload) for _, tc := range []struct{ name, key, value string }{ - {"unsupported version", MacOSMachineVersionLabel, "2"}, {"container not machine", MacOSMachineKindLabel, ""}, - {"wrong format", MacOSMachineFormatLabel, "qcow2"}, {"absolute path", MacOSMachineDiskLabel, "/etc/passwd"}, - {"traversal", MacOSMachineDiskLabel, "../disk.img"}, {"missing aux", MacOSMachineAuxLabel, "missing.img"}, - {"duplicate file", MacOSMachineAuxLabel, "disk.img"}, + {"unsupported version", MacOSMachineVersionLabel, "1"}, {"container not machine", MacOSMachineKindLabel, ""}, + {"raw format", MacOSMachineFormatLabel, "raw"}, {"absolute path", MacOSMachineDiskLabel, "/etc/passwd"}, + {"traversal", MacOSMachineDiskLabel, "../disk.asif"}, {"missing aux", MacOSMachineAuxLabel, "missing.img"}, + {"duplicate file", MacOSMachineAuxLabel, "disk.asif"}, } { t.Run(tc.name, func(t *testing.T) { meta := macOSFixtureMetadata() @@ -89,7 +112,7 @@ func TestMacOSMachineValidation(t *testing.T) { require.NoError(t, err, alias) require.NotNil(t, payload, alias) } - require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), []byte(`{}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "0", "config.json"), []byte(`{}`), 0600)) _, err = parseMacOSMachine(root, macOSFixtureMetadata()) require.Error(t, err) @@ -97,49 +120,54 @@ func TestMacOSMachineValidation(t *testing.T) { for _, mac := range []string{"02:00:00:00:00:00:00:01", strings.Repeat("00:", 19) + "01"} { b, err := json.Marshal(MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: mac, CPUs: 4, Memory: 8 << 30}) require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), b, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "0", "config.json"), b, 0600)) _, err = parseMacOSMachine(root, macOSFixtureMetadata()) require.Error(t, err, mac) } } -// TestMacOSMachineOCIRoundTrip pushes a machine bundle to a registry and pulls it -// through the image manager. It is not a VZ boot. -func TestMacOSMachineOCIRoundTrip(t *testing.T) { - allowMacOSMachines(t) - source := macOSFixture(t) - work := t.TempDir() - layerPath := filepath.Join(work, "bundle.tar.gz") - file, err := os.OpenFile(layerPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) +// machineFixtureLayer writes files into a gzipped tar layer and returns it. +func machineFixtureLayer(t *testing.T, dir, name string, files map[string]string) gcr.Layer { + t.Helper() + path := filepath.Join(dir, name+".tar.gz") + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) require.NoError(t, err) gz, err := gzip.NewWriterLevel(file, gzip.BestSpeed) require.NoError(t, err) tw := tar.NewWriter(gz) - hashes := map[string]string{} - for _, f := range []string{"disk.img", "aux.img", "config.json"} { - input, err := os.Open(filepath.Join(source, f)) + for _, entry := range slices.Sorted(maps.Keys(files)) { + input, err := os.Open(files[entry]) require.NoError(t, err) info, err := input.Stat() require.NoError(t, err) - // The registry controls modes; a permissive disk must not become the canonical image mode. - mode := int64(0600) - if f == "disk.img" { - mode = 0666 - } - require.NoError(t, tw.WriteHeader(&tar.Header{Name: f, Mode: mode, Size: info.Size(), Typeflag: tar.TypeReg})) - h := sha256.New() - _, err = io.Copy(io.MultiWriter(tw, h), input) + // The registry controls modes; a permissive file must not become the canonical image mode. + require.NoError(t, tw.WriteHeader(&tar.Header{Name: entry, Mode: 0666, Size: info.Size(), Typeflag: tar.TypeReg})) + _, err = io.Copy(tw, input) require.NoError(t, err) require.NoError(t, input.Close()) - hashes[f] = fmt.Sprintf("%x", h.Sum(nil)) } require.NoError(t, tw.Close()) require.NoError(t, gz.Close()) require.NoError(t, file.Close()) - layer, err := tarball.LayerFromFile(layerPath) + layer, err := tarball.LayerFromFile(path) require.NoError(t, err) - image, err := mutate.AppendLayers(empty.Image, layer) + return layer +} + +// TestMacOSMachineOCIRoundTrip pushes a two-layer machine image, a base disk and an +// overlay with the auxiliary storage and platform, and pulls it through the image +// manager. It is not a VZ boot. +func TestMacOSMachineOCIRoundTrip(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + base, overlay := macOSFixtureChain(t, work) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + image, err := mutate.AppendLayers(empty.Image, + machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": base}), + machineFixtureLayer(t, work, "top", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) require.NoError(t, err) cfg, err := image.ConfigFile() require.NoError(t, err) @@ -166,26 +194,37 @@ func TestMacOSMachineOCIRoundTrip(t *testing.T) { require.Equal(t, StatusReady, pulled.Status) require.Equal(t, "darwin/arm64", pulled.Platform) require.NotNil(t, pulled.MacOS) - disk, err := GetDiskPath(p, pulled.Name, pulled.Digest) + require.Equal(t, int64(64<<20), *pulled.SizeBytes, "the recorded size is the chain's logical size") + + chain, err := MacOSDiskChain(p, pulled.Name, pulled.Digest) require.NoError(t, err) - require.Equal(t, machineDiskFile, filepath.Base(disk), "a raw machine disk is not named as a filesystem") - for f, path := range map[string]string{"disk.img": disk, "aux.img": filepath.Join(filepath.Dir(disk), "aux.img")} { - sum, err := fileSHA256(ctx, path) + require.Len(t, chain, 2) + for i, want := range []string{base, overlay} { + got, err := fileSHA256(ctx, chain[i]) require.NoError(t, err) - require.Equal(t, "sha256:"+hashes[f], sum, f) - info, err := os.Stat(path) + expected, err := fileSHA256(ctx, want) require.NoError(t, err) - require.Equal(t, os.FileMode(0600), info.Mode().Perm(), f) - if f == "disk.img" { - control := sparseControlBytes(t, filepath.Dir(path), info.Size()) - require.LessOrEqual(t, allocatedBytes(t, path), control+2*machineUnpackChunk, "the pulled disk stays sparse") - } + require.Equal(t, expected, got, "layer %d is installed byte for byte", i) + info, err := os.Stat(chain[i]) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + } + stacked, err := OpenMacOSDiskChain(chain) + require.NoError(t, err) + for offset, want := range map[int64]string{0: "base data", 32 << 20: "overlay data"} { + buf := make([]byte, len(want)) + _, err := stacked.ReadAt(buf, offset) + require.NoError(t, err) + require.Equal(t, want, string(buf)) } - expected, err := os.ReadFile(filepath.Join(source, "config.json")) + require.NoError(t, stacked.Close()) + installedAux, err := os.ReadFile(filepath.Join(filepath.Dir(chain[0]), "aux.img")) require.NoError(t, err) - var config MacOSImage - require.NoError(t, json.Unmarshal(expected, &config)) - require.Equal(t, config, *pulled.MacOS) + require.Equal(t, "synthetic aux", string(installedAux)) + var platform MacOSImage + require.NoError(t, json.Unmarshal(macOSFixturePlatform(t), &platform)) + require.Equal(t, platform, *pulled.MacOS) + digest, err := image.Digest() require.NoError(t, err) require.Equal(t, digest.String(), pulled.Digest) @@ -196,20 +235,125 @@ func TestMacOSMachineOCIRoundTrip(t *testing.T) { require.NoError(t, err) require.Equal(t, pulled.Digest, tagged.Digest) require.Equal(t, pulled.MacOS, tagged.MacOS) - tagDisk, err := GetDiskPath(p, tagged.Name, tagged.Digest) + tagChain, err := MacOSDiskChain(p, tagged.Name, tagged.Digest) require.NoError(t, err) - require.Equal(t, disk, tagDisk) + require.Equal(t, chain, tagChain) _, err = manager.TagImage(ctx, ref.Name(), strings.TrimPrefix(server.URL, "http://")+"/other:stable") require.ErrorIs(t, err, ErrInvalidPlatform) _, err = manager.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "linux/arm64"}) require.ErrorIs(t, err, ErrInvalidPlatform) - t.Log("round-trip verified", pulled.Digest) +} + +// TestMacOSMachinePullReusesReadyLayers pulls two images that share a base layer +// and expects the second to clone the first's installed base disk, not unpack it. +func TestMacOSMachinePullReusesReadyLayers(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + base, overlay := macOSFixtureChain(t, work) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + baseLayer := machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": base}) + server := httptest.NewServer(registry.New()) + defer server.Close() + p := paths.New(filepath.Join(work, "hypeman-data")) + mgr, err := NewManager(p, 1, nil) + require.NoError(t, err) + client := mgr.(*manager).ociClient + lookup := client.reusableMachineLayers + var reused []string + client.reusableMachineLayers = func(diffIDs []string) []string { + paths := lookup(diffIDs) + for _, path := range paths { + if path != "" { + reused = append(reused, path) + } + } + return paths + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + pull := func(repo string, top gcr.Layer) []string { + image, err := mutate.AppendLayers(empty.Image, baseLayer, top) + require.NoError(t, err) + cfg, err := image.ConfigFile() + require.NoError(t, err) + cfg.OS, cfg.Architecture = "darwin", "arm64" + cfg.Config.Labels = macOSFixtureMetadata().Labels + image, err = mutate.ConfigFile(image, cfg) + require.NoError(t, err) + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://") + "/" + repo + ":latest") + require.NoError(t, err) + require.NoError(t, remote.Write(ref, image)) + pulled, err := mgr.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "darwin/arm64"}) + require.NoError(t, err) + require.NoError(t, mgr.WaitForReady(ctx, pulled.Name)) + pulled, err = mgr.GetImage(ctx, pulled.Name) + require.NoError(t, err) + chain, err := MacOSDiskChain(p, pulled.Name, pulled.Digest) + require.NoError(t, err) + return chain + } + first := pull("first", machineFixtureLayer(t, work, "first", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) + require.Empty(t, reused, "no ready image has the base layer yet") + + img, err := diskimagekit.Open(base, diskimagekit.WithOpenMode(diskimagekit.ReadOnly)) + require.NoError(t, err) + second := filepath.Join(work, "second.asif") + require.NoError(t, img.CreateLayer(second, diskimagekit.Overlay)) + require.NoError(t, img.Close()) + chain := pull("second", machineFixtureLayer(t, work, "second", map[string]string{"disk.asif": second, "aux.img": aux, "config.json": config})) + require.Equal(t, []string{first[0]}, reused, "the shared base layer is cloned from the first image; the top layer is unpacked") + want, err := fileSHA256(ctx, base) + require.NoError(t, err) + got, err := fileSHA256(ctx, chain[0]) + require.NoError(t, err) + require.Equal(t, want, got) + stacked, err := OpenMacOSDiskChain(chain) + require.NoError(t, err) + require.NoError(t, stacked.Close()) +} + +// TestMacOSMachinePullRejectsBrokenChain pulls an image whose overlay layer does not +// belong to its base, and expects the pull to fail. +func TestMacOSMachinePullRejectsBrokenChain(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + _, overlay := macOSFixtureChain(t, work) + otherBase, _ := macOSFixtureChain(t, t.TempDir()) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + image, err := mutate.AppendLayers(empty.Image, + machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": otherBase}), + machineFixtureLayer(t, work, "top", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) + require.NoError(t, err) + cfg, err := image.ConfigFile() + require.NoError(t, err) + cfg.OS, cfg.Architecture, cfg.Config.Labels = "darwin", "arm64", macOSFixtureMetadata().Labels + image, err = mutate.ConfigFile(image, cfg) + require.NoError(t, err) + server := httptest.NewServer(registry.New()) + defer server.Close() + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://") + "/macos:broken") + require.NoError(t, err) + require.NoError(t, remote.Write(ref, image)) + manager, err := NewManager(paths.New(filepath.Join(work, "hypeman-data")), 1, nil) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + pulled, err := manager.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "darwin/arm64"}) + require.NoError(t, err) + require.Error(t, manager.WaitForReady(ctx, pulled.Name)) + pulled, err = manager.GetImage(ctx, pulled.Name) + require.NoError(t, err) + require.Equal(t, StatusFailed, pulled.Status) } func TestValidateMacOSMachineRejectsBeforeUnpack(t *testing.T) { allowMacOSMachines(t) - valid := map[string]string{MacOSMachineVersionLabel: "1", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "raw", - MacOSMachineDiskLabel: "disk.img", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "platform.json"} + valid := map[string]string{MacOSMachineVersionLabel: "2", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "asif", + MacOSMachineDiskLabel: "disk.asif", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "platform.json"} require.False(t, isMacOSMachine(&containerMetadata{OS: "linux", Architecture: "amd64"})) require.True(t, isMacOSMachine(&containerMetadata{OS: "Darwin", Architecture: "aarch64"})) require.NoError(t, validateMacOSMachine(&containerMetadata{OS: "darwin", Architecture: "arm64", Labels: valid})) @@ -220,9 +364,9 @@ func TestValidateMacOSMachineRejectsBeforeUnpack(t *testing.T) { "missing disk path": func(l map[string]string) { delete(l, MacOSMachineDiskLabel) }, "missing aux path": func(l map[string]string) { delete(l, MacOSMachineAuxLabel) }, "missing platform path": func(l map[string]string) { delete(l, MacOSMachinePlatformLabel) }, - "absolute path": func(l map[string]string) { l[MacOSMachineDiskLabel] = "/disk.img" }, + "absolute path": func(l map[string]string) { l[MacOSMachineDiskLabel] = "/disk.asif" }, "escaping path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "../aux.img" }, - "duplicate path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "./disk.img" }, + "duplicate path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "./disk.asif" }, } { labels := maps.Clone(valid) change(labels) diff --git a/lib/images/macos_stage_test.go b/lib/images/macos_stage_test.go index 5acd8e0ad..a369d7b89 100644 --- a/lib/images/macos_stage_test.go +++ b/lib/images/macos_stage_test.go @@ -14,9 +14,7 @@ func TestStageMachineFileNeverKeepsRegistryMode(t *testing.T) { require.NoError(t, os.WriteFile(src, []byte("synthetic"), 0644)) require.NoError(t, os.Chmod(src, 0644)) dst := filepath.Join(root, "staged.raw") - size, err := stageMachineFile(src, dst) - require.NoError(t, err) - require.EqualValues(t, len("synthetic"), size) + require.NoError(t, stageMachineFile(src, dst)) info, err := os.Stat(dst) require.NoError(t, err) require.Equal(t, os.FileMode(0600), info.Mode().Perm()) diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index ad08493ec..53ab5a80a 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -11,10 +11,12 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/kernel/hypeman/lib/forkvm" ) // machineUnpackChunk is the granularity of hole detection when writing machine files. @@ -28,11 +30,13 @@ var machineZeroChunk = make([]byte, machineUnpackChunk) // lower it. var maxMachineUnpackedBytes int64 = 1 << 40 -// unpackMachineLayers extracts a macOS machine image's layers into targetDir. Unlike +// unpackMachineLayers extracts each of a macOS machine image's layers into its own +// numbered directory under targetDir, since every layer carries a disk. Unlike // the rootfs unpack, it writes files sparsely: zero chunks become holes, so a // mostly empty boot disk occupies only its data. Machine bundles hold only regular // files and directories; any other entry is rejected. Each layer is checked -// against its config diff ID. +// against its config diff ID. A layer below the top that a ready image already has, +// on the same layers beneath it, is cloned from that image instead of unpacked. func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDir string) error { path, err := layout.FromPath(c.cacheDir) if err != nil { @@ -60,15 +64,41 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi if err := os.MkdirAll(targetDir, 0700); err != nil { return fmt.Errorf("create target dir: %w", err) } + diffIDs := make([]string, len(configFile.RootFS.DiffIDs)) + for i, diffID := range configFile.RootFS.DiffIDs { + diffIDs[i] = diffID.String() + } + reuse := make([]string, len(layers)) + if c.reusableMachineLayers != nil { + reuse = c.reusableMachineLayers(diffIDs) + } remaining := maxMachineUnpackedBytes for i, l := range layers { - if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], targetDir, &remaining); err != nil { + dir := filepath.Join(targetDir, strconv.Itoa(i)) + if reuse[i] != "" && cloneMachineLayer(reuse[i], dir, configFile.Config.Labels[MacOSMachineDiskLabel]) { + continue + } + if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], dir, &remaining); err != nil { return fmt.Errorf("unpack machine layer %d: %w", i, err) } } return nil } +// cloneMachineLayer clones a ready image's disk into dir, where unpacking the layer +// would put it, and reports whether it did. A failed clone, as when that image is +// being deleted, leaves dir absent for an unpack. +func cloneMachineLayer(source, dir, diskName string) bool { + target := filepath.Join(dir, diskName) + if err := os.MkdirAll(filepath.Dir(target), 0700); err == nil { + if err = forkvm.CopyRegularFile(source, target); err == nil { + return true + } + } + os.RemoveAll(dir) + return false +} + // unpackMachineLayer extracts one layer, spending its decompressed bytes from the // bundle's remaining budget. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { diff --git a/lib/images/manager.go b/lib/images/manager.go index 6618b3176..8b1f733c3 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -110,6 +110,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } + ociClient.reusableMachineLayers = m.reusableMacOSLayers // Initialize metrics if meter is provided if meter != nil { @@ -522,7 +523,7 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials diskPath := layout.disk // Keep the temporary filesystem beside its final path so finalization stays // atomic even when system/builds and images are on different filesystems. - staged, err := m.stageMacOSOutputs(ctx, ref.Digest(), tempDir, result.Metadata, layout.dir, diskPath+".tmp-"+buildID, buildID) + staged, err := m.stageMacOSOutputs(ctx, ref.Digest(), tempDir, result, layout.dir, diskPath+".tmp-"+buildID, buildID) defer staged.remove() if err != nil { m.updateStatusByDigest(ref, StatusFailed, err, buildID) diff --git a/lib/images/oci.go b/lib/images/oci.go index 3a3ad743f..abb744f6a 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -25,6 +25,9 @@ import ( // ociClient handles OCI image operations without requiring Docker daemon type ociClient struct { cacheDir string + // reusableMachineLayers returns, for each layer of a macOS chain with diffIDs, a + // ready image's disk to clone instead of unpacking it, or "". + reusableMachineLayers func(diffIDs []string) []string } type credentialedManifestInspector struct { diff --git a/lib/images/storage.go b/lib/images/storage.go index f9f9de89c..ab78dac8d 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -18,6 +18,8 @@ type imageMetadata struct { Digest string `json:"digest"` // Always present: sha256:... Platform string `json:"platform,omitempty"` MacOS *MacOSImage `json:"macos,omitempty"` + MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // length of the macOS boot disk chain + MacOSDiskDiffIDs []string `json:"macos_disk_diff_ids,omitempty"` // OCI diff ID of each chain layer; empty for local imports Status string `json:"status"` Error *string `json:"error,omitempty"` Request *CreateImageRequest `json:"request,omitempty"` @@ -154,15 +156,15 @@ func legacyLayout(p *paths.Paths, repository, digestHex string) imageLayout { return withMachineDisk(imageLayout{dir: p.ImageDigestDir(repository, digestHex), metadata: p.ImageMetadata(repository, digestHex), disk: p.ImageDigestPath(repository, digestHex)}) } -// machineDiskFile is a macOS machine image's raw boot disk, whether imported locally -// or pulled. Rootfs images use erofs or ext4 instead. -const machineDiskFile = "rootfs.raw" +// machineDiskFile is the base of a macOS machine image's ASIF boot disk chain, +// whether imported locally or pulled. Rootfs images use erofs or ext4 instead. +const machineDiskFile = "disk.0.asif" // MacOSDiskLayerFile(0) -// withMachineDisk points layout at a machine image's raw disk when the directory has -// one. Rootfs layouts never do, so a stat is enough to pick the file. +// withMachineDisk points layout at a machine image's base disk when the directory +// has one. Rootfs layouts never do, so a stat is enough to pick the file. func withMachineDisk(layout imageLayout) imageLayout { - if raw := filepath.Join(layout.dir, machineDiskFile); pathExists(raw) { - layout.disk = raw + if base := filepath.Join(layout.dir, machineDiskFile); pathExists(base) { + layout.disk = base } return layout } diff --git a/lib/instances/admission.go b/lib/instances/admission.go index cc22a099f..b735f0c83 100644 --- a/lib/instances/admission.go +++ b/lib/instances/admission.go @@ -1,5 +1,13 @@ package instances +import ( + "os" + "path/filepath" + "syscall" + + "github.com/kernel/hypeman/lib/paths" +) + func volumeOverlayReservationBytes(volumes []VolumeAttachment) int64 { var total int64 for _, vol := range volumes { @@ -18,5 +26,26 @@ func storedDiskReservationBytes(stored *StoredMetadata) int64 { if stored == nil { return 0 } + if stored.MacOS != nil { + return macOSBootDiskReservationBytes(stored) + } return requestedDiskReservationBytes(stored.OverlaySize, stored.Volumes) } + +// macOSBootDiskAllowance is the disk reserved for a macOS guest's writes. Its boot +// disk starts as an empty ASIF overlay on layers its image already holds, and grows +// with the guest's writes, with no fixed cap. The allowance matches the Linux +// default overlay size. +const macOSBootDiskAllowance = 10 << 30 + +// macOSBootDiskReservationBytes reserves the allowance, or what the guest's boot +// overlay already holds once it has written more. +func macOSBootDiskReservationBytes(stored *StoredMetadata) int64 { + var allocated int64 + if info, err := os.Stat(filepath.Join(stored.DataDir, paths.MacOSBootDiskFile)); err == nil { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + allocated = stat.Blocks * 512 + } + } + return max(macOSBootDiskAllowance, allocated) +} diff --git a/lib/instances/admission_allocations.go b/lib/instances/admission_allocations.go index 1f2699ac8..9087bff25 100644 --- a/lib/instances/admission_allocations.go +++ b/lib/instances/admission_allocations.go @@ -145,7 +145,7 @@ func (m *manager) allocationFromStoredMetadata(stored *StoredMetadata, active bo Name: stored.Name, Vcpus: stored.Vcpus, MemoryBytes: stored.Size + stored.HotplugSize, - OverlayBytes: stored.OverlaySize, + OverlayBytes: storedDiskReservationBytes(stored) - volumeOverlayBytes, VolumeOverlayBytes: volumeOverlayBytes, NetworkDownloadBps: stored.NetworkBandwidthDownload, NetworkUploadBps: stored.NetworkBandwidthUpload, diff --git a/lib/instances/admission_macos_test.go b/lib/instances/admission_macos_test.go new file mode 100644 index 000000000..a3a8ab1a5 --- /dev/null +++ b/lib/instances/admission_macos_test.go @@ -0,0 +1,23 @@ +package instances + +import ( + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSDiskReservationIsTheAllowanceNotTheImageSize(t *testing.T) { + dir := t.TempDir() + stored := &StoredMetadata{DataDir: dir, OverlaySize: 64 << 30, MacOS: &images.MacOSImage{}} + require.Equal(t, int64(macOSBootDiskAllowance), storedDiskReservationBytes(stored), "a stored image-sized OverlaySize is not reserved") + + require.NoError(t, os.WriteFile(filepath.Join(dir, paths.MacOSBootDiskFile), make([]byte, 1<<20), 0600)) + require.Equal(t, int64(macOSBootDiskAllowance), storedDiskReservationBytes(stored), "a boot overlay within the allowance reserves the allowance") + + linux := &StoredMetadata{OverlaySize: 64 << 30} + require.Equal(t, int64(64<<30), storedDiskReservationBytes(linux)) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..2bae403f4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,6 +99,7 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } + req.Platform = DefaultPlatform(m.macOSOnly, req.Platform) if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } @@ -213,7 +214,7 @@ func (m *manager) createInstance( if overlaySize == 0 { overlaySize = 10 * 1024 * 1024 * 1024 // 10GB default } - // Validate overlay size against max. A macOS boot disk is sized by its image. + // Validate overlay size against max. A macOS guest reserves a fixed allowance instead. if imageInfo.MacOS == nil && overlaySize > m.limits.MaxOverlaySize { return nil, fmt.Errorf("overlay size %d exceeds maximum allowed size %d", overlaySize, m.limits.MaxOverlaySize) } @@ -421,7 +422,7 @@ func (m *manager) createInstance( // 13. Create overlay disk with specified size log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", stored.OverlaySize) - if err := m.createBootDisk(stored, imageInfo); err != nil { + if err := m.createBootDisk(ctx, stored, imageInfo); err != nil { log.ErrorContext(ctx, "failed to create overlay disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create overlay disk: %w", err) } diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index cd605597f..62508c947 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -20,16 +20,19 @@ type createImageResolver interface { func resolveImageForCreate(ctx context.Context, imageManager createImageResolver, imageName, platform string, log *slog.Logger) (*images.Image, error) { if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { img, err := imageManager.GetImage(ctx, imageName) - if err != nil { - return nil, fmt.Errorf("get imported macOS image: %w", err) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return nil, fmt.Errorf("get macOS image: %w", err) } - if img.MacOS == nil { - return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) + if err != nil || img.MacOS == nil { + if img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}); err != nil { + return nil, fmt.Errorf("pull macOS image: %w", err) + } } - if err := validateResolvedImagePlatform(img, platform); err != nil { - return nil, err + img, err = awaitResolvedImage(ctx, imageManager, img, imageName, platform, log) + if err == nil && img.MacOS == nil { + err = fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) } - return img, nil + return img, err } // An empty platform means the host platform: a no-platform run must boot a // host-native guest and never silently emulate, so we resolve the host variant @@ -58,6 +61,12 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver if err != nil { return nil, fmt.Errorf("resolve image %s for platform %s: %w", imageName, resolvePlatform, err) } + return awaitResolvedImage(ctx, imageManager, img, imageName, resolvePlatform, log) +} + +// awaitResolvedImage pins img by digest, waits for it to be ready and checks it +// is for platform. +func awaitResolvedImage(ctx context.Context, imageManager createImageResolver, img *images.Image, imageName, platform string, log *slog.Logger) (*images.Image, error) { pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { return nil, err @@ -66,7 +75,7 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver if err != nil { return nil, err } - if err := validateResolvedImagePlatform(img, resolvePlatform); err != nil { + if err := validateResolvedImagePlatform(img, platform); err != nil { return nil, err } return img, nil diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..fea162437 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "strings" "testing" "github.com/kernel/hypeman/lib/images" @@ -30,6 +31,55 @@ func (f createImageResolverFake) WaitForReady(ctx context.Context, name string) return f.waitReady(ctx, name) } +func TestResolveImageForCreatePullsUncachedMacOSImage(t *testing.T) { + t.Parallel() + + const imageName = "registry.example/macos/app:latest" + digest := "sha256:" + strings.Repeat("a", 64) + pinnedName := "registry.example/macos/app@" + digest + createPlatform := "" + waitName := "" + resolver := createImageResolverFake{ + getImage: func(_ context.Context, name string) (*images.Image, error) { + if name == imageName { + return nil, images.ErrNotFound + } + if name != pinnedName { + t.Fatalf("unexpected image lookup %q", name) + } + return &images.Image{ + Name: imageName, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusReady, MacOS: &images.MacOSImage{}, + }, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{ + Name: req.Name, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusPending, MacOS: &images.MacOSImage{}, + }, nil + }, + waitReady: func(_ context.Context, name string) error { + waitName = name + return nil + }, + } + + img, err := resolveImageForCreate(context.Background(), resolver, imageName, "darwin/arm64", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != "darwin/arm64" { + t.Fatalf("expected Darwin pull, got %q", createPlatform) + } + if waitName != pinnedName { + t.Fatalf("expected readiness wait on %q, got %q", pinnedName, waitName) + } + if img.MacOS == nil || img.Platform != "darwin/arm64" { + t.Fatalf("expected ready macOS image, got %+v", img) + } +} + func TestResolveImageForCreateWithPlatformForcesPlatformResolve(t *testing.T) { t.Parallel() diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9e8e4d901..8c49444a2 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "net" + "os" "path/filepath" "strings" "syscall" "time" + "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" @@ -19,25 +21,22 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/kernel/hypeman/lib/system" ) -// macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. -const macOSAuxStorageFile = "mac-aux.img" +const macOSAuxStorageFile = paths.MacOSAuxStorageFile -// requireMacOSImageName refuses a create on a macOS-only server unless the image -// is an already imported macOS image, so a Linux image is never pulled there. +// requireMacOSImageName checks lookup errors before resolving the reference as +// darwin/arm64. The resolved image is validated by checkMacOSCreate. func (m *manager) requireMacOSImageName(ctx context.Context, name string) error { if !m.macOSOnly { return nil } - cached, err := m.imageManager.GetImage(ctx, name) - if err != nil && !errors.Is(err, images.ErrNotFound) { - return err - } - if err != nil || cached.MacOS == nil { - return fmt.Errorf("%w: macOS-only server requires an already imported macOS image", ErrInvalidRequest) + _, err := m.imageManager.GetImage(ctx, name) + if errors.Is(err, images.ErrNotFound) { + return nil } - return nil + return err } // checkMacOSCreate applies macOS machine defaults to a create from a macOS image. @@ -60,8 +59,8 @@ func (m *manager) checkMacOSCreate(req *CreateInstanceRequest, img *images.Image } config := hypervisor.VMConfig{ BootMode: hypervisor.BootModeMacOS, VCPUs: prepared.Vcpus, MemoryBytes: prepared.Size, - MacOS: &hypervisor.MacOSPlatform{HardwareModelData: img.MacOS.HardwareModel, MachineIdentifierData: img.MacOS.MachineIdentifier, AuxStoragePath: "mac-aux.img"}, - Disks: []hypervisor.DiskConfig{{Path: "disk.img"}}, + MacOS: &hypervisor.MacOSPlatform{HardwareModelData: img.MacOS.HardwareModel, MachineIdentifierData: img.MacOS.MachineIdentifier, AuxStoragePath: macOSAuxStorageFile}, + Disks: []hypervisor.DiskConfig{{Path: paths.MacOSBootDiskFile, Format: hypervisor.DiskFormatASIF, Parents: []string{images.MacOSDiskLayerFile(0)}}}, } if err := validatePlannedVMConfig(starter, hvType, config); err != nil { return err @@ -102,7 +101,7 @@ func prepareMacOSCreate(req *CreateInstanceRequest, img *images.Image, caps hype return fmt.Errorf("%w: macOS requires at least 4 GiB in whole MiB and 2 vCPUs", ErrInvalidRequest) } req.Size, req.Vcpus = size, vcpus - req.OverlaySize = *img.SizeBytes // Reserve the writable boot disk, not a Linux overlay. + req.OverlaySize = macOSBootDiskAllowance // Reserve room for the guest's writes, not the whole disk. req.SkipGuestAgent = req.SkipGuestAgent || !img.MacOS.GuestAgent req.SkipKernelHeaders = true return nil @@ -162,18 +161,80 @@ func (m *manager) checkMacOSStart(stored *StoredMetadata, req StartInstanceReque return nil } -// createBootDisk creates the Linux overlay, or clones a macOS image's boot disk and -// aux storage into the instance. -func (m *manager) createBootDisk(stored *StoredMetadata, img *images.Image) error { +// DefaultPlatform returns the create platform: a macOS-only server defaults an +// unset one to darwin/arm64. +func DefaultPlatform(macOSOnly bool, platform string) string { + if macOSOnly && strings.TrimSpace(platform) == "" { + return "darwin/arm64" + } + return platform +} + +// createBootDisk creates the Linux overlay, or the macOS boot disk: the image's +// ASIF chain and auxiliary storage cloned into the instance, with an empty +// writable overlay on top. +func (m *manager) createBootDisk(ctx context.Context, stored *StoredMetadata, img *images.Image) error { if stored.MacOS == nil { return m.createOverlayDisk(stored.Id, stored.OverlaySize) } - disk, err := images.GetDiskPath(m.paths, img.Name, img.Digest) + chain, err := images.MacOSDiskChain(m.paths, img.Name, img.Digest) + if err != nil { + return err + } + aux := filepath.Join(filepath.Dir(chain[0]), images.MacOSMachineAuxFile) + if err := createMacOSStorage(chain, aux, stored.DataDir); err != nil { + return err + } + stored.MacOSDiskLayers = len(chain) + return m.installMacOSAgentDisk(ctx, stored) +} + +// macOSAgentDiskSource returns the host's current guest agent disk. Tests replace it. +var macOSAgentDiskSource = system.MacOSAgentDisk + +// installMacOSAgentDisk gives a macOS guest that runs the guest agent a copy of the +// host's current guest agent disk. Each guest keeps its own copy, so saved state +// restores with the disk it was saved with; a cold start takes the host's current +// agent. +func (m *manager) installMacOSAgentDisk(ctx context.Context, stored *StoredMetadata) error { + if stored.MacOS == nil || !stored.GuestAgentEnabled() { + return nil + } + source, err := macOSAgentDiskSource(ctx, m.paths) if err != nil { return err } - aux := filepath.Join(filepath.Dir(disk), "aux.img") - return cloneMacOSStorage(disk, aux, m.paths.InstanceMacOSBootDisk(stored.Id), filepath.Join(stored.DataDir, macOSAuxStorageFile)) + disk := m.paths.InstanceMacOSAgentDisk(stored.Id) + staged := disk + ".tmp" + _ = os.Remove(staged) + if err := forkvm.CopyRegularFile(source, staged); err != nil { + return fmt.Errorf("copy macOS agent disk: %w", err) + } + if err := os.Chmod(staged, 0400); err != nil { + _ = os.Remove(staged) + return err + } + return os.Rename(staged, disk) +} + +// macOSDisks returns a macOS guest's disks: its boot disk, then, when it runs the +// guest agent, its read-only guest agent disk. The boot disk must come first; VZ +// does not boot with it second. +func macOSDisks(p *paths.Paths, stored *StoredMetadata) []hypervisor.DiskConfig { + disks := []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk(stored.Id), Format: hypervisor.DiskFormatASIF, Parents: macOSDiskParents(stored)}} + if stored.GuestAgentEnabled() { + disks = append(disks, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk(stored.Id), Readonly: true}) + } + return disks +} + +// macOSDiskParents returns the read-only layers below a macOS guest's boot overlay. +func macOSDiskParents(stored *StoredMetadata) []string { + parents := make([]string, stored.MacOSDiskLayers) + for i := range parents { + parents[i] = filepath.Join(stored.DataDir, images.MacOSDiskLayerFile(i)) + } + return parents } // usesLinuxNetwork reports whether the instance's network is a hypeman TAP @@ -217,7 +278,7 @@ func (m *manager) admitMacOSBoot(ctx context.Context, stored *StoredMetadata) (f func (m *manager) macOSVMConfig(inst *Instance) hypervisor.VMConfig { c := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: inst.Vcpus, MemoryBytes: inst.Size, VsockCID: inst.VsockCID, VsockSocket: inst.VsockSocket, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: inst.MacOS.HardwareModel, MachineIdentifierData: inst.MacOS.MachineIdentifier, AuxStoragePath: filepath.Join(inst.DataDir, macOSAuxStorageFile)}, - Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(inst.Id)}}} + Disks: macOSDisks(m.paths, &inst.StoredMetadata)} if inst.NetworkEnabled { c.Networks = []hypervisor.NetworkConfig{{MAC: inst.MAC, IP: inst.IP, Netmask: macOSGuestNetmask}} } diff --git a/lib/instances/macos_agent_disk_test.go b/lib/instances/macos_agent_disk_test.go new file mode 100644 index 000000000..5b7795ff5 --- /dev/null +++ b/lib/instances/macos_agent_disk_test.go @@ -0,0 +1,88 @@ +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// useMacOSAgentDisk makes the host's agent disk a file holding content. +func useMacOSAgentDisk(t *testing.T, content string) { + t.Helper() + disk := filepath.Join(t.TempDir(), "agent.img") + require.NoError(t, os.WriteFile(disk, []byte(content), 0644)) + prev := macOSAgentDiskSource + macOSAgentDiskSource = func(context.Context, *paths.Paths) (string, error) { return disk, nil } + t.Cleanup(func() { macOSAgentDiskSource = prev }) +} + +func TestMacOSAgentDiskInstallAndRefresh(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + stored := &StoredMetadata{Id: "mac", MacOS: macOSGuestMetadata(2).MacOS} + require.NoError(t, os.MkdirAll(m.paths.InstanceDir(stored.Id), 0700)) + + useMacOSAgentDisk(t, "agent v1") + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) + require.NoFileExists(t, m.paths.InstanceMacOSAgentDisk(stored.Id), "a guest without the agent gets no agent disk") + + stored.MacOS.GuestAgent = true + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) + data, err := os.ReadFile(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, "agent v1", string(data)) + info, err := os.Stat(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, os.FileMode(0400), info.Mode().Perm()) + + useMacOSAgentDisk(t, "agent v2") + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored), "a cold start replaces the guest's copy") + data, err = os.ReadFile(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, "agent v2", string(data)) +} + +func TestMacOSDisksAttachAgentDiskSecond(t *testing.T) { + p := paths.New(t.TempDir()) + stored := &StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOSDiskLayers: 1, MacOS: &images.MacOSImage{}} + require.Equal(t, []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk("mac"), Format: hypervisor.DiskFormatASIF, Parents: []string{filepath.Join(stored.DataDir, images.MacOSDiskLayerFile(0))}}}, macOSDisks(p, stored)) + stored.MacOS.GuestAgent = true + disks := macOSDisks(p, stored) + require.Len(t, disks, 2) + require.Equal(t, p.InstanceMacOSBootDisk("mac"), disks[0].Path, "the boot disk stays first") + require.Equal(t, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk("mac"), Readonly: true}, disks[1]) +} + +func TestMacOSMemorySnapshotRestoresAgentDiskItWasSavedWith(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.MacOS.GuestAgent, meta.SkipGuestAgent = true, false + require.NoError(t, m.saveMetadata(meta)) + agentDisk := m.paths.InstanceMacOSAgentDisk(meta.Id) + require.NoError(t, os.WriteFile(agentDisk, []byte("agent-at-checkpoint"), 0400)) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + ctx := context.Background() + snapshot, err := m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.NoError(t, err) + + require.NoError(t, os.Remove(agentDisk)) + require.NoError(t, os.WriteFile(agentDisk, []byte("agent-after"), 0400)) + _, err = m.RestoreSnapshot(ctx, meta.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStandby}) + require.NoError(t, err) + data, err := os.ReadFile(agentDisk) + require.NoError(t, err) + require.Equal(t, "agent-at-checkpoint", string(data), "saved state restores with the agent disk it was saved with") +} + +func TestMacOSMemorySnapshotRequiresItsAgentDisk(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.MacOS.GuestAgent, meta.SkipGuestAgent = true, false + require.NoError(t, m.saveMetadata(meta)) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + _, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.ErrorIs(t, err, ErrInvalidRequest, "a payload without its agent disk is refused") +} diff --git a/lib/instances/macos_darwin.go b/lib/instances/macos_darwin.go deleted file mode 100644 index 884c794d7..000000000 --- a/lib/instances/macos_darwin.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build darwin - -package instances - -import ( - "os" - - "golang.org/x/sys/unix" -) - -func cloneMacOSStorage(rootDisk, rootAux, disk, aux string) (err error) { - var created []string - defer func() { - if err != nil { - // Only remove files this operation created, never pre-existing storage. - for _, path := range created { - os.Remove(path) - } - } - }() - for _, file := range []struct{ src, dst string }{ - {rootDisk, disk}, - {rootAux, aux}, - } { - if err = unix.Clonefile(file.src, file.dst, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil { - return err - } - created = append(created, file.dst) - if err = os.Chmod(file.dst, 0600); err != nil { - return err - } - } - return nil -} diff --git a/lib/instances/macos_darwin_test.go b/lib/instances/macos_darwin_test.go deleted file mode 100644 index 795a71199..000000000 --- a/lib/instances/macos_darwin_test.go +++ /dev/null @@ -1,72 +0,0 @@ -//go:build darwin - -package instances - -import ( - "errors" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" -) - -func TestMacOSStorageClone(t *testing.T) { - root := t.TempDir() - source := filepath.Join(root, "rootfs.raw") - auxSource := filepath.Join(root, "aux.img") - require.NoError(t, os.WriteFile(source, []byte("template disk"), 0644)) - require.NoError(t, os.WriteFile(auxSource, []byte("template aux"), 0644)) - probe := filepath.Join(root, "probe") - err := unix.Clonefile(source, probe, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY) - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EXDEV) { - t.Skip("test filesystem does not support clonefile") - } - require.NoError(t, err) - require.NoError(t, os.Remove(probe)) - - t.Run("private writable copies", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, cloneMacOSStorage(source, auxSource, disk, aux)) - for path, expected := range map[string]string{disk: "template disk", aux: "template aux"} { - data, err := os.ReadFile(path) - require.NoError(t, err) - require.Equal(t, expected, string(data)) - info, err := os.Stat(path) - require.NoError(t, err) - require.Equal(t, os.FileMode(0600), info.Mode().Perm()) - require.NoError(t, os.WriteFile(path, []byte("instance changes"), 0600)) - } - for path, expected := range map[string]string{source: "template disk", auxSource: "template aux"} { - data, err := os.ReadFile(path) - require.NoError(t, err) - require.Equal(t, expected, string(data)) - } - }) - - t.Run("aux failure rolls back only new disk", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, os.WriteFile(aux, []byte("existing aux"), 0600)) - require.Error(t, cloneMacOSStorage(source, auxSource, disk, aux)) - _, err := os.Stat(disk) - require.ErrorIs(t, err, os.ErrNotExist) - data, err := os.ReadFile(aux) - require.NoError(t, err) - require.Equal(t, "existing aux", string(data)) - }) - - t.Run("existing disk is preserved", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, os.WriteFile(disk, []byte("existing disk"), 0600)) - require.Error(t, cloneMacOSStorage(source, auxSource, disk, aux)) - data, err := os.ReadFile(disk) - require.NoError(t, err) - require.Equal(t, "existing disk", string(data)) - _, err = os.Stat(aux) - require.ErrorIs(t, err, os.ErrNotExist) - }) -} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 7e97f93cd..b21b56a2f 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -230,17 +230,23 @@ func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { if clone.MacOS == nil || !clone.NetworkEnabled { return nil } - used, err := m.storedMACs(clone.Id) + return m.assignMacOSMAC(clone) +} + +// assignMacOSMAC gives a macOS instance a MAC no stored instance uses. Its platform +// is copied first, since a clone's may be shared with its source. +func (m *manager) assignMacOSMAC(stored *StoredMetadata) error { + used, err := m.storedMACs(stored.Id) if err != nil { - return fmt.Errorf("list MACs for macOS clone: %w", err) + return fmt.Errorf("list MACs for macOS instance: %w", err) } mac, err := network.NewUniqueMAC(used) if err != nil { - return fmt.Errorf("allocate macOS clone MAC: %w", err) + return fmt.Errorf("allocate macOS MAC: %w", err) } - macos := *clone.MacOS + macos := *stored.MacOS macos.MAC = mac - clone.MacOS = &macos - clone.MAC = mac + stored.MacOS = &macos + stored.MAC = mac return nil } diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go index 28fb142b8..de9e9397a 100644 --- a/lib/instances/macos_memory_snapshot_test.go +++ b/lib/instances/macos_memory_snapshot_test.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -34,8 +35,10 @@ func macOSMemorySnapshotFixture(t *testing.T) (*manager, *metadata) { meta.MAC, meta.NetworkEnabled = meta.MacOS.MAC, true meta.SkipGuestAgent = true meta.StopGraceful = true + meta.MacOSDiskLayers = 1 require.NoError(t, m.saveMetadata(meta)) require.NoError(t, os.Rename(m.paths.InstanceOverlay(id), m.paths.InstanceMacOSBootDisk(id))) + require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, images.MacOSDiskLayerFile(0)), []byte("image layer"), 0400)) require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, macOSAuxStorageFile), []byte("aux-at-checkpoint"), 0600)) return m, meta } @@ -49,8 +52,10 @@ func writeMacOSMemorySnapshotPayload(t *testing.T, m *manager, guestDir string, VCPUs: stored.Vcpus, MemoryBytes: stored.Size, MacHardwareModelData: base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel), MacMachineIdentifierData: base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier), MacAuxStoragePath: filepath.Join(m.paths.InstanceDir(stored.Id), macOSAuxStorageFile), - Disks: []shimconfig.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(stored.Id)}}, }} + for _, disk := range macOSDisks(m.paths, stored) { + manifest.ShimConfig.Disks = append(manifest.ShimConfig.Disks, shimconfig.DiskConfig{Path: disk.Path, Readonly: disk.Readonly, Parents: disk.Parents}) + } if stored.NetworkEnabled { manifest.ShimConfig.Networks = []shimconfig.NetworkConfig{{MAC: stored.MAC}} if stored.IP != "" { @@ -112,13 +117,13 @@ func TestMacOSMemorySnapshotCaptureRestoreAndIsolation(t *testing.T) { require.Equal(t, current.Phases.Cumulative[phasetracking.PhaseRunning], restored.Phases.Cumulative[phasetracking.PhaseRunning], "same-instance restore must not rewind host phase accounting") require.NotNil(t, restored.ExpiresAt) require.True(t, expiresAt.Equal(*restored.ExpiresAt)) - for name, want := range map[string]string{"boot.raw": "overlay", macOSAuxStorageFile: "aux-at-checkpoint", "snapshots/snapshot-latest/" + shimconfig.SnapshotMachineStateFile: "saved-memory"} { + for name, want := range map[string]string{paths.MacOSBootDiskFile: "overlay", macOSAuxStorageFile: "aux-at-checkpoint", "snapshots/snapshot-latest/" + shimconfig.SnapshotMachineStateFile: "saved-memory"} { data, err := os.ReadFile(filepath.Join(meta.DataDir, name)) require.NoError(t, err) require.Equal(t, want, string(data)) } require.NoError(t, os.WriteFile(m.paths.InstanceMacOSBootDisk(meta.Id), []byte("restored-write"), 0600)) - data, err := os.ReadFile(filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "overlay", string(data), "restoring must not alias the persistent snapshot") @@ -142,6 +147,9 @@ func TestMacOSMemorySnapshotRejectsInvalidPayloadBeforeStorageSwap(t *testing.T) {"hardware", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacHardwareModelData = "different" }}, {"disk", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Disks[0].Path = "/wrong-disk" }}, {"aux", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacAuxStoragePath = "/wrong-aux" }}, + {"extra-disk", func(m *shimconfig.SnapshotManifest) { + m.ShimConfig.Disks = append(m.ShimConfig.Disks, shimconfig.DiskConfig{Path: "/agent.img", Readonly: true}) + }}, {"network", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].MAC = "02:00:00:00:00:02" }}, {"network-address", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].IPv4Address, m.ShimConfig.Networks[0].IPv4Subnet = "192.168.128.2", "192.168.128.0/29" diff --git a/lib/instances/macos_other.go b/lib/instances/macos_other.go deleted file mode 100644 index 581e3ba18..000000000 --- a/lib/instances/macos_other.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !darwin - -package instances - -import "fmt" - -func cloneMacOSStorage(string, string, string, string) error { - return fmt.Errorf("macOS image clones require APFS on macOS") -} diff --git a/lib/instances/macos_storage.go b/lib/instances/macos_storage.go new file mode 100644 index 000000000..21d2b6fa8 --- /dev/null +++ b/lib/instances/macos_storage.go @@ -0,0 +1,62 @@ +package instances + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" +) + +// createMacOSStorage gives a macOS guest its own copy of an image's boot disk chain +// and auxiliary storage in dataDir, and an empty writable overlay above the chain. +// Copies are clones where the filesystem supports them. On failure only files this +// call created are removed. +func createMacOSStorage(chain []string, aux, dataDir string) (err error) { + var created []string + defer func() { + if err != nil { + for _, path := range created { + os.Remove(path) + } + } + }() + copies := make([]string, len(chain)) + for i, layer := range chain { + copies[i] = filepath.Join(dataDir, images.MacOSDiskLayerFile(i)) + if err = copyMacOSStorageFile(layer, copies[i], 0400, &created); err != nil { + return err + } + } + if err = copyMacOSStorageFile(aux, filepath.Join(dataDir, macOSAuxStorageFile), 0600, &created); err != nil { + return err + } + img, err := images.OpenMacOSDiskChain(copies) + if err != nil { + return err + } + defer img.Close() + overlay := filepath.Join(dataDir, paths.MacOSBootDiskFile) + if _, err = os.Lstat(overlay); err == nil { + return fmt.Errorf("macOS boot overlay already exists: %s", overlay) + } + created = append(created, overlay) + if err = img.CreateLayer(overlay, diskimagekit.Overlay); err != nil { + return fmt.Errorf("create macOS boot overlay: %w", err) + } + return os.Chmod(overlay, 0600) +} + +func copyMacOSStorageFile(src, dst string, mode os.FileMode, created *[]string) error { + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("macOS storage file already exists: %s", dst) + } + if err := forkvm.CopyRegularFile(src, dst); err != nil { + return err + } + *created = append(*created, dst) + return os.Chmod(dst, mode) +} diff --git a/lib/instances/macos_storage_test.go b/lib/instances/macos_storage_test.go new file mode 100644 index 000000000..d712f6480 --- /dev/null +++ b/lib/instances/macos_storage_test.go @@ -0,0 +1,100 @@ +package instances + +import ( + "crypto/sha256" + "os" + "path/filepath" + "testing" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// imageChainFixture writes a two-layer ASIF chain and an aux file, as an image has. +func imageChainFixture(t *testing.T) ([]string, string) { + t.Helper() + dir := t.TempDir() + chain := []string{filepath.Join(dir, "disk.0.asif"), filepath.Join(dir, "disk.1.asif")} + img, err := diskimagekit.CreateASIF(chain[0], (16<<20)/512) + require.NoError(t, err) + _, err = img.WriteAt([]byte("base"), 0) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(chain[1], diskimagekit.Overlay)) + _, err = img.WriteAt([]byte("image"), 1<<20) + require.NoError(t, err) + require.NoError(t, img.Close()) + aux := filepath.Join(dir, "aux.img") + require.NoError(t, os.WriteFile(aux, []byte("aux"), 0600)) + return chain, aux +} + +func fileSum(t *testing.T, path string) [32]byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return sha256.Sum256(data) +} + +func TestCreateMacOSStorage(t *testing.T) { + chain, aux := imageChainFixture(t) + before := []([32]byte){fileSum(t, chain[0]), fileSum(t, chain[1])} + dataDir := t.TempDir() + require.NoError(t, createMacOSStorage(chain, aux, dataDir)) + + stored := &StoredMetadata{DataDir: dataDir, MacOSDiskLayers: len(chain)} + parents := macOSDiskParents(stored) + for i, layer := range parents { + require.Equal(t, before[i], fileSum(t, layer), "layer %d is a copy of the image's", i) + info, err := os.Stat(layer) + require.NoError(t, err) + require.Equal(t, os.FileMode(0400), info.Mode().Perm(), "image layers are read-only in the instance") + } + overlay := filepath.Join(dataDir, paths.MacOSBootDiskFile) + info, err := os.Stat(overlay) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.Equal(t, []byte("aux"), must(os.ReadFile(filepath.Join(dataDir, macOSAuxStorageFile)))) + + // The guest's view: the image's data through the overlay, and writes land only in the overlay. + disk, err := images.OpenMacOSDiskChain(parents) + require.NoError(t, err) + require.NoError(t, disk.Append(overlay, diskimagekit.WithOpenMode(diskimagekit.ReadWrite))) + _, err = disk.WriteAt([]byte("guest"), 2<<20) + require.NoError(t, err) + for offset, want := range map[int64]string{0: "base", 1 << 20: "image", 2 << 20: "guest"} { + buf := make([]byte, len(want)) + _, err := disk.ReadAt(buf, offset) + require.NoError(t, err) + require.Equal(t, want, string(buf)) + } + require.NoError(t, disk.Close()) + for i, layer := range parents { + require.Equal(t, before[i], fileSum(t, layer), "writes do not reach layer %d", i) + } + require.Equal(t, before[0], fileSum(t, chain[0])) + require.Equal(t, before[1], fileSum(t, chain[1])) + + require.Error(t, createMacOSStorage(chain, aux, dataDir), "existing storage is never replaced") + require.FileExists(t, overlay) +} + +func TestCreateMacOSStorageRemovesOnlyWhatItCreated(t *testing.T) { + chain, _ := imageChainFixture(t) + dataDir := t.TempDir() + existing := filepath.Join(dataDir, "unrelated") + require.NoError(t, os.WriteFile(existing, []byte("keep"), 0600)) + require.Error(t, createMacOSStorage(chain, filepath.Join(t.TempDir(), "missing-aux.img"), dataDir)) + entries, err := os.ReadDir(dataDir) + require.NoError(t, err) + require.Len(t, entries, 1, "the copied layers are removed") + require.FileExists(t, existing) +} + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} diff --git a/lib/instances/macos_test.go b/lib/instances/macos_test.go index 6bf99636d..9260db061 100644 --- a/lib/instances/macos_test.go +++ b/lib/instances/macos_test.go @@ -28,7 +28,7 @@ func TestMacOSRequestDefaultsAndRejections(t *testing.T) { require.NoError(t, prepareMacOSCreate(&req, testMacImage(), caps)) require.Equal(t, int64(8<<30), req.Size) require.Equal(t, 4, req.Vcpus) - require.Equal(t, int64(64<<30), req.OverlaySize) + require.Equal(t, int64(macOSBootDiskAllowance), req.OverlaySize) require.True(t, req.SkipGuestAgent) for _, r := range []CreateInstanceRequest{ {Env: map[string]string{"X": "Y"}}, {Cmd: []string{"sh"}}, {HotplugSize: 1}, {OverlaySize: 1}, {DiskIOBps: 1}, {Size: 1 << 30}, {Size: (4 << 30) + 1}, {Vcpus: 1}, {Volumes: []VolumeAttachment{{VolumeID: "v"}}}, diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index e120fa38f..a8f4c494b 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -9,12 +9,14 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "time" "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -162,7 +164,9 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St if cfg.VCPUs != stored.Vcpus || cfg.MemoryBytes != stored.Size || cfg.MacHardwareModelData != base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel) || cfg.MacMachineIdentifierData != base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier) { return fmt.Errorf("%w: macOS snapshot configuration does not match its metadata", ErrInvalidRequest) } - if len(cfg.Disks) != 1 || cfg.Disks[0].Readonly || cfg.Disks[0].Path != m.paths.InstanceMacOSBootDisk(stored.Id) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { + if !slices.EqualFunc(cfg.Disks, macOSDisks(m.paths, stored), func(saved shimconfig.DiskConfig, want hypervisor.DiskConfig) bool { + return saved.Path == want.Path && saved.Readonly == want.Readonly && slices.Equal(saved.Parents, want.Parents) + }) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { return fmt.Errorf("%w: macOS snapshot storage configuration does not match its instance", ErrInvalidRequest) } if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.EnableMemoryBalloon || cfg.RequireMemoryBalloon || cfg.EnableRosetta { @@ -185,7 +189,14 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St } else if len(cfg.Networks) != 0 { return fmt.Errorf("%w: macOS snapshot unexpectedly enables networking", ErrInvalidRequest) } - for _, path := range []string{filepath.Join(guestDir, paths.MacOSBootDiskFile), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} { + payload := []string{filepath.Join(guestDir, paths.MacOSBootDiskFile), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} + for i := range stored.MacOSDiskLayers { + payload = append(payload, filepath.Join(guestDir, images.MacOSDiskLayerFile(i))) + } + if stored.GuestAgentEnabled() { + payload = append(payload, filepath.Join(guestDir, paths.MacOSAgentDiskFile)) + } + for _, path := range payload { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { return fmt.Errorf("%w: missing or invalid macOS snapshot payload %s", ErrInvalidRequest, filepath.Base(path)) diff --git a/lib/instances/snapshot_macos_darwin_test.go b/lib/instances/snapshot_macos_darwin_test.go index edf1db8a8..8b3275783 100644 --- a/lib/instances/snapshot_macos_darwin_test.go +++ b/lib/instances/snapshot_macos_darwin_test.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" @@ -30,13 +31,14 @@ func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { source.MAC = source.MacOS.MAC source.MacOS.GuestAgent = true source.StopGraceful = true + source.MacOSDiskLayers = 1 now := time.Now() source.GuestAgentReadyAt = &now source.Phases.Record(phasetracking.PhaseRunning, now.Add(-time.Hour)) source.Phases.Record(phasetracking.PhaseStopped, now) require.NoError(t, vz.NewStarter().ValidateConfig(m.plannedStoredVMConfig(SnapshotKindStopped, source))) require.NoError(t, m.saveSnapshotRecord(&snapshotRecord{Snapshot: Snapshot{Id: snapshotID, Kind: SnapshotKindStopped, SourceInstanceID: source.Id, SourceName: source.Name, SourceHypervisor: source.HypervisorType, CreatedAt: now}, StoredMetadata: source})) - writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{"boot.raw": "snapshot-disk", macOSAuxStorageFile: "snapshot-aux"}) + writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{paths.MacOSBootDiskFile: "snapshot-disk", images.MacOSDiskLayerFile(0): "snapshot-layer", macOSAuxStorageFile: "snapshot-aux"}) first, err := m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: "mac-snapshot-clone-1"}) require.NoError(t, err) @@ -67,7 +69,7 @@ func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { require.NoError(t, err) require.Equal(t, source.MacOS, persisted.StoredMetadata.MacOS) require.NoError(t, os.WriteFile(p.InstanceMacOSBootDisk(first.Id), []byte("clone-write"), 0600)) - data, err := os.ReadFile(filepath.Join(p.SnapshotGuestDir(snapshotID), "boot.raw")) + data, err := os.ReadFile(filepath.Join(p.SnapshotGuestDir(snapshotID), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "snapshot-disk", string(data)) } diff --git a/lib/instances/snapshot_macos_test.go b/lib/instances/snapshot_macos_test.go index 47cad876d..46a393427 100644 --- a/lib/instances/snapshot_macos_test.go +++ b/lib/instances/snapshot_macos_test.go @@ -76,13 +76,13 @@ func writeGuestFiles(t *testing.T, dir string, files map[string]string) { func TestMacOSRestoreSwapsPayloadWithMetadata(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} const id, snap = "mac-restore", "snap-1" - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "after", macOSAuxStorageFile: "aux-after"}) - writeGuestFiles(t, m.paths.SnapshotGuestDir(snap), map[string]string{"boot.raw": "before", macOSAuxStorageFile: "aux-before", "metadata.json": "stale-metadata"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "after", macOSAuxStorageFile: "aux-after"}) + writeGuestFiles(t, m.paths.SnapshotGuestDir(snap), map[string]string{paths.MacOSBootDiskFile: "before", macOSAuxStorageFile: "aux-before", "metadata.json": "stale-metadata"}) expiresAt := time.Now().Add(time.Hour) restored := &metadata{StoredMetadata: StoredMetadata{Id: id, Name: "current-name", ExpiresAt: &expiresAt, Vcpus: 4, Size: 8 << 30}} require.NoError(t, m.replaceMacOSInstanceWithSnapshotPayload(snap, restored)) - data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "before", string(data)) require.NoDirExists(t, m.paths.InstanceRestoreStaging(id), "staging is removed after a successful swap") @@ -96,9 +96,9 @@ func TestMacOSRestoreSwapsPayloadWithMetadata(t *testing.T) { require.True(t, expiresAt.Equal(*got.ExpiresAt)) // A missing snapshot payload fails before the instance directory is retired. - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "current"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "current"}) require.Error(t, m.replaceMacOSInstanceWithSnapshotPayload("missing-snapshot", &metadata{StoredMetadata: StoredMetadata{Id: id}})) - data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "current", string(data), "a failed staging leaves the previous disk untouched") } @@ -110,7 +110,7 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { return m, m.paths.InstanceRestoreStaging(id) } readBoot := func(t *testing.T, m *manager) string { - data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) return string(data) } @@ -118,8 +118,8 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { // Crash while retiring files, before the swap commits: the retired files come back. m, root := newManager(t) writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{macOSAuxStorageFile: "aux-previous"}) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) - writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{"boot.raw": "restored"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) + writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{paths.MacOSBootDiskFile: "restored"}) require.NoError(t, m.recoverMacOSRestoreStaging(id)) require.Equal(t, "previous", readBoot(t, m)) require.FileExists(t, filepath.Join(m.paths.InstanceDir(id), macOSAuxStorageFile)) @@ -128,28 +128,28 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { // Crash while installing a committed swap: the rest of the payload is installed. m, root = newManager(t) writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{macOSAuxStorageFile: "aux-restored"}) - writeGuestFiles(t, filepath.Join(root, "installing"), map[string]string{"boot.raw": "restored"}) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "old", macOSAuxStorageFile: "aux-old"}) + writeGuestFiles(t, filepath.Join(root, "installing"), map[string]string{paths.MacOSBootDiskFile: "restored"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "old", macOSAuxStorageFile: "aux-old"}) require.NoError(t, m.recoverMacOSRestoreStaging(id)) require.Equal(t, "restored", readBoot(t, m)) require.NoDirExists(t, root) // Crash during the staging copy: nothing was retired, so the partial copy is removed. m, root = newManager(t) - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "current"}) - writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{"boot.raw": "partial"}) - writeGuestFiles(t, m.paths.InstanceDir("other"), map[string]string{"boot.raw": "untouched"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "current"}) + writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{paths.MacOSBootDiskFile: "partial"}) + writeGuestFiles(t, m.paths.InstanceDir("other"), map[string]string{paths.MacOSBootDiskFile: "untouched"}) require.NoError(t, m.recoverMacOSRestoreStaging("")) require.Equal(t, "current", readBoot(t, m)) require.NoDirExists(t, root) - require.FileExists(t, filepath.Join(m.paths.InstanceDir("other"), "boot.raw")) + require.FileExists(t, filepath.Join(m.paths.InstanceDir("other"), paths.MacOSBootDiskFile)) } func TestMacOSRestoreStagingIsDeletedWithInstance(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} const id = "mac-deleted" root := m.paths.InstanceRestoreStaging(id) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) require.Equal(t, m.paths.InstanceDir(id), filepath.Dir(root), "staging lives inside the instance directory") require.NoError(t, os.RemoveAll(m.paths.InstanceDir(id))) @@ -164,14 +164,14 @@ func TestMacOSRestoreRecoveryKeepsBackupWhenStagingIsUnreadable(t *testing.T) { const id = "mac-unknown" m := &manager{paths: paths.New(t.TempDir())} root := m.paths.InstanceRestoreStaging(id) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) // An unreadable staging directory makes the swap's progress unknown. require.NoError(t, os.Chmod(root, 0)) t.Cleanup(func() { _ = os.Chmod(root, 0755) }) require.Error(t, m.recoverMacOSRestoreStaging(id)) require.NoError(t, os.Chmod(root, 0755)) - data, err := os.ReadFile(filepath.Join(root, "previous", "boot.raw")) + data, err := os.ReadFile(filepath.Join(root, "previous", paths.MacOSBootDiskFile)) require.NoError(t, err, "the parked backup must survive an inconclusive check") require.Equal(t, "previous", string(data)) } diff --git a/lib/instances/start.go b/lib/instances/start.go index 898d6a166..af1f5f3c1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -97,6 +97,12 @@ func (m *manager) startInstance( }() } + // A cold start takes the host's current guest agent. + if err := m.installMacOSAgentDisk(ctx, stored); err != nil { + log.ErrorContext(ctx, "failed to refresh macOS agent disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("refresh macOS agent disk: %w", err) + } + // 3. Get image info (needed for buildHypervisorConfig). imageInfo, err := m.resolveBootImage(ctx, stored) if err != nil { diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 1ee166fc2..d26307166 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -26,7 +26,8 @@ const ( // {dataDir}/guests/{instance-id}/ // metadata.json # Instance metadata // overlay.raw # Configurable sparse overlay disk (default 10GB) -// boot.raw # macOS guests: complete boot disk, instead of overlay.raw +// boot.asif # macOS guests: writable ASIF overlay, instead of overlay.raw +// disk.{n}.asif # macOS guests: read-only ASIF layers below boot.asif, base first // config.ext4 # Read-only config disk (generated) // ch.sock # Hypervisor API socket (abbreviated name for SUN_LEN limit) // logs/ diff --git a/lib/instances/types.go b/lib/instances/types.go index bd8989885..f42ff6089 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -94,6 +94,9 @@ type StoredMetadata struct { // Read-only; echoed on the instance API. Platform string MacOS *images.MacOSImage `json:"MacOS,omitempty"` // Non-nil for experimental local macOS disk guests. + // MacOSDiskLayers is the number of read-only image layers below a macOS guest's + // boot overlay, cloned into its data directory. + MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // IdentityRekeyPending marks a macOS clone whose SSH host keys still come from its // source. The next start rekeys them and clears the flag. IdentityRekeyPending bool `json:"identity_rekey_pending,omitempty"` diff --git a/lib/instances/vm_storage_path_test.go b/lib/instances/vm_storage_path_test.go index 62fd284e5..659b7f41b 100644 --- a/lib/instances/vm_storage_path_test.go +++ b/lib/instances/vm_storage_path_test.go @@ -10,6 +10,6 @@ import ( func TestVMStoragePathOnlyForMacOSGuests(t *testing.T) { require.Empty(t, (&StoredMetadata{DataDir: "/data/guests/i1"}).VMStoragePath()) mac := &StoredMetadata{DataDir: "/data/guests/i1", MacOS: &images.MacOSImage{}} - require.Equal(t, "/data/guests/i1/boot.raw", mac.VMStoragePath()) + require.Equal(t, "/data/guests/i1/boot.asif", mac.VMStoragePath()) require.Empty(t, (&StoredMetadata{MacOS: &images.MacOSImage{}}).VMStoragePath(), "no data dir, no path") } diff --git a/lib/machineimage/layer_test.go b/lib/machineimage/layer_test.go new file mode 100644 index 000000000..5ad9f65b1 --- /dev/null +++ b/lib/machineimage/layer_test.go @@ -0,0 +1,65 @@ +package machineimage + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { + parent := t.TempDir() + disk := filepath.Join(parent, "disk.asif") + data := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(data[1<<20:], "data") + require.NoError(t, os.WriteFile(disk, data, 0600)) + + layer, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}, {name: "config.json", data: []byte("{}")}}) + require.NoError(t, err) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) + for range 2 { + body, err := layer.Compressed() + require.NoError(t, err) + sum := sha256.New() + n, err := io.Copy(sum, body) + require.NoError(t, err) + require.NoError(t, body.Close()) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + } + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing is spooled beside the disk") +} + +func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { + disk := filepath.Join(t.TempDir(), "disk.asif") + require.NoError(t, os.WriteFile(disk, make([]byte, 8<<20), 0600)) + layer, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}}) + require.NoError(t, err) + before := runtime.NumGoroutine() + body, err := layer.Compressed() + require.NoError(t, err) + _, err = io.CopyN(io.Discard, body, 16) + require.NoError(t, err) + require.NoError(t, body.Close()) + // Polled directly: require.Eventually runs its condition in another goroutine. + deadline := time.Now().Add(5 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), before, "the layer generator exits when the upload is abandoned") +} diff --git a/lib/machineimage/publisher.go b/lib/machineimage/publisher.go new file mode 100644 index 000000000..c8bbd6081 --- /dev/null +++ b/lib/machineimage/publisher.go @@ -0,0 +1,425 @@ +// Package machineimage publishes macOS machine images to hypeman's registry, from +// stopped Tart VMs or other ASIF disk chains. +package machineimage + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "slices" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/registrytoken" +) + +// Storage is a closed macOS boot disk and platform to publish: the read-only ASIF +// layers below its overlay, base first, the overlay, and its auxiliary storage. A +// standalone base disk is an Overlay with no Layers. LayerDigests, when known, are +// the sha256 of each layer file; otherwise the publisher hashes them. +type Storage struct { + Layers []string + LayerDigests []string + Overlay string + Aux string + Platform images.MacOSImage +} + +// Publication is a verified publication: a digest-pinned reference. +type Publication struct{ Reference, Digest string } + +// validDigest reports whether s is a canonical sha256 digest. +func validDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +// validate checks storage before publication: the platform is valid, the files +// are distinct nonempty regular files, and the overlay stacks on its layers as ASIF. +func (s Storage) validate() error { + if err := s.Platform.Validate(); err != nil { + return err + } + if len(s.LayerDigests) != 0 && len(s.LayerDigests) != len(s.Layers) { + return fmt.Errorf("machine storage has %d layer digests for %d layers", len(s.LayerDigests), len(s.Layers)) + } + if err := images.CheckMachineFiles(append([]string{s.Overlay, s.Aux}, s.Layers...)...); err != nil { + return err + } + disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), s.Layers...), s.Overlay)) + if err != nil { + return fmt.Errorf("machine disk: %w", err) + } + return disk.Close() +} + +// Publisher targets only the configured internal registry and a build-scoped +// repository. go-containerregistry hashes and compresses each layer from a +// reopenable tar stream, so nothing is written to disk, skips blobs the registry +// already holds, and commits the manifest last. The registry stores each upload on +// disk and commits it only if it matches its digest. +// No credentials, redirect destinations or arbitrary output repositories come from the source. +type Publisher struct { + RegistryURL string + // Token is a registry token from MintToken. Without one, Tokens mints a token + // for each publication. + Token string + Tokens *registrytoken.Generator + // NewMachineIdentifier returns a fresh VZ machine identifier. Every published + // image gets its own identifier and MAC, so it never runs with its source's. + NewMachineIdentifier func() ([]byte, error) +} + +// Base is a pinned base image in the publisher's registry. A machine published +// over it reuses its layers, so only the new overlay is uploaded. +type Base struct { + Repo string // builds/{id} repository holding the base manifest and layers + Digest string // canonical sha256 digest of the base manifest +} + +func (b Base) validate() error { + if !strings.HasPrefix(b.Repo, "builds/") || strings.Contains(b.Repo, "..") || len(b.Repo) > 256 { + return fmt.Errorf("machine base repository must be under builds/") + } + if !validDigest(b.Digest) { + return fmt.Errorf("machine base digest must be a canonical sha256 digest") + } + return nil +} + +// Publish commits storage as a machine image in builds/{id}, with a fresh machine +// identifier and MAC. With a base, the storage's layers must be the base's, by the +// disk digests the base records, and the base's registry layers are reused; +// without one, every layer is uploaded. +func (p *Publisher) Publish(ctx context.Context, id string, storage Storage, base *Base) (Publication, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return Publication{}, fmt.Errorf("bounded publication required") + } + if !validName(id) || (p.Token == "" && p.Tokens == nil) || p.NewMachineIdentifier == nil { + return Publication{}, fmt.Errorf("scoped machine publisher required") + } + if base != nil { + if err := base.validate(); err != nil { + return Publication{}, err + } + } + registry, err := parseRegistryURL(p.RegistryURL) + if err != nil { + return Publication{}, err + } + if err := storage.validate(); err != nil { + return Publication{}, err + } + if storage.Platform.MachineIdentifier, err = p.NewMachineIdentifier(); err != nil { + return Publication{}, fmt.Errorf("new machine identifier: %w", err) + } + if storage.Platform.MAC, err = randomMAC(); err != nil { + return Publication{}, err + } + repo := "builds/" + id + token := p.Token + if token == "" { + if token, err = p.Tokens.GenerateToken(id, publishAccess(id, base), time.Until(deadline)); err != nil { + return Publication{}, err + } + } + transport := &http.Transport{} + if defaults, ok := http.DefaultTransport.(*http.Transport); ok { + transport = defaults.Clone() + } + transport.Proxy = nil + defer transport.CloseIdleConnections() + options := []remote.Option{remote.WithContext(ctx), remote.WithAuth(&authn.Bearer{Token: token}), remote.WithTransport(refuseRedirects{transport})} + var nameOptions []name.Option + if registry.Scheme == "http" { + nameOptions = append(nameOptions, name.Insecure) + } + img, err := machineImage(ctx, registry.Host, storage, base, nameOptions, options) + if err != nil { + return Publication{}, err + } + target, err := name.NewTag(registry.Host+"/"+repo+":latest", nameOptions...) + if err != nil { + return Publication{}, err + } + if err = remote.Write(target, img, options...); err != nil { + return Publication{}, fmt.Errorf("write machine image: %w", err) + } + digest, err := img.Digest() + if err != nil { + return Publication{}, err + } + return Publication{Reference: registry.Host + "/" + repo + "@" + digest.String(), Digest: digest.String()}, nil +} + +// MintToken returns a registry token that can publish the image name, over base +// when it is set, for ttl. A publisher given it needs no registry secret. +func MintToken(tokens *registrytoken.Generator, name string, base *Base, ttl time.Duration) (string, error) { + if !validName(name) { + return "", fmt.Errorf("invalid machine image name %q", name) + } + if base != nil { + if err := base.validate(); err != nil { + return "", err + } + } + return tokens.GenerateToken(name, publishAccess(name, base), ttl) +} + +// publishAccess is the registry access a publication needs: push on its own +// builds/{name} repository and pull on its base's. +func publishAccess(name string, base *Base) []registrytoken.RepoPermission { + access := []registrytoken.RepoPermission{{Repo: "builds/" + name, Scope: "push"}} + if base != nil { + access = append(access, registrytoken.RepoPermission{Repo: base.Repo, Scope: "pull"}) + } + return access +} + +func validName(name string) bool { + return name != "" && len(name) <= 128 && strings.Trim(name, "abcdefghijklmnopqrstuvwxyz0123456789-") == "" +} + +// randomMAC returns a random locally administered unicast MAC address. +func randomMAC() (string, error) { + mac := make([]byte, 6) + if _, err := rand.Read(mac); err != nil { + return "", err + } + mac[0] = mac[0]&^0x01 | 0x02 + return net.HardwareAddr(mac).String(), nil +} + +// refuseRedirects fails a redirected registry request rather than follow it, so +// the token never reaches another host. +type refuseRedirects struct{ http.RoundTripper } + +func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { + response, err := t.RoundTripper.RoundTrip(req) + if err == nil && response.StatusCode >= 300 && response.StatusCode < 400 { + response.Body.Close() + return nil, fmt.Errorf("registry redirect refused") + } + return response, err +} + +// ParseBase returns ref, a digest-pinned image in the registry at registryURL, as a +// publication base. +func ParseBase(registryURL, ref string) (*Base, error) { + registry, err := parseRegistryURL(registryURL) + if err != nil { + return nil, err + } + parsed, err := images.ParseNormalizedRef(ref) + if err != nil || !parsed.IsDigest() { + return nil, fmt.Errorf("machine base must be pinned by digest") + } + repo, ok := strings.CutPrefix(parsed.Repository(), registry.Host+"/") + if !ok { + return nil, fmt.Errorf("machine base %s is not in the registry %s", ref, registry.Host) + } + base := &Base{Repo: repo, Digest: parsed.Digest()} + return base, base.validate() +} + +// parseRegistryURL parses a registry address, defaulting to HTTPS. Plaintext is +// allowed only on loopback. +func parseRegistryURL(address string) (*url.URL, error) { + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return nil, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return nil, fmt.Errorf("plaintext registry must be loopback") + } + return registry, nil +} + +// machineImage returns the machine image for storage: the base's layers, from the +// registry or the storage's own layer files, then a top layer with the overlay, +// auxiliary storage and platform. Its labels record every layer's disk digest. +func machineImage(ctx context.Context, host string, storage Storage, base *Base, nameOptions []name.Option, options []remote.Option) (v1.Image, error) { + layerDigests := storage.LayerDigests + if len(layerDigests) == 0 { + for _, path := range storage.Layers { + digest, err := fileDigest(ctx, path) + if err != nil { + return nil, err + } + layerDigests = append(layerDigests, digest) + } + } + img := empty.Image + if base != nil { + ref, err := name.NewDigest(host+"/"+base.Repo+"@"+base.Digest, nameOptions...) + if err != nil { + return nil, err + } + if img, err = remote.Image(ref, options...); err != nil { + return nil, fmt.Errorf("base image: %w", err) + } + config, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" { + return nil, fmt.Errorf("base is not a darwin/arm64 machine image") + } + for label, want := range images.MacOSMachineLabels() { + if config.Config.Labels[label] != want { + return nil, fmt.Errorf("base is not an ASIF machine image: %s=%q", label, config.Config.Labels[label]) + } + } + if layers := len(config.RootFS.DiffIDs); layers != len(storage.Layers) { + return nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", layers, len(storage.Layers)) + } + recorded := config.Config.Labels[images.MacOSMachineDiskDigestsLabel] + if recorded == "" { + return nil, fmt.Errorf("machine base records no disk digests; publish it again without a base first") + } + if !slices.Equal(strings.Split(recorded, ","), layerDigests) { + return nil, fmt.Errorf("machine base's disks are not the layers the disk stacks on: base records %q", recorded) + } + } else { + for _, path := range storage.Layers { + layer, err := machineLayer(ctx, []machineTarEntry{{name: images.MacOSMachineDiskFile, path: path}}) + if err != nil { + return nil, err + } + if img, err = mutate.AppendLayers(img, layer); err != nil { + return nil, err + } + } + } + overlayDigest, err := fileDigest(ctx, storage.Overlay) + if err != nil { + return nil, err + } + platform, err := json.Marshal(storage.Platform) + if err != nil { + return nil, err + } + top, err := machineLayer(ctx, []machineTarEntry{{name: images.MacOSMachineDiskFile, path: storage.Overlay}, {name: images.MacOSMachineAuxFile, path: storage.Aux}, {name: images.MacOSMachinePlatformFile, data: platform}}) + if err != nil { + return nil, err + } + if img, err = mutate.AppendLayers(img, top); err != nil { + return nil, err + } + config, err := img.ConfigFile() + if err != nil { + return nil, err + } + labels := images.MacOSMachineLabels() + labels[images.MacOSMachineDiskDigestsLabel] = strings.Join(append(slices.Clone(layerDigests), overlayDigest), ",") + config.OS, config.Architecture, config.Config.Labels = "darwin", "arm64", labels + if img, err = mutate.ConfigFile(img, config); err != nil { + return nil, err + } + return mutate.ConfigMediaType(mutate.MediaType(img, types.OCIManifestSchema1), types.OCIConfigJSON), nil +} + +// fileDigest returns the sha256 digest of a file. +func fileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, contextReader{ctx, file}); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil +} + +// machineTarEntry is one file in a machine layer, from a path or in memory. +type machineTarEntry struct { + name string + path string + data []byte +} + +// machineLayer returns a gzipped layer of entries, regenerated from the files +// each time it is read. +func machineLayer(ctx context.Context, entries []machineTarEntry) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(writeMachineTar(ctx, entries, writer)) }() + return reader, nil + }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) +} + +// writeMachineTar writes a deterministic tar of entries: fixed modes and no +// timestamps, so regenerating it yields the bytes that were hashed. +func writeMachineTar(ctx context.Context, entries []machineTarEntry, w io.Writer) error { + archive := tar.NewWriter(w) + for _, entry := range entries { + if entry.path == "" { + if err := archive.WriteHeader(&tar.Header{Name: entry.name, Mode: 0600, Size: int64(len(entry.data)), Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := archive.Write(entry.data); err != nil { + return err + } + continue + } + source, err := os.Open(entry.path) + if err != nil { + return err + } + info, err := source.Stat() + if err == nil { + err = archive.WriteHeader(&tar.Header{Name: entry.name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) + } + if err == nil { + _, err = io.CopyN(archive, contextReader{ctx, source}, info.Size()) + } + closeErr := source.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + } + return archive.Close() +} + +// contextReader stops a long read when ctx is done. +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} diff --git a/lib/machineimage/publisher_test.go b/lib/machineimage/publisher_test.go new file mode 100644 index 000000000..1526f4462 --- /dev/null +++ b/lib/machineimage/publisher_test.go @@ -0,0 +1,172 @@ +package machineimage_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/machineimage" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/kernel/hypeman/lib/registrytoken" + "github.com/stretchr/testify/require" +) + +type machineImportStub struct { + images.Manager + imported chan string +} + +func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, digest string) (*images.Image, error) { + m.imported <- repo + return &images.Image{Status: images.StatusReady, Digest: digest}, nil +} + +func testMachineIdentifier() ([]byte, error) { return []byte{9}, nil } + +func machinePublisherPlatform() images.MacOSImage { + return images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} +} + +// machinePublisherStorage writes a standalone ASIF disk and aux file, as a base +// bundle has. +func machinePublisherStorage(t *testing.T) machineimage.Storage { + t.Helper() + root := t.TempDir() + storage := machineimage.Storage{Overlay: filepath.Join(root, "disk.asif"), Aux: filepath.Join(root, "aux.img"), Platform: machinePublisherPlatform()} + img, e := diskimagekit.CreateASIF(storage.Overlay, (4<<20)/512) + require.NoError(t, e) + _, e = img.WriteAt([]byte("synthetic disk"), 0) + require.NoError(t, e) + require.NoError(t, img.Close()) + require.NoError(t, os.WriteFile(storage.Aux, []byte("synthetic aux"), 0600)) + return storage +} + +func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { + // Tiny storage is intentional: this proves auth/streaming/commit ordering, not bootability. + imports := &machineImportStub{imported: make(chan string, 1)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads atomic.Int32 + handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/blobs/uploads/") { + uploads.Add(1) + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + require.Equal(t, int32(2), uploads.Load(), "manifest must be last") + } + reg.Handler().ServeHTTP(w, r) + })) + server := httptest.NewServer(handler) + defer server.Close() + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic registry secret"), NewMachineIdentifier: testMachineIdentifier} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + result, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) + require.NoError(t, e) + require.Contains(t, result.Reference, "/builds/qa-build@sha256:") + select { + case repo := <-imports.imported: + require.Equal(t, "builds/qa-build", repo) + case <-ctx.Done(): + t.Fatal("manifest did not reach production conversion boundary") + } +} + +func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing.T) { + var visited atomic.Bool + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic secret"), NewMachineIdentifier: testMachineIdentifier} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) + require.Error(t, e) + require.False(t, visited.Load()) +} + +func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { + imports := &machineImportStub{imported: make(chan string, 2)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads atomic.Int32 + server := httptest.NewServer(middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/blobs/uploads/") { + uploads.Add(1) + } + reg.Handler().ServeHTTP(w, r) + }))) + defer server.Close() + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic registry secret"), NewMachineIdentifier: testMachineIdentifier} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + baseStorage := machinePublisherStorage(t) + base, e := publisher.Publish(ctx, "qa-base", baseStorage, nil) + require.NoError(t, e) + require.Equal(t, int32(2), uploads.Load(), "config and one layer") + + // A build stacks a writable overlay on the base disk. + img, e := diskimagekit.Open(baseStorage.Overlay) + require.NoError(t, e) + derived := machineimage.Storage{Layers: []string{baseStorage.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: filepath.Join(t.TempDir(), "aux.img"), Platform: machinePublisherPlatform()} + require.NoError(t, img.CreateLayer(derived.Overlay, diskimagekit.Overlay)) + _, e = img.WriteAt([]byte("build output"), 1<<20) + require.NoError(t, e) + require.NoError(t, img.Close()) + require.NoError(t, os.WriteFile(derived.Aux, []byte("build aux"), 0600)) + derived.Platform.MachineIdentifier = []byte{3} + + uploads.Store(0) + machineBase := &machineimage.Base{Repo: "builds/qa-base", Digest: base.Digest} + result, e := publisher.Publish(ctx, "qa-derived", derived, machineBase) + require.NoError(t, e) + require.Equal(t, int32(2), uploads.Load(), "only the config and the overlay layer are uploaded") + require.NotEqual(t, base.Digest, result.Digest) + + derived.Layers = nil + _, e = publisher.Publish(ctx, "qa-derived", derived, machineBase) + require.ErrorContains(t, e, "machine base has 1 layers but the disk stacks on 0") + + // The same number of layers, but stacked on a different base disk. + other := machinePublisherStorage(t) + img, e = diskimagekit.Open(other.Overlay) + require.NoError(t, e) + mismatched := machineimage.Storage{Layers: []string{other.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: derived.Aux, Platform: machinePublisherPlatform()} + require.NoError(t, img.CreateLayer(mismatched.Overlay, diskimagekit.Overlay)) + require.NoError(t, img.Close()) + _, e = publisher.Publish(ctx, "qa-mismatched", mismatched, machineBase) + require.ErrorContains(t, e, "not the layers the disk stacks on") +} + +func TestMintedTokenPublishesOnlyItsImage(t *testing.T) { + imports := &machineImportStub{imported: make(chan string, 2)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + server := httptest.NewServer(middleware.JwtAuth("synthetic registry secret")(reg.Handler())) + defer server.Close() + token, e := machineimage.MintToken(registrytoken.NewGenerator("synthetic registry secret"), "qa-minted", nil, time.Hour) + require.NoError(t, e) + publisher := &machineimage.Publisher{RegistryURL: server.URL, Token: token, NewMachineIdentifier: testMachineIdentifier} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + _, e = publisher.Publish(ctx, "qa-minted", machinePublisherStorage(t), nil) + require.NoError(t, e) + _, e = publisher.Publish(ctx, "qa-other", machinePublisherStorage(t), nil) + require.Error(t, e, "the token cannot publish another image") +} diff --git a/lib/machineimage/tart.go b/lib/machineimage/tart.go new file mode 100644 index 000000000..4deabe484 --- /dev/null +++ b/lib/machineimage/tart.go @@ -0,0 +1,128 @@ +package machineimage + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/images" +) + +// Tart's OCI media types and annotations for stacked images. +const ( + tartDiskMediaType = "application/vnd.cirruslabs.tart.disk.v2" + tartOverlayMediaType = "application/vnd.cirruslabs.tart.disk.asif.overlay.v1" + tartFileDigestAnnotation = "org.cirruslabs.tart.disk-file-content-digest" +) + +type tartConfig struct { + OS string `json:"os"` + Arch string `json:"arch"` + ECID []byte `json:"ecid"` + HardwareModel []byte `json:"hardwareModel"` + CPUCount uint `json:"cpuCount"` + MemorySize uint64 `json:"memorySize"` + MACAddress string `json:"macAddress"` + DiskFormat string `json:"diskFormat"` +} + +type tartManifest struct { + Layers []struct { + MediaType string `json:"mediaType"` + Annotations map[string]string `json:"annotations"` + } `json:"layers"` +} + +// ReadTartVM returns the storage of a stopped Tart macOS VM directory, for +// publishing as a machine image. A standalone VM's disk.img must be ASIF and +// becomes a one-layer image. A stacked VM's read-only layers, base first, come +// from Tart's content store, read from its manifest.json, with its writable +// overlay.asif on top; the content store is under TART_HOME, or two levels above +// the VM directory by default. The platform keeps Tart's identity; the publisher +// gives each image a fresh one. +func ReadTartVM(dir string, guestAgent bool) (Storage, error) { + var config tartConfig + data, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + return Storage{}, err + } + if err := json.Unmarshal(data, &config); err != nil { + return Storage{}, fmt.Errorf("tart config: %w", err) + } + if config.OS != "darwin" || config.Arch != "arm64" { + return Storage{}, fmt.Errorf("tart VM is %s/%s, want darwin/arm64", config.OS, config.Arch) + } + if _, err := os.Lstat(filepath.Join(dir, "state.vzvmsave")); err == nil { + return Storage{}, fmt.Errorf("tart VM is suspended; shut it down from the guest first") + } + storage := Storage{ + Aux: filepath.Join(dir, "nvram.bin"), + Platform: images.MacOSImage{ + HardwareModel: config.HardwareModel, + MachineIdentifier: config.ECID, + MAC: config.MACAddress, + CPUs: config.CPUCount, + Memory: config.MemorySize, + GuestAgent: guestAgent, + }, + } + manifest := filepath.Join(dir, "manifest.json") + overlay := filepath.Join(dir, "overlay.asif") + if _, err := os.Lstat(overlay); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return Storage{}, err + } + if config.DiskFormat != "asif" { + return Storage{}, fmt.Errorf("tart VM disk format is %q, want asif", config.DiskFormat) + } + storage.Overlay = filepath.Join(dir, "disk.img") + return storage, nil + } + storage.Overlay = overlay + home := os.Getenv("TART_HOME") + if home == "" { + home = filepath.Dir(filepath.Dir(dir)) + } + storage.Layers, storage.LayerDigests, err = tartStackLayers(manifest, filepath.Join(home, "cache", "content")) + return storage, err +} + +// tartStackLayers returns the content-store paths and file digests of a stacked +// VM's read-only layers, base first: the whole-file digest on the first chunk of +// the base and of each overlay. Tart names content files by that digest. +func tartStackLayers(path, contentDir string) ([]string, []string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, err + } + var manifest tartManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, nil, fmt.Errorf("tart manifest: %w", err) + } + var layers, digests []string + for i, layer := range manifest.Layers { + if layer.MediaType != tartDiskMediaType && layer.MediaType != tartOverlayMediaType { + continue + } + digest, ok := layer.Annotations[tartFileDigestAnnotation] + if !ok { + continue + } + hex, found := strings.CutPrefix(digest, "sha256:") + if !found || !validDigest(digest) { + return nil, nil, fmt.Errorf("tart manifest layer %d has an invalid disk digest", i) + } + if len(layers) == 0 && layer.MediaType != tartDiskMediaType { + return nil, nil, fmt.Errorf("tart manifest disk layers must start with the base") + } + layers = append(layers, filepath.Join(contentDir, "sha256", hex)) + digests = append(digests, digest) + } + if len(layers) == 0 { + return nil, nil, fmt.Errorf("tart manifest names no stacked disk files") + } + return layers, digests, nil +} diff --git a/lib/machineimage/tart_test.go b/lib/machineimage/tart_test.go new file mode 100644 index 000000000..40a10e55d --- /dev/null +++ b/lib/machineimage/tart_test.go @@ -0,0 +1,115 @@ +package machineimage + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/stretchr/testify/require" +) + +// tartTestVM writes a Tart VM directory under home/vms/name with config.json and +// nvram.bin, and returns its path. +func tartTestVM(t *testing.T, home, name string, config map[string]any) string { + t.Helper() + dir := filepath.Join(home, "vms", name) + require.NoError(t, os.MkdirAll(dir, 0700)) + base := map[string]any{"version": 1, "os": "darwin", "arch": "arm64", "ecid": []byte{2}, "hardwareModel": []byte{1}, + "cpuCountMin": 4, "cpuCount": 4, "memorySizeMin": 8 << 30, "memorySize": 8 << 30, "macAddress": "02:00:00:00:00:01", "diskFormat": "asif"} + for k, v := range config { + base[k] = v + } + data, err := json.Marshal(base) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), data, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "nvram.bin"), []byte("nvram"), 0600)) + return dir +} + +func TestReadTartVMStandalone(t *testing.T) { + dir := tartTestVM(t, t.TempDir(), "base", nil) + img, err := diskimagekit.CreateASIF(filepath.Join(dir, "disk.img"), (4<<20)/512) + require.NoError(t, err) + require.NoError(t, img.Close()) + + storage, err := ReadTartVM(dir, true) + require.NoError(t, err) + require.Empty(t, storage.Layers) + require.Equal(t, filepath.Join(dir, "disk.img"), storage.Overlay) + require.Equal(t, filepath.Join(dir, "nvram.bin"), storage.Aux) + require.Equal(t, []byte{1}, storage.Platform.HardwareModel) + require.Equal(t, []byte{2}, storage.Platform.MachineIdentifier) + require.Equal(t, uint(4), storage.Platform.CPUs) + require.True(t, storage.Platform.GuestAgent) + require.NoError(t, storage.validate()) + + raw := tartTestVM(t, t.TempDir(), "raw", map[string]any{"diskFormat": "raw"}) + _, err = ReadTartVM(raw, false) + require.ErrorContains(t, err, "want asif") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "state.vzvmsave"), []byte("x"), 0600)) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "suspended") + + linux := tartTestVM(t, t.TempDir(), "linux", map[string]any{"os": "linux"}) + _, err = ReadTartVM(linux, false) + require.ErrorContains(t, err, "want darwin/arm64") +} + +func TestReadTartVMStacked(t *testing.T) { + home := t.TempDir() + content := filepath.Join(home, "cache", "content", "sha256") + require.NoError(t, os.MkdirAll(content, 0700)) + // A base and one immutable overlay in the content store, named by digest. + digest := func(name string) string { + sum := sha256.Sum256([]byte(name)) + return hex.EncodeToString(sum[:]) + } + basePath, layerPath := filepath.Join(content, digest("base")), filepath.Join(content, digest("layer")) + img, err := diskimagekit.CreateASIF(basePath, (4<<20)/512) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(layerPath, diskimagekit.Overlay)) + + dir := tartTestVM(t, home, "build", nil) + require.NoError(t, img.CreateLayer(filepath.Join(dir, "overlay.asif"), diskimagekit.Overlay)) + require.NoError(t, img.Close()) + disk := func(media, digest string) map[string]any { + layer := map[string]any{"mediaType": media} + if digest != "" { + layer["annotations"] = map[string]string{tartFileDigestAnnotation: "sha256:" + digest} + } + return layer + } + writeManifest := func(layers ...map[string]any) { + all := append([]map[string]any{{"mediaType": "application/vnd.cirruslabs.tart.config.v1"}}, layers...) + all = append(all, map[string]any{"mediaType": "application/vnd.cirruslabs.tart.nvram.v1"}) + data, err := json.Marshal(map[string]any{"layers": all}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0600)) + } + writeManifest(disk(tartDiskMediaType, digest("base")), disk(tartDiskMediaType, ""), disk(tartOverlayMediaType, digest("layer"))) + + storage, err := ReadTartVM(dir, false) + require.NoError(t, err) + require.Equal(t, []string{basePath, layerPath}, storage.Layers) + require.Equal(t, filepath.Join(dir, "overlay.asif"), storage.Overlay) + require.NoError(t, storage.validate()) + + writeManifest(disk(tartOverlayMediaType, digest("layer"))) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "must start with the base") + + writeManifest(disk(tartDiskMediaType, "")) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "names no stacked disk files") + + // A layer missing from the content store fails validation. + writeManifest(disk(tartDiskMediaType, digest("missing"))) + storage, err = ReadTartVM(dir, false) + require.NoError(t, err) + require.Error(t, storage.validate()) +} diff --git a/lib/paths/paths.go b/lib/paths/paths.go index f8fba4cb3..00d76cb4c 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -251,14 +251,32 @@ func (p *Paths) InstanceOverlay(id string) string { return filepath.Join(p.InstanceDir(id), "overlay.raw") } -// MacOSBootDiskFile is a macOS guest's complete raw boot disk, cloned from its image. -const MacOSBootDiskFile = "boot.raw" +// MacOSBootDiskFile is a macOS guest's writable ASIF overlay, the top of its boot +// disk chain. The read-only layers below it are cloned from its image. +const MacOSBootDiskFile = "boot.asif" // InstanceMacOSBootDisk returns the path to a macOS guest's boot disk. func (p *Paths) InstanceMacOSBootDisk(id string) string { return filepath.Join(p.InstanceDir(id), MacOSBootDiskFile) } +// MacOSAuxStorageFile is a macOS guest's copy of its image's auxiliary storage. +const MacOSAuxStorageFile = "mac-aux.img" + +// MacOSAgentDiskFile is a macOS guest's copy of the read-only guest agent disk. +const MacOSAgentDiskFile = "agent.img" + +// InstanceMacOSAgentDisk returns the path to a macOS guest's guest agent disk. +func (p *Paths) InstanceMacOSAgentDisk(id string) string { + return filepath.Join(p.InstanceDir(id), MacOSAgentDiskFile) +} + +// SystemMacOSAgentDisk returns the path to the guest agent disk built for an agent +// version, from which macOS guests take their copies. +func (p *Paths) SystemMacOSAgentDisk(version string) string { + return filepath.Join(p.DataDir(), "system", "macos-agent", version+".img") +} + // RestoreStagingDir holds a macOS restore in progress inside its instance directory, // so deleting the instance also deletes any staging it left behind. const RestoreStagingDir = "restore-staging" diff --git a/lib/registry/blob_store.go b/lib/registry/blob_store.go index e7d92fb02..5d3c2fb45 100644 --- a/lib/registry/blob_store.go +++ b/lib/registry/blob_store.go @@ -104,6 +104,16 @@ func (s *BlobStore) Put(_ context.Context, repo string, h v1.Hash, r io.ReadClos return nil } +// install moves a verified file into the store as the blob h, keeping an existing +// copy. The file must be on the store's filesystem. +func (s *BlobStore) install(path string, h v1.Hash) error { + target := s.blobPath(h.String()) + if _, err := os.Stat(target); err == nil { + return os.Remove(path) + } + return os.Rename(path, target) +} + func (s *BlobStore) Delete(_ context.Context, repo string, h v1.Hash) error { path := s.blobPath(h.String()) err := os.Remove(path) diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 77249b920..e17183cee 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -7,11 +7,15 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" "os" + "path/filepath" "regexp" + "slices" + "strconv" "strings" "sync" @@ -28,6 +32,7 @@ type Registry struct { paths *paths.Paths imageManager images.Manager blobStore *BlobStore + uploads *chunkedUploads handler http.Handler // pushedTags tracks pushed (repo, reference) tags to their manifest @@ -38,7 +43,9 @@ type Registry struct { // the manifest blobs the registry is still serving on tag/digest pulls. // Cleared with the process; tags do not survive restart. pushedTagsMu sync.RWMutex - pushedTags map[string]string + // manifestReposMu serializes recording which repositories a manifest was pushed to. + manifestReposMu sync.Mutex + pushedTags map[string]string } // manifestPutPattern matches PUT requests to /v2/{name}/manifests/{reference} @@ -52,6 +59,11 @@ func New(p *paths.Paths, imgManager images.Manager) (*Registry, error) { return nil, err } + uploads, err := newChunkedUploads(filepath.Join(p.SystemOCICache(), "uploads"), blobStore) + if err != nil { + return nil, fmt.Errorf("create upload directory: %w", err) + } + // Create registry with custom blob handler regHandler := registry.New( registry.WithBlobHandler(blobStore), @@ -61,6 +73,7 @@ func New(p *paths.Paths, imgManager images.Manager) (*Registry, error) { paths: p, imageManager: imgManager, blobStore: blobStore, + uploads: uploads, handler: regHandler, pushedTags: make(map[string]string), } @@ -111,6 +124,9 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if r.uploads.serve(w, req) || r.serveManifestByDigest(w, req) { + return + } // Intercept manifest PUT requests to store in blob store and trigger conversion if req.Method == http.MethodPut { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) @@ -142,6 +158,9 @@ func (r *Registry) Handler() http.Handler { r.handler.ServeHTTP(wrapper, req) if wrapper.statusCode == http.StatusCreated { + if err := r.recordManifestRepo(pathRepo, digest); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to record manifest repository: %v\n", err) + } // Root every pushed manifest: the Docker manifest blob is // served by the registry but not reachable from the OCI // layout (which stores the converted manifest), so GC @@ -161,6 +180,109 @@ func (r *Registry) Handler() http.Handler { }) } +// serveManifestByDigest answers a manifest read by digest, from the upstream +// registry or, when it does not know the manifest, from the blob store, where every +// pushed manifest is kept, if it was pushed to the requested repository. The +// upstream registry holds manifests in memory, so a manifest pushed before a +// restart is otherwise unknown by digest. Other requests are not handled. +func (r *Registry) serveManifestByDigest(w http.ResponseWriter, req *http.Request) bool { + matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) + if matches == nil || (req.Method != http.MethodGet && req.Method != http.MethodHead) { + return false + } + h, err := v1.NewHash(matches[2]) + if err != nil { + return false + } + // Manifests are small, so the upstream answer is buffered to fall back on a 404. + upstream := &bufferedResponse{header: make(http.Header), status: http.StatusOK} + r.handler.ServeHTTP(upstream, req) + if upstream.status != http.StatusNotFound || !r.manifestPushedTo(matches[1], h) || !r.serveStoredManifest(w, req, h) { + for key, values := range upstream.header { + w.Header()[key] = values + } + w.WriteHeader(upstream.status) + w.Write(upstream.body.Bytes()) + } + return true +} + +// bufferedResponse records a response for serveManifestByDigest. +type bufferedResponse struct { + header http.Header + status int + body bytes.Buffer +} + +func (b *bufferedResponse) Header() http.Header { return b.header } +func (b *bufferedResponse) WriteHeader(status int) { b.status = status } +func (b *bufferedResponse) Write(data []byte) (int, error) { return b.body.Write(data) } + +// serveStoredManifest writes the manifest h from the blob store and reports +// whether it had one. +func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request, h v1.Hash) bool { + path := r.blobStore.blobPath(h.String()) + if info, err := os.Stat(path); err != nil || info.Size() > maxStoredManifestBytes { + return false + } + data, err := os.ReadFile(path) + var manifest struct { + MediaType string `json:"mediaType"` + } + if err != nil || json.Unmarshal(data, &manifest) != nil || manifest.MediaType == "" || computeDigest(data) != h.String() { + return false + } + w.Header().Set("Content-Type", manifest.MediaType) + w.Header().Set("Docker-Content-Digest", h.String()) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(http.StatusOK) + if req.Method == http.MethodGet { + w.Write(data) + } + return true +} + +// manifestReposPath lists, one per line, the repositories a manifest was pushed to. +func (r *Registry) manifestReposPath(h v1.Hash) string { + return filepath.Join(r.paths.SystemOCICache(), "manifest-repos", h.Hex) +} + +// recordManifestRepo records that the manifest digest was pushed to repo. +func (r *Registry) recordManifestRepo(repo, digest string) error { + h, err := v1.NewHash(digest) + if err != nil { + return err + } + r.manifestReposMu.Lock() + defer r.manifestReposMu.Unlock() + if r.manifestPushedTo(repo, h) { + return nil + } + path := r.manifestReposPath(h) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return err + } + _, err = file.WriteString(repo + "\n") + if closeErr := file.Close(); err == nil { + err = closeErr + } + return err +} + +// manifestPushedTo reports whether the manifest h was pushed to repo. +func (r *Registry) manifestPushedTo(repo string, h v1.Hash) bool { + data, err := os.ReadFile(r.manifestReposPath(h)) + return err == nil && slices.Contains(strings.Split(string(data), "\n"), repo) +} + +// maxStoredManifestBytes bounds a stored blob served as a manifest, so a layer +// requested as one is not read into memory. +const maxStoredManifestBytes = 4 << 20 + // storeManifestBlob stores a manifest in the blob store by its digest. func (r *Registry) storeManifestBlob(digest string, data []byte) error { digestHex := strings.TrimPrefix(digest, "sha256:") diff --git a/lib/registry/uploads.go b/lib/registry/uploads.go new file mode 100644 index 000000000..ff8b76715 --- /dev/null +++ b/lib/registry/uploads.go @@ -0,0 +1,251 @@ +package registry + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "sync" + "sync/atomic" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// uploadPattern matches /v2/{name}/blobs/uploads/ and /v2/{name}/blobs/uploads/{id}. +var uploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/([0-9a-f]*)$`) + +// Bounds on chunked uploads, which otherwise hold host disk until they end or the +// process restarts. One upload fits the largest machine image layer. +const ( + maxUploadBytes = 128 << 30 + maxOpenUploadBytes = 256 << 30 + maxOpenUploads = 64 + uploadIdleTimeout = time.Hour +) + +// chunkedUploads stores chunked blob uploads (POST, PATCH..., PUT ?digest=) in +// files beside the blob store, hashing as they arrive. The upstream registry keeps +// them in memory, which a multi-gigabyte layer cannot afford. Monolithic uploads +// (POST ?digest=) already stream and are left to it. +type chunkedUploads struct { + dir string + store *BlobStore + mu sync.Mutex + open map[string]*chunkedUpload + openBytes atomic.Int64 +} + +type chunkedUpload struct { + mu sync.Mutex + file *os.File + hash hash.Hash + size int64 + lastUsed time.Time +} + +// newChunkedUploads removes uploads left by an earlier process: their sessions +// were in memory, so no client can finish them. +func newChunkedUploads(dir string, store *BlobStore) (*chunkedUploads, error) { + if err := os.RemoveAll(dir); err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + return &chunkedUploads{dir: dir, store: store, open: make(map[string]*chunkedUpload)}, nil +} + +// serve handles req if it is part of a chunked upload and reports whether it did. +func (u *chunkedUploads) serve(w http.ResponseWriter, req *http.Request) bool { + match := uploadPattern.FindStringSubmatch(req.URL.Path) + if match == nil { + return false + } + repo, id := match[1], match[2] + switch { + case req.Method == http.MethodPost && id == "" && req.URL.Query().Get("digest") == "": + u.removeIdle() + u.start(w, repo) + case id != "" && (req.Method == http.MethodPatch || req.Method == http.MethodPut || req.Method == http.MethodDelete): + upload := u.lock(id) + if upload == nil { + http.Error(w, "blob upload unknown", http.StatusNotFound) + return true + } + defer upload.mu.Unlock() + upload.lastUsed = time.Now() + switch req.Method { + case http.MethodPatch: + u.append(w, req, repo, id, upload) + case http.MethodPut: + u.commit(w, req, repo, id, upload) + default: + u.remove(id, upload) + w.WriteHeader(http.StatusNoContent) + } + default: + return false + } + return true +} + +// lock returns the open upload id, locked, or nil. An upload ended while this +// request waited for it is not returned. +func (u *chunkedUploads) lock(id string) *chunkedUpload { + u.mu.Lock() + upload := u.open[id] + u.mu.Unlock() + if upload == nil { + return nil + } + upload.mu.Lock() + u.mu.Lock() + current := u.open[id] == upload + u.mu.Unlock() + if !current { + upload.mu.Unlock() + return nil + } + return upload +} + +// removeIdle ends uploads unused for uploadIdleTimeout. An upload busy with a +// request is not idle. +func (u *chunkedUploads) removeIdle() { + u.mu.Lock() + idle := make(map[string]*chunkedUpload) + for id, upload := range u.open { + if upload.mu.TryLock() { + if time.Since(upload.lastUsed) > uploadIdleTimeout { + idle[id] = upload + } else { + upload.mu.Unlock() + } + } + } + u.mu.Unlock() + for id, upload := range idle { + u.remove(id, upload) + upload.mu.Unlock() + } +} + +func (u *chunkedUploads) start(w http.ResponseWriter, repo string) { + u.mu.Lock() + full := len(u.open) >= maxOpenUploads + u.mu.Unlock() + if full { + http.Error(w, "too many open blob uploads", http.StatusTooManyRequests) + return + } + var raw [16]byte + rand.Read(raw[:]) + id := hex.EncodeToString(raw[:]) + file, err := os.OpenFile(filepath.Join(u.dir, id), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + http.Error(w, "start upload failed", http.StatusInternalServerError) + return + } + u.mu.Lock() + u.open[id] = &chunkedUpload{file: file, hash: sha256.New(), lastUsed: time.Now()} + u.mu.Unlock() + w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) + w.Header().Set("Range", "0-0") + w.WriteHeader(http.StatusAccepted) +} + +// write appends body to upload, within the size bounds. It reports whether the +// body fit; an upload that does not fit is ended. +func (u *chunkedUploads) write(w http.ResponseWriter, id string, upload *chunkedUpload, body io.Reader) bool { + allowed := min(maxUploadBytes-upload.size, maxOpenUploadBytes-u.openBytes.Load()) + n, err := io.Copy(io.MultiWriter(upload.file, upload.hash), io.LimitReader(body, max(allowed, 0)+1)) + upload.size += n + u.openBytes.Add(n) + switch { + case err != nil: + u.remove(id, upload) + http.Error(w, "write upload failed", http.StatusInternalServerError) + return false + case n > allowed: + u.remove(id, upload) + http.Error(w, "blob upload too large", http.StatusRequestEntityTooLarge) + return false + } + return true +} + +// append writes a chunk. A Content-Range must start where the upload ends. +func (u *chunkedUploads) append(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { + if r := req.Header.Get("Content-Range"); r != "" { + var start, end int64 + if _, err := fmt.Sscanf(r, "%d-%d", &start, &end); err != nil || start != upload.size { + http.Error(w, "content range does not continue the upload", http.StatusRequestedRangeNotSatisfiable) + return + } + } + if !u.write(w, id, upload, req.Body) { + return + } + w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) + if upload.size > 0 { + w.Header().Set("Range", "0-"+strconv.FormatInt(upload.size-1, 10)) + } + w.WriteHeader(http.StatusAccepted) +} + +// commit writes the final chunk and installs the blob if it matches the digest. +// The upload ends either way. +func (u *chunkedUploads) commit(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { + want, err := v1.NewHash(req.URL.Query().Get("digest")) + if err != nil { + u.remove(id, upload) + http.Error(w, "invalid digest", http.StatusBadRequest) + return + } + if !u.write(w, id, upload, req.Body) { + return + } + defer u.remove(id, upload) + if err = upload.file.Sync(); err == nil { + err = upload.file.Close() + } + if err != nil { + http.Error(w, "write upload failed", http.StatusInternalServerError) + return + } + if got := "sha256:" + hex.EncodeToString(upload.hash.Sum(nil)); got != want.String() { + http.Error(w, "digest mismatch", http.StatusBadRequest) + return + } + if err = u.store.install(upload.file.Name(), want); err != nil { + http.Error(w, "store blob failed", http.StatusInternalServerError) + return + } + w.Header().Set("Docker-Content-Digest", want.String()) + w.Header().Set("Location", "/v2/"+repo+"/blobs/"+want.String()) + w.WriteHeader(http.StatusCreated) +} + +// remove ends an upload and deletes its file. Ending an ended upload does nothing. +func (u *chunkedUploads) remove(id string, upload *chunkedUpload) { + u.mu.Lock() + current := u.open[id] == upload + if current { + delete(u.open, id) + } + u.mu.Unlock() + if !current { + return + } + u.openBytes.Add(-upload.size) + upload.file.Close() + os.Remove(upload.file.Name()) +} diff --git a/lib/registry/uploads_test.go b/lib/registry/uploads_test.go new file mode 100644 index 000000000..3b31ae739 --- /dev/null +++ b/lib/registry/uploads_test.go @@ -0,0 +1,175 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "math/rand/v2" + "net/http" + "net/http/httptest" + "os" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// randomBlob is an incompressible blob generated on demand, so a large blob needs +// no memory to push. +type randomBlob struct { + size int64 + digest v1.Hash +} + +func newRandomBlob(t *testing.T, size int64) *randomBlob { + b := &randomBlob{size: size} + hash := sha256.New() + body, err := b.Compressed() + require.NoError(t, err) + _, err = io.Copy(hash, body) + require.NoError(t, err) + b.digest = v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(hash.Sum(nil))} + return b +} + +func (b *randomBlob) Compressed() (io.ReadCloser, error) { + return io.NopCloser(io.LimitReader(rand.NewChaCha8([32]byte{}), b.size)), nil +} +func (b *randomBlob) Uncompressed() (io.ReadCloser, error) { return b.Compressed() } +func (b *randomBlob) Digest() (v1.Hash, error) { return b.digest, nil } +func (b *randomBlob) DiffID() (v1.Hash, error) { return b.digest, nil } +func (b *randomBlob) Size() (int64, error) { return b.size, nil } +func (b *randomBlob) MediaType() (types.MediaType, error) { return types.OCILayer, nil } + +// TestChunkedUploadStreamsToDisk pushes a layer far larger than the heap bound +// through go-containerregistry's chunked upload, and checks it lands in the blob +// store without being held in memory. +func TestChunkedUploadStreamsToDisk(t *testing.T) { + const size = 768 << 20 + p := paths.New(t.TempDir()) + reg, err := New(p, nil) + require.NoError(t, err) + server := httptest.NewServer(reg.Handler()) + defer server.Close() + + layer := newRandomBlob(t, size) + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://")+"/cache/big:latest", name.Insecure) + require.NoError(t, err) + + runtime.GC() + var base runtime.MemStats + runtime.ReadMemStats(&base) + var peak atomic.Uint64 + done := make(chan struct{}) + go func() { + for { + var m runtime.MemStats + runtime.ReadMemStats(&m) + if m.HeapInuse > peak.Load() { + peak.Store(m.HeapInuse) + } + select { + case <-done: + return + case <-time.After(20 * time.Millisecond): + } + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + err = remote.WriteLayer(ref.Context(), layer, remote.WithContext(ctx)) + close(done) + require.NoError(t, err) + + stored, err := reg.blobStore.Stat(ctx, "cache/big", layer.digest) + require.NoError(t, err) + require.Equal(t, int64(size), stored) + grew := int64(peak.Load()) - int64(base.HeapInuse) + t.Logf("pushed %d MiB; heap grew %d MiB", size>>20, grew>>20) + require.Less(t, grew, int64(64<<20), "the upload is not buffered in memory") +} + +func TestChunkedUploadRejectsBadChunks(t *testing.T) { + reg, err := New(paths.New(t.TempDir()), nil) + require.NoError(t, err) + server := httptest.NewServer(reg.Handler()) + defer server.Close() + do := func(method, url, contentRange string, body string) *http.Response { + req, err := http.NewRequest(method, url, strings.NewReader(body)) + require.NoError(t, err) + if contentRange != "" { + req.Header.Set("Content-Range", contentRange) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + resp.Body.Close() + return resp + } + start := func() string { + resp := do(http.MethodPost, server.URL+"/v2/repo/blobs/uploads/", "", "") + require.Equal(t, http.StatusAccepted, resp.StatusCode) + return server.URL + resp.Header.Get("Location") + } + sum := sha256.Sum256([]byte("abcdef")) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + upload := start() + require.Equal(t, http.StatusAccepted, do(http.MethodPatch, upload, "0-2", "abc").StatusCode) + require.Equal(t, http.StatusRequestedRangeNotSatisfiable, do(http.MethodPatch, upload, "0-2", "def").StatusCode, "a chunk must continue the upload") + require.Equal(t, http.StatusAccepted, do(http.MethodPatch, upload, "3-5", "def").StatusCode) + require.Equal(t, http.StatusCreated, do(http.MethodPut, upload+"?digest="+digest, "", "").StatusCode) + require.Equal(t, http.StatusNotFound, do(http.MethodPatch, upload, "", "x").StatusCode, "a committed upload is closed") + + other := sha256.Sum256([]byte("xyz")) + upload = start() + require.Equal(t, http.StatusBadRequest, do(http.MethodPut, upload+"?digest=sha256:"+hex.EncodeToString(other[:]), "", "xyZ").StatusCode) + _, err = reg.blobStore.Stat(context.Background(), "repo", v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(other[:])}) + require.ErrorIs(t, err, ErrNotFound, "a mismatched upload is not stored") + entries, err := os.ReadDir(reg.uploads.dir) + require.NoError(t, err) + require.Empty(t, entries, "ended uploads leave no files") +} + +func TestManifestByDigestSurvivesRestart(t *testing.T) { + p := paths.New(t.TempDir()) + serve := func() *httptest.Server { + reg, err := New(p, nil) + require.NoError(t, err) + return httptest.NewServer(reg.Handler()) + } + img, err := random.Image(64, 1) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + + first := serve() + ref, err := name.ParseReference(strings.TrimPrefix(first.URL, "http://")+"/cache/app:latest", name.Insecure) + require.NoError(t, err) + require.NoError(t, remote.Write(ref, img)) + first.Close() + + second := serve() + defer second.Close() + byDigest, err := name.NewDigest(strings.TrimPrefix(second.URL, "http://")+"/cache/app@"+digest.String(), name.Insecure) + require.NoError(t, err) + pulled, err := remote.Image(byDigest) + require.NoError(t, err) + got, err := pulled.Digest() + require.NoError(t, err) + require.Equal(t, digest, got) + + otherRepo, err := name.NewDigest(strings.TrimPrefix(second.URL, "http://")+"/cache/other@"+digest.String(), name.Insecure) + require.NoError(t, err) + _, err = remote.Image(otherRepo) + require.Error(t, err, "a stored manifest is served only from the repository it was pushed to") +} diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go new file mode 100644 index 000000000..40b16d0e9 --- /dev/null +++ b/lib/registrypush/macos_test.go @@ -0,0 +1,48 @@ +package registrypush + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache/testutil" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { + host, _ := newTestRegistry(t) + img, err := random.Image(256, 1) + require.NoError(t, err) + config, err := img.ConfigFile() + require.NoError(t, err) + config.OS, config.Architecture = "darwin", "arm64" + config.Config.Labels = map[string]string{ + images.MacOSMachineVersionLabel: "2", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "asif", images.MacOSMachineDiskLabel: "disk.asif", + images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + } + img, err = mutate.ConfigFile(img, config) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + img = mutate.ConfigMediaType(img, types.OCIConfigJSON) + p := paths.New(t.TempDir()) + digest, err := testutil.WriteImage(p, img) + require.NoError(t, err) + target := host + "/releases/macos:canary" + result, err := PushFromCache(context.Background(), p, digest, target, nil, Options{Insecure: true}) + require.NoError(t, err) + require.Equal(t, digest, result.Digest) + ref, err := name.ParseReference(host+"/releases/macos@"+digest, name.Insecure) + require.NoError(t, err) + published, err := remote.Image(ref) + require.NoError(t, err) + actual, err := published.ConfigFile() + require.NoError(t, err) + require.Equal(t, config, actual) +} diff --git a/lib/registrytoken/token.go b/lib/registrytoken/token.go new file mode 100644 index 000000000..df69a9790 --- /dev/null +++ b/lib/registrytoken/token.go @@ -0,0 +1,221 @@ +// Package registrytoken mints and checks the JWTs that scope access to hypeman's +// registry, for builder VMs and image publication. +package registrytoken + +import ( + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// RepoPermission defines access permissions for a specific repository. +// This enables fine-grained control where different repos can have different scopes. +type RepoPermission struct { + // Repo is the repository path (e.g., "builds/abc123", "cache/tenant-x", "cache/global/node") + Repo string `json:"repo"` + + // Scope is the access scope for this repo: "pull" for read-only, "push" for read+write + Scope string `json:"scope"` +} + +// Claims contains the claims for a scoped registry access token. +// These tokens are issued to builder VMs to grant limited access to specific repositories. +type Claims struct { + jwt.RegisteredClaims + + // BuildID is the build job identifier for audit purposes + BuildID string `json:"build_id"` + + // RepoAccess defines per-repository access permissions (new two-tier format) + // If present, this takes precedence over the legacy Repositories/Scope fields + RepoAccess []RepoPermission `json:"repo_access,omitempty"` + + // Repositories is the list of allowed repository paths (legacy format, kept for backward compat) + // Deprecated: Use RepoAccess for new tokens + Repositories []string `json:"repos,omitempty"` + + // Scope is the access scope (legacy format, kept for backward compat) + // Deprecated: Use RepoAccess for new tokens + Scope string `json:"scope,omitempty"` +} + +// Generator creates scoped registry access tokens +type Generator struct { + secret []byte +} + +// NewGenerator creates a new token generator with the given secret +func NewGenerator(secret string) *Generator { + return &Generator{ + secret: []byte(secret), + } +} + +// GeneratePushToken creates a short-lived token granting push access to specific repositories. +// The token expires after the specified duration (typically matching the build timeout). +// Deprecated: Use GenerateToken for new code that needs per-repo scopes. +func (g *Generator) GeneratePushToken(buildID string, repos []string, ttl time.Duration) (string, error) { + if buildID == "" { + return "", fmt.Errorf("build ID is required") + } + if len(repos) == 0 { + return "", fmt.Errorf("at least one repository is required") + } + + now := time.Now() + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Subject: "builder-" + buildID, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), + Issuer: "hypeman", + }, + BuildID: buildID, + Repositories: repos, + Scope: "push", + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString(g.secret) +} + +// GenerateToken creates a short-lived token with per-repository access permissions. +// This supports the two-tier cache model where different repos can have different scopes. +// For example: pull on cache/global/*, push on cache/{tenant} +func (g *Generator) GenerateToken(buildID string, repoAccess []RepoPermission, ttl time.Duration) (string, error) { + if buildID == "" { + return "", fmt.Errorf("build ID is required") + } + if len(repoAccess) == 0 { + return "", fmt.Errorf("at least one repository permission is required") + } + + now := time.Now() + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Subject: "builder-" + buildID, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), + Issuer: "hypeman", + }, + BuildID: buildID, + RepoAccess: repoAccess, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString(g.secret) +} + +// ValidateToken parses and validates a registry token, returning the claims if valid. +func (g *Generator) ValidateToken(tokenString string) (*Claims, error) { + token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) { + // Validate signing method + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return g.secret, nil + }) + + if err != nil { + return nil, fmt.Errorf("parse token: %w", err) + } + + claims, ok := token.Claims.(*Claims) + if !ok || !token.Valid { + return nil, fmt.Errorf("invalid token") + } + + return claims, nil +} + +// IsRepositoryAllowed checks if the given repository path is allowed by the token claims. +func (c *Claims) IsRepositoryAllowed(repo string) bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Repo == repo { + return true + } + } + return false + } + + // Fall back to legacy format + for _, allowed := range c.Repositories { + if allowed == repo { + return true + } + } + return false +} + +// GetRepoScope returns the scope for a specific repository. +// Returns empty string if the repository is not allowed. +func (c *Claims) GetRepoScope(repo string) string { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Repo == repo { + return perm.Scope + } + } + return "" + } + + // Fall back to legacy format - all repos have the same scope + for _, allowed := range c.Repositories { + if allowed == repo { + return c.Scope + } + } + return "" +} + +// IsPushAllowedForRepo returns true if the token grants push (write) access to the given repo. +func (c *Claims) IsPushAllowedForRepo(repo string) bool { + scope := c.GetRepoScope(repo) + return scope == "push" +} + +// IsPullAllowedForRepo returns true if the token grants pull (read) access to the given repo. +// Push scope also implicitly grants pull access. +func (c *Claims) IsPullAllowedForRepo(repo string) bool { + scope := c.GetRepoScope(repo) + return scope == "push" || scope == "pull" +} + +// IsPushAllowed returns true if the token grants push (write) access to any repo. +// Deprecated: Use IsPushAllowedForRepo for per-repo scope checking. +func (c *Claims) IsPushAllowed() bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Scope == "push" { + return true + } + } + return false + } + + // Fall back to legacy format + return c.Scope == "push" +} + +// IsPullAllowed returns true if the token grants pull (read) access. +// Push tokens also implicitly grant pull access. +// Deprecated: Use IsPullAllowedForRepo for per-repo scope checking. +func (c *Claims) IsPullAllowed() bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Scope == "push" || perm.Scope == "pull" { + return true + } + } + return false + } + + // Fall back to legacy format + return c.Scope == "push" || c.Scope == "pull" +} diff --git a/lib/builds/registry_token_test.go b/lib/registrytoken/token_test.go similarity index 93% rename from lib/builds/registry_token_test.go rename to lib/registrytoken/token_test.go index e8eaf1037..898155ee5 100644 --- a/lib/builds/registry_token_test.go +++ b/lib/registrytoken/token_test.go @@ -1,4 +1,4 @@ -package builds +package registrytoken import ( "testing" @@ -9,7 +9,7 @@ import ( ) func TestRegistryTokenGenerator_GeneratePushToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token generation", func(t *testing.T) { token, err := generator.GeneratePushToken("build-123", []string{"builds/build-123", "cache/tenant-x"}, 30*time.Minute) @@ -40,7 +40,7 @@ func TestRegistryTokenGenerator_GeneratePushToken(t *testing.T) { } func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token", func(t *testing.T) { token, err := generator.GeneratePushToken("build-abc", []string{"builds/build-abc"}, time.Hour) @@ -63,12 +63,12 @@ func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { t.Run("invalid signature", func(t *testing.T) { // Generate with one secret - gen1 := NewRegistryTokenGenerator("secret-1") + gen1 := NewGenerator("secret-1") token, err := gen1.GeneratePushToken("build-123", []string{"builds/build-123"}, time.Hour) require.NoError(t, err) // Validate with different secret - gen2 := NewRegistryTokenGenerator("secret-2") + gen2 := NewGenerator("secret-2") _, err = gen2.ValidateToken(token) require.Error(t, err) assert.Contains(t, err.Error(), "signature is invalid") @@ -81,7 +81,7 @@ func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { } func TestRegistryTokenClaims_IsRepositoryAllowed(t *testing.T) { - claims := &RegistryTokenClaims{ + claims := &Claims{ Repositories: []string{"builds/abc123", "cache/tenant-x"}, } @@ -98,20 +98,20 @@ func TestRegistryTokenClaims_IsRepositoryAllowed(t *testing.T) { func TestRegistryTokenClaims_IsPushAllowed(t *testing.T) { t.Run("push scope", func(t *testing.T) { - claims := &RegistryTokenClaims{Scope: "push"} + claims := &Claims{Scope: "push"} assert.True(t, claims.IsPushAllowed()) assert.True(t, claims.IsPullAllowed()) // push implies pull }) t.Run("pull scope", func(t *testing.T) { - claims := &RegistryTokenClaims{Scope: "pull"} + claims := &Claims{Scope: "pull"} assert.False(t, claims.IsPushAllowed()) assert.True(t, claims.IsPullAllowed()) }) } func TestRegistryTokenGenerator_GenerateToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token with per-repo permissions", func(t *testing.T) { repoAccess := []RepoPermission{ @@ -147,7 +147,7 @@ func TestRegistryTokenGenerator_GenerateToken(t *testing.T) { func TestRegistryTokenClaims_RepoAccess(t *testing.T) { // Test claims with new per-repo access format - claims := &RegistryTokenClaims{ + claims := &Claims{ RepoAccess: []RepoPermission{ {Repo: "builds/abc123", Scope: "push"}, {Repo: "cache/global/node", Scope: "pull"}, @@ -195,7 +195,7 @@ func TestRegistryTokenClaims_RepoAccess(t *testing.T) { func TestRegistryTokenClaims_RepoAccessPullOnly(t *testing.T) { // Test claims with only pull access - claims := &RegistryTokenClaims{ + claims := &Claims{ RepoAccess: []RepoPermission{ {Repo: "cache/global/node", Scope: "pull"}, }, @@ -212,7 +212,7 @@ func TestRegistryTokenClaims_RepoAccessPullOnly(t *testing.T) { func TestRegistryTokenClaims_LegacyFallback(t *testing.T) { // Test that legacy format still works when RepoAccess is empty - claims := &RegistryTokenClaims{ + claims := &Claims{ Repositories: []string{"builds/abc123", "cache/tenant-x"}, Scope: "push", } diff --git a/lib/system/guest_agent/macos/install-launcher.sh b/lib/system/guest_agent/macos/install-launcher.sh new file mode 100755 index 000000000..c74b0d879 --- /dev/null +++ b/lib/system/guest_agent/macos/install-launcher.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Install the launcher that runs hypeman's guest agent from the read-only agent +# disk hypeman attaches to macOS guests. Run as root in the base image, for +# example from a Packer provisioner. A guest agent installed in the image itself +# is removed, so the agent always comes from the disk. +set -eu +launchctl bootout system/com.kernel.hypeman.guest-agent 2>/dev/null || true +rm -f /Library/LaunchDaemons/com.kernel.hypeman.guest-agent.plist /usr/local/sbin/hypeman-guest-agent +install -d -o root -g wheel -m 0755 /usr/local/libexec +cat > /usr/local/libexec/hypeman-agent-launcher <<'SH' +#!/bin/sh +# Run hypeman's guest agent from its read-only agent disk, mounting it if macOS +# has not yet. Only the disk hypeman attaches is trusted: a fixed internal device, +# not a disk image a guest user attached under the same name. +volume=/Volumes/HypemanAgent +install -d -o root -g wheel -m 0755 /var/run/hypeman +for i in $(seq 1 120); do + if [ -x "$volume/hypeman-guest-agent" ]; then + info=$(/usr/sbin/diskutil info "$volume" 2>/dev/null) + if echo "$info" | grep -Eq '^ *Device Location: +Internal$' && + echo "$info" | grep -Eq '^ *Removable Media: +Fixed$' && + ! echo "$info" | grep -Eq '^ *Protocol: +Disk Image$'; then + exec "$volume/hypeman-guest-agent" + fi + echo "refusing $volume: it is not hypeman's attached agent disk" >&2 + exit 1 + fi + /usr/sbin/diskutil mount readOnly HypemanAgent >/dev/null 2>&1 + sleep 1 +done +echo "hypeman agent disk not found" >&2 +exit 1 +SH +chown root:wheel /usr/local/libexec/hypeman-agent-launcher +chmod 0755 /usr/local/libexec/hypeman-agent-launcher +cat > /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist <<'PLIST' + + + + + Labelcom.kernel.hypeman.agent-launcher + ProgramArguments/usr/local/libexec/hypeman-agent-launcher + RunAtLoad + KeepAlive + UserNameroot + GroupNamewheel + StandardOutPath/var/log/hypeman-guest-agent.log + StandardErrorPath/var/log/hypeman-guest-agent.log + + +PLIST +chown root:wheel /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist +chmod 0644 /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist diff --git a/lib/system/macos_agent_disk.go b/lib/system/macos_agent_disk.go new file mode 100644 index 000000000..8f6695db6 --- /dev/null +++ b/lib/system/macos_agent_disk.go @@ -0,0 +1,8 @@ +package system + +// MacOSAgentVolume is the volume name of the read-only guest agent disk attached +// to macOS guests. The launcher in a macOS image runs MacOSAgentBinary from it. +const ( + MacOSAgentVolume = "HypemanAgent" + MacOSAgentBinary = "hypeman-guest-agent" +) diff --git a/lib/system/macos_agent_disk_darwin.go b/lib/system/macos_agent_disk_darwin.go new file mode 100644 index 000000000..ee38b2513 --- /dev/null +++ b/lib/system/macos_agent_disk_darwin.go @@ -0,0 +1,94 @@ +//go:build darwin + +package system + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + + "github.com/kernel/hypeman/lib/paths" +) + +// DarwinGuestAgentBinary is the macOS guest agent, built by the Makefile. +// +//go:embed guest_agent/guest-agent-darwin +var DarwinGuestAgentBinary []byte + +// macOSAgentDiskBuild serializes disk builds; waiting for one honors ctx. +var macOSAgentDiskBuild = make(chan struct{}, 1) + +// MacOSAgentDisk returns the read-only guest agent disk for the embedded macOS +// agent, building it on first use. Disks are named by the agent's hash, so a new +// agent gets a new disk; disks for other agents are removed, since every guest +// has its own copy. +func MacOSAgentDisk(ctx context.Context, p *paths.Paths) (string, error) { + sum := sha256.Sum256(DarwinGuestAgentBinary) + version := hex.EncodeToString(sum[:6]) + path := p.SystemMacOSAgentDisk(version) + select { + case macOSAgentDiskBuild <- struct{}{}: + case <-ctx.Done(): + return "", ctx.Err() + } + defer func() { <-macOSAgentDiskBuild }() + if _, err := os.Stat(path); err != nil { + if err := buildMacOSAgentDisk(ctx, DarwinGuestAgentBinary, version, path); err != nil { + return "", err + } + } + removeOtherMacOSAgentDisks(path) + return path, nil +} + +// removeOtherMacOSAgentDisks deletes the disks and build directories beside path, +// left by other agents or interrupted builds. +func removeOtherMacOSAgentDisks(path string) { + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + return + } + for _, entry := range entries { + if entry.Name() != filepath.Base(path) { + if err := os.RemoveAll(filepath.Join(filepath.Dir(path), entry.Name())); err != nil { + slog.Warn("remove old macOS agent disk", "path", entry.Name(), "error", err) + } + } + } +} + +// buildMacOSAgentDisk writes a raw APFS disk image at path holding the agent and a +// VERSION file. +func buildMacOSAgentDisk(ctx context.Context, binary []byte, version, path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + work, err := os.MkdirTemp(filepath.Dir(path), ".build-") + if err != nil { + return err + } + defer os.RemoveAll(work) + src := filepath.Join(work, "src") + if err := os.Mkdir(src, 0755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(src, MacOSAgentBinary), binary, 0755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(src, "VERSION"), []byte(version+"\n"), 0644); err != nil { + return err + } + // UDTO is a raw image, which Virtualization attaches as is. hdiutil appends .cdr. + out := filepath.Join(work, "agent") + cmd := exec.CommandContext(ctx, "/usr/bin/hdiutil", "create", "-quiet", "-srcfolder", src, "-volname", MacOSAgentVolume, "-fs", "APFS", "-format", "UDTO", out) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build macOS agent disk: %w: %s", err, output) + } + return os.Rename(out+".cdr", path) +} diff --git a/lib/system/macos_agent_disk_darwin_test.go b/lib/system/macos_agent_disk_darwin_test.go new file mode 100644 index 000000000..dcc45ae84 --- /dev/null +++ b/lib/system/macos_agent_disk_darwin_test.go @@ -0,0 +1,23 @@ +//go:build darwin + +package system + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildMacOSAgentDisk(t *testing.T) { + path := filepath.Join(t.TempDir(), "macos-agent", "v1.img") + require.NoError(t, buildMacOSAgentDisk(context.Background(), []byte("#!/bin/sh\n"), "v1", path)) + info, err := os.Stat(path) + require.NoError(t, err) + require.Zero(t, info.Size()%512, "a raw disk is whole sectors") + entries, err := os.ReadDir(filepath.Dir(path)) + require.NoError(t, err) + require.Len(t, entries, 1, "the build directory is removed") +} diff --git a/lib/system/macos_agent_disk_other.go b/lib/system/macos_agent_disk_other.go new file mode 100644 index 000000000..f2f504f29 --- /dev/null +++ b/lib/system/macos_agent_disk_other.go @@ -0,0 +1,15 @@ +//go:build !darwin + +package system + +import ( + "context" + "errors" + + "github.com/kernel/hypeman/lib/paths" +) + +// MacOSAgentDisk is only available on macOS hosts. +func MacOSAgentDisk(context.Context, *paths.Paths) (string, error) { + return "", errors.New("the macOS guest agent disk requires a macOS host") +}