macOS guests 6/6: experimental native image builds and release tooling

cmd/api/api/builds.go

@@ -44,9 +44,15 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ
 // CreateBuild creates a new build job
 func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) {
 	if s.Config != nil && s.Config.MacOSOnly {
-		return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil
+		if !s.Config.MacOSBuilds {
+			return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil
+		}
+		req, source, err := parseMachineBuildMultipart(request.Body)
+		if err != nil {
+			return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil
+		}
+		return s.submitBuild(ctx, req, source)
 	}
-	log := logger.FromContext(ctx)
 
 	// Parse multipart form fields
 	var sourceData []byte
@@ -69,6 +75,8 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe
 		}
 
 		switch part.FormName() {
+		case "machine_base_image", "machine_network_mode":
+			return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil
 		case "source":
 			sourceData, err = io.ReadAll(part)
 			if err != nil {
@@ -256,6 +264,11 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe
 		}
 	}
 
+	return s.submitBuild(ctx, domainReq, sourceData)
+}
+
+func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) {
+	log := logger.FromContext(ctx)
 	build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData)
 	if err != nil {
 		switch {

cmd/api/api/capabilities.go

@@ -6,7 +6,6 @@ import (
 	"sync"
 
 	"github.com/kernel/hypeman/lib/hypervisor"
-	"github.com/kernel/hypeman/lib/images"
 	"github.com/kernel/hypeman/lib/logger"
 	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
@@ -26,6 +25,8 @@ const (
 	featureLogs             = "logs"
 	featureDevices          = "devices"
 	featureRosettaEmulation = "rosetta-emulation"
+	featureMacOSImages      = "macos-machine-images"
+	featureMacOSBuilds      = "macos-machine-builds-v1"
 )
 
 // apiVersion is the API contract version from the embedded OpenAPI document.
@@ -133,13 +134,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities
 		},
 		Runtimes: runtimes,
 		Network:  *networkCaps,
-		Images: oapi.CapabilitiesImages{
-			Platforms:       imagePlatforms(runtime.GOARCH, emulation),
-			DefaultPlatform: images.HostPlatformString(),
-		},
+		Images:   hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly),
 		Features: serverFeatures(runtime.GOOS, emulation),
 	}
 
+	resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSBuilds)...)
 	return oapi.GetCapabilities200JSONResponse(resp), nil
 }
 
@@ -210,6 +209,33 @@ func imagePlatforms(goarch string, emulation bool) []string {
 	return platforms
 }
 
+func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages {
+	caps := oapi.CapabilitiesImages{
+		Platforms:       imagePlatforms(goarch, emulation),
+		DefaultPlatform: "linux/" + goarch,
+	}
+	if goos == "darwin" && goarch == "arm64" {
+		if macOSOnly {
+			caps.Platforms = []string{"darwin/arm64"}
+			caps.DefaultPlatform = "darwin/arm64"
+		} else {
+			caps.Platforms = append(caps.Platforms, "darwin/arm64")
+		}
+	}
+	return caps
+}
+
+func machineImageFeatures(goos, goarch string, builds bool) []string {
+	if goos != "darwin" || goarch != "arm64" {
+		return nil
+	}
+	features := []string{featureMacOSImages}
+	if builds {
+		features = append(features, featureMacOSBuilds)
+	}
+	return features
+}
+
 // serverFeatures builds the server-level feature ID list: API surfaces that
 // are always present plus host-platform conditionals.
 func serverFeatures(goos string, emulation bool) []string {

cmd/api/api/capabilities_machine_test.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHostMachineImageCapabilities(t *testing.T) {
+	for _, tc := range []struct {
+		name, os, arch, defaultPlatform string
+		emulation, macOSOnly            bool
+		platforms                       []string
+	}{
+		{"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}},
+		{"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}},
+		{"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}},
+		{"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}},
+		{"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly)
+			require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform)
+			require.Equal(t, tc.platforms, caps.Platforms)
+		})
+	}
+}
+
+func TestMachineImageFeatureActivation(t *testing.T) {
+	require.Empty(t, machineImageFeatures("linux", "arm64", true))
+	require.Empty(t, machineImageFeatures("darwin", "amd64", true))
+	require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false))
+	require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true))
+}

cmd/api/api/instances.go

@@ -63,7 +63,11 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc
 func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) {
 	log := logger.FromContext(ctx)
 
-	macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image)
+	platform := derefString(request.Body.Platform)
+	if s.Config != nil && s.Config.MacOSOnly && platform == "" {
+		platform = "darwin/arm64"
+	}
+	macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform)
 	if err != nil {
 		log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image)
 		return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil
@@ -337,7 +341,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		Name:                     request.Body.Name,
 		Image:                    request.Body.Image,
 		Size:                     size,
-		Platform:                 derefString(request.Body.Platform),
+		Platform:                 platform,
 		HotplugSize:              hotplugSize,
 		OverlaySize:              overlaySize,
 		Vcpus:                    vcpus,

cmd/api/api/macos.go

@@ -16,11 +16,19 @@ import (
 // macOSHost is true where macOS guests can exist. Tests override it.
 var macOSHost = runtime.GOOS == "darwin"
 
-// isMacOSImage reports whether a create names an imported macOS image. Other
-// hosts skip the lookup, so Linux creates are unchanged. An image that is not
-// cached yet is not a macOS image; any other lookup error is returned.
-func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) {
-	if !macOSHost || s.ImageManager == nil {
+// isMacOSImage identifies macOS creates before Linux defaults are applied. An
+// explicit Darwin platform and macOS-only servers can resolve uncached images.
+func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) {
+	if !macOSHost {
+		return false, nil
+	}
+	if s.Config != nil && s.Config.MacOSOnly {
+		return true, nil
+	}
+	if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" {
+		return true, nil
+	}
+	if s.ImageManager == nil {
 		return false, nil
 	}
 	img, err := s.ImageManager.GetImage(ctx, name)

cmd/api/api/macos_builds.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+
+	"github.com/kernel/hypeman/lib/builds"
+	"github.com/kernel/hypeman/lib/images"
+	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/scopes"
+)
+
+// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up
+// to 65 MiB, are read into memory at once.
+const maxConcurrentMachineBuildUploads = 4
+
+// LimitMachineBuildBody must wrap the validator, which also consumes request bodies.
+// A build upload is authenticated before it takes an upload slot, so unauthenticated
+// requests cannot hold the slots.
+func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		uploads := make(chan struct{}, maxConcurrentMachineBuildUploads)
+		upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			select {
+			case uploads <- struct{}{}:
+				defer func() { <-uploads }()
+			default:
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusTooManyRequests)
+				_, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`))
+				return
+			}
+			r.Body = http.MaxBytesReader(w, r.Body, 65<<20)
+			next.ServeHTTP(w, r)
+		})))
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method == http.MethodPost && r.URL.Path == "/builds" {
+				upload.ServeHTTP(w, r)
+				return
+			}
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// Machine-only HTTP admission is bounded regardless of multipart field order.
+// Linux retains its existing parser and source contract.
+func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) {
+	req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}}
+	policy := req.BuildPolicy
+	numbers := map[string]*int{"timeout_seconds": &policy.TimeoutSeconds, "memory_mb": &policy.MemoryMB, "cpus": &policy.CPUs}
+	var source []byte
+	seen := map[string]bool{}
+	if body == nil {
+		return req, nil, fmt.Errorf("multipart source is required")
+	}
+	for {
+		part, err := body.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return req, nil, fmt.Errorf("invalid multipart form")
+		}
+		name := part.FormName()
+		if seen[name] {
+			return req, nil, fmt.Errorf("duplicate multipart field")
+		}
+		seen[name] = true
+		limit := int64(64 << 10)
+		switch name {
+		case "source":
+			limit = 64 << 20
+		case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags":
+		default:
+			return req, nil, fmt.Errorf("unsupported machine build field: %s", name)
+		}
+		data, err := io.ReadAll(io.LimitReader(part, limit+1))
+		if err != nil || int64(len(data)) > limit {
+			return req, nil, fmt.Errorf("invalid or oversized %s field", name)
+		}
+		if err = part.Close(); err != nil {
+			return req, nil, fmt.Errorf("invalid multipart field")
+		}
+		switch name {
+		case "source":
+			source = data
+		case "machine_base_image":
+			req.MachineBaseImage = string(data)
+		case "machine_network_mode":
+			if string(data) != "isolated" && string(data) != "egress" {
+				return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress")
+			}
+			req.BuildPolicy.NetworkMode = string(data)
+		case "tags":
+			req.Tags, err = parseTagsJSON(string(data))
+			if err != nil {
+				return req, nil, fmt.Errorf("tags must be a JSON object of strings")
+			}
+		default:
+			value, err := strconv.Atoi(string(data))
+			if err != nil || value < 0 {
+				return req, nil, fmt.Errorf("%s must be a nonnegative integer", name)
+			}
+			*numbers[name] = value
+		}
+	}
+	if len(source) == 0 {
+		return req, nil, fmt.Errorf("source is required")
+	}
+	ref, err := images.ParseNormalizedRef(req.MachineBaseImage)
+	if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) {
+		return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest")
+	}
+	if req.BuildPolicy.TimeoutSeconds > 86400 {
+		return req, nil, fmt.Errorf("timeout_seconds exceeds 86400")
+	}
+	if err = req.BuildPolicy.Validate(); err != nil {
+		return req, nil, err
+	}
+	return req, source, nil
+}