From ae399703469b07eae379a5f9c26f718c2dafa877 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:07:04 +0000 Subject: [PATCH 1/2] macOS guests 6/6: experimental native image builds Opt-in native macOS image builds, off by default. POST /builds takes an installed, digest-pinned darwin/arm64 base and a source tarball containing build.sh. The script runs as root in an isolated VZ guest; the guest is strictly stopped, its closed storage exported, and the result published to this server's registry as an OCI machine image. Derivatives of a base in the same registry publish a delta layer of changed 64 KiB disk chunks. Mac hosts pull the image on instance create. Linux builds are unchanged. Requires macos_only and macos_builds, with a loopback or system-trusted registry URL. Base publication tooling is a separate change. --- cmd/api/api/builds.go | 17 +- cmd/api/api/capabilities.go | 36 +- cmd/api/api/capabilities_machine_test.go | 34 + cmd/api/api/instances.go | 8 +- cmd/api/api/macos.go | 18 +- cmd/api/api/macos_builds.go | 125 +++ cmd/api/api/macos_builds_test.go | 226 +++++ cmd/api/api/macos_test.go | 38 + cmd/api/config/config.go | 24 +- cmd/api/config/macos_builds_test.go | 26 + cmd/api/main.go | 13 +- config.example.darwin.yaml | 8 + docs/macos-builds.md | 125 +++ docs/macos-support.md | 2 +- lib/builds/macos_build.go | 300 +++++++ lib/builds/macos_build_cancel.go | 70 ++ lib/builds/macos_build_failure_test.go | 198 +++++ lib/builds/macos_build_guest.go | 185 ++++ lib/builds/macos_build_job_test.go | 37 + lib/builds/macos_build_layer_test.go | 70 ++ lib/builds/macos_build_lifecycle_test.go | 135 +++ lib/builds/macos_build_manager.go | 199 +++++ lib/builds/macos_build_manager_test.go | 228 +++++ lib/builds/macos_build_policy_test.go | 44 + lib/builds/macos_build_publisher.go | 194 +++++ lib/builds/macos_build_publisher_test.go | 96 +++ lib/builds/macos_build_recovery_test.go | 88 ++ lib/builds/macos_build_source.go | 119 +++ lib/builds/macos_build_source_test.go | 67 ++ lib/builds/macos_build_test.go | 226 +++++ lib/builds/macos_delta_publish.go | 172 ++++ lib/builds/macos_delta_publish_test.go | 202 +++++ lib/builds/macos_registry.go | 105 +++ lib/builds/macos_registry_test.go | 54 ++ lib/builds/manager.go | 105 ++- lib/builds/source.go | 75 ++ lib/builds/source_test.go | 36 + lib/builds/storage.go | 8 +- lib/builds/types.go | 7 + lib/images/macos_bundle.go | 15 + lib/images/macos_delta.go | 261 ++++++ lib/images/macos_delta_test.go | 203 +++++ lib/images/macos_import_darwin.go | 8 +- lib/images/macos_unpack.go | 23 +- lib/instances/create.go | 3 + lib/instances/create_image.go | 16 +- lib/instances/create_image_test.go | 50 ++ lib/instances/delete.go | 2 +- lib/instances/fork.go | 2 +- lib/instances/macos.go | 15 +- lib/instances/macos_agent_startup.go | 2 +- lib/instances/macos_build_export.go | 5 + lib/instances/macos_build_export_darwin.go | 133 +++ .../macos_build_export_darwin_test.go | 53 ++ lib/instances/macos_identity.go | 2 +- lib/instances/manager.go | 2 +- lib/instances/process_identity_linux_test.go | 8 +- lib/instances/restart_policy.go | 2 +- lib/instances/stop.go | 32 +- lib/middleware/oapi_auth.go | 24 + lib/oapi/oapi.go | 812 +++++++++--------- lib/providers/providers.go | 23 +- lib/registry/registry.go | 27 + lib/registrypush/macos_test.go | 48 ++ openapi.yaml | 18 + 65 files changed, 5043 insertions(+), 466 deletions(-) create mode 100644 cmd/api/api/capabilities_machine_test.go create mode 100644 cmd/api/api/macos_builds.go create mode 100644 cmd/api/api/macos_builds_test.go create mode 100644 cmd/api/config/macos_builds_test.go create mode 100644 docs/macos-builds.md create mode 100644 lib/builds/macos_build.go create mode 100644 lib/builds/macos_build_cancel.go create mode 100644 lib/builds/macos_build_failure_test.go create mode 100644 lib/builds/macos_build_guest.go create mode 100644 lib/builds/macos_build_job_test.go create mode 100644 lib/builds/macos_build_layer_test.go create mode 100644 lib/builds/macos_build_lifecycle_test.go create mode 100644 lib/builds/macos_build_manager.go create mode 100644 lib/builds/macos_build_manager_test.go create mode 100644 lib/builds/macos_build_policy_test.go create mode 100644 lib/builds/macos_build_publisher.go create mode 100644 lib/builds/macos_build_publisher_test.go create mode 100644 lib/builds/macos_build_recovery_test.go create mode 100644 lib/builds/macos_build_source.go create mode 100644 lib/builds/macos_build_source_test.go create mode 100644 lib/builds/macos_build_test.go create mode 100644 lib/builds/macos_delta_publish.go create mode 100644 lib/builds/macos_delta_publish_test.go create mode 100644 lib/builds/macos_registry.go create mode 100644 lib/builds/macos_registry_test.go create mode 100644 lib/builds/source.go create mode 100644 lib/builds/source_test.go create mode 100644 lib/images/macos_delta.go create mode 100644 lib/images/macos_delta_test.go create mode 100644 lib/instances/macos_build_export.go create mode 100644 lib/instances/macos_build_export_darwin.go create mode 100644 lib/instances/macos_build_export_darwin_test.go create mode 100644 lib/registrypush/macos_test.go diff --git a/cmd/api/api/builds.go b/cmd/api/api/builds.go index ae8e74c27..2f4174b3b 100644 --- a/cmd/api/api/builds.go +++ b/cmd/api/api/builds.go @@ -44,9 +44,15 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ // CreateBuild creates a new build job func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) { if s.Config != nil && s.Config.MacOSOnly { - return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + if !s.Config.MacOSBuilds { + return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + } + req, source, err := parseMachineBuildMultipart(request.Body) + if err != nil { + return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil + } + return s.submitBuild(ctx, req, source) } - log := logger.FromContext(ctx) // Parse multipart form fields var sourceData []byte @@ -69,6 +75,8 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } switch part.FormName() { + case "machine_base_image", "machine_network_mode": + return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil case "source": sourceData, err = io.ReadAll(part) if err != nil { @@ -256,6 +264,11 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } } + return s.submitBuild(ctx, domainReq, sourceData) +} + +func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) { + log := logger.FromContext(ctx) build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData) if err != nil { switch { diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index e19ca8035..63c82e31f 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" @@ -26,6 +25,8 @@ const ( featureLogs = "logs" featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" + featureMacOSImages = "macos-machine-images" + featureMacOSBuilds = "macos-machine-builds-v1" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -133,13 +134,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities }, Runtimes: runtimes, Network: *networkCaps, - Images: oapi.CapabilitiesImages{ - Platforms: imagePlatforms(runtime.GOARCH, emulation), - DefaultPlatform: images.HostPlatformString(), - }, + Images: hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly), Features: serverFeatures(runtime.GOOS, emulation), } + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSBuilds)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -210,6 +209,33 @@ func imagePlatforms(goarch string, emulation bool) []string { return platforms } +func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages { + caps := oapi.CapabilitiesImages{ + Platforms: imagePlatforms(goarch, emulation), + DefaultPlatform: "linux/" + goarch, + } + if goos == "darwin" && goarch == "arm64" { + if macOSOnly { + caps.Platforms = []string{"darwin/arm64"} + caps.DefaultPlatform = "darwin/arm64" + } else { + caps.Platforms = append(caps.Platforms, "darwin/arm64") + } + } + return caps +} + +func machineImageFeatures(goos, goarch string, builds bool) []string { + if goos != "darwin" || goarch != "arm64" { + return nil + } + features := []string{featureMacOSImages} + if builds { + features = append(features, featureMacOSBuilds) + } + return features +} + // serverFeatures builds the server-level feature ID list: API surfaces that // are always present plus host-platform conditionals. func serverFeatures(goos string, emulation bool) []string { diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go new file mode 100644 index 000000000..705eabf86 --- /dev/null +++ b/cmd/api/api/capabilities_machine_test.go @@ -0,0 +1,34 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHostMachineImageCapabilities(t *testing.T) { + for _, tc := range []struct { + name, os, arch, defaultPlatform string + emulation, macOSOnly bool + platforms []string + }{ + {"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}}, + {"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}}, + {"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}}, + {"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}}, + {"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}}, + } { + t.Run(tc.name, func(t *testing.T) { + caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly) + require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform) + require.Equal(t, tc.platforms, caps.Platforms) + }) + } +} + +func TestMachineImageFeatureActivation(t *testing.T) { + require.Empty(t, machineImageFeatures("linux", "arm64", true)) + require.Empty(t, machineImageFeatures("darwin", "amd64", true)) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false)) + require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true)) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..ced753723 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,7 +63,11 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image) + platform := derefString(request.Body.Platform) + if s.Config != nil && s.Config.MacOSOnly && platform == "" { + platform = "darwin/arm64" + } + macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil @@ -337,7 +341,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Name: request.Body.Name, Image: request.Body.Image, Size: size, - Platform: derefString(request.Body.Platform), + Platform: platform, HotplugSize: hotplugSize, OverlaySize: overlaySize, Vcpus: vcpus, diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 5dc42cbdb..62e6652ce 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -16,11 +16,19 @@ import ( // macOSHost is true where macOS guests can exist. Tests override it. var macOSHost = runtime.GOOS == "darwin" -// isMacOSImage reports whether a create names an imported macOS image. Other -// hosts skip the lookup, so Linux creates are unchanged. An image that is not -// cached yet is not a macOS image; any other lookup error is returned. -func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) { - if !macOSHost || s.ImageManager == nil { +// isMacOSImage identifies macOS creates before Linux defaults are applied. An +// explicit Darwin platform and macOS-only servers can resolve uncached images. +func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) { + if !macOSHost { + return false, nil + } + if s.Config != nil && s.Config.MacOSOnly { + return true, nil + } + if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { + return true, nil + } + if s.ImageManager == nil { return false, nil } img, err := s.ImageManager.GetImage(ctx, name) diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go new file mode 100644 index 000000000..9ec28c825 --- /dev/null +++ b/cmd/api/api/macos_builds.go @@ -0,0 +1,125 @@ +package api + +import ( + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/scopes" +) + +// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up +// to 65 MiB, are read into memory at once. +const maxConcurrentMachineBuildUploads = 4 + +// LimitMachineBuildBody must wrap the validator, which also consumes request bodies. +// A build upload is authenticated before it takes an upload slot, so unauthenticated +// requests cannot hold the slots. +func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + uploads := make(chan struct{}, maxConcurrentMachineBuildUploads) + upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case uploads <- struct{}{}: + defer func() { <-uploads }() + default: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`)) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 65<<20) + next.ServeHTTP(w, r) + }))) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/builds" { + upload.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// Machine-only HTTP admission is bounded regardless of multipart field order. +// Linux retains its existing parser and source contract. +func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) { + req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}} + policy := req.BuildPolicy + numbers := map[string]*int{"timeout_seconds": &policy.TimeoutSeconds, "memory_mb": &policy.MemoryMB, "cpus": &policy.CPUs} + var source []byte + seen := map[string]bool{} + if body == nil { + return req, nil, fmt.Errorf("multipart source is required") + } + for { + part, err := body.NextPart() + if err == io.EOF { + break + } + if err != nil { + return req, nil, fmt.Errorf("invalid multipart form") + } + name := part.FormName() + if seen[name] { + return req, nil, fmt.Errorf("duplicate multipart field") + } + seen[name] = true + limit := int64(64 << 10) + switch name { + case "source": + limit = 64 << 20 + case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags": + default: + return req, nil, fmt.Errorf("unsupported machine build field: %s", name) + } + data, err := io.ReadAll(io.LimitReader(part, limit+1)) + if err != nil || int64(len(data)) > limit { + return req, nil, fmt.Errorf("invalid or oversized %s field", name) + } + if err = part.Close(); err != nil { + return req, nil, fmt.Errorf("invalid multipart field") + } + switch name { + case "source": + source = data + case "machine_base_image": + req.MachineBaseImage = string(data) + case "machine_network_mode": + if string(data) != "isolated" && string(data) != "egress" { + return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress") + } + req.BuildPolicy.NetworkMode = string(data) + case "tags": + req.Tags, err = parseTagsJSON(string(data)) + if err != nil { + return req, nil, fmt.Errorf("tags must be a JSON object of strings") + } + default: + value, err := strconv.Atoi(string(data)) + if err != nil || value < 0 { + return req, nil, fmt.Errorf("%s must be a nonnegative integer", name) + } + *numbers[name] = value + } + } + if len(source) == 0 { + return req, nil, fmt.Errorf("source is required") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) { + return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest") + } + if req.BuildPolicy.TimeoutSeconds > 86400 { + return req, nil, fmt.Errorf("timeout_seconds exceeds 86400") + } + if err = req.BuildPolicy.Validate(); err != nil { + return req, nil, err + } + return req, source, nil +} diff --git a/cmd/api/api/macos_builds_test.go b/cmd/api/api/macos_builds_test.go new file mode 100644 index 000000000..2fe07942f --- /dev/null +++ b/cmd/api/api/macos_builds_test.go @@ -0,0 +1,226 @@ +package api + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func machineMultipart(t *testing.T, fields ...[2]string) *multipart.Reader { + t.Helper() + var data bytes.Buffer + writer := multipart.NewWriter(&data) + for _, field := range fields { + require.NoError(t, writer.WriteField(field[0], field[1])) + } + require.NoError(t, writer.Close()) + return multipart.NewReader(&data, writer.Boundary()) +} + +type machineZeroReader struct{} + +func (machineZeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } + +func TestMachineBuildBodyBoundBeforeValidation(t *testing.T) { + for _, path := range []string{"/builds", "/images"} { + r := authorizedMachineBuildRequest(t, path, io.LimitReader(machineZeroReader{}, (65<<20)+1)) + LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, err := io.Copy(io.Discard, r.Body) + if path == "/builds" { + require.Error(t, err) + require.EqualValues(t, 65<<20, n) + } else { + require.NoError(t, err) + require.EqualValues(t, (65<<20)+1, n) + } + })).ServeHTTP(httptest.NewRecorder(), r) + } +} + +func TestMachineBuildUploadsAreBounded(t *testing.T) { + release := make(chan struct{}) + entered := make(chan struct{}, maxConcurrentMachineBuildUploads) + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entered <- struct{}{} + <-release + })) + done := make(chan struct{}) + for range maxConcurrentMachineBuildUploads { + go func() { + handler.ServeHTTP(httptest.NewRecorder(), authorizedMachineBuildRequest(t, "/builds", nil)) + done <- struct{}{} + }() + <-entered + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusTooManyRequests, w.Code) + + close(release) + for range maxConcurrentMachineBuildUploads { + <-done + } + w = httptest.NewRecorder() + go func() { <-entered }() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "a slot frees once an upload finishes") +} + +func TestMachineBuildMultipartAdmission(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + valid := [][2]string{{"source", "synthetic"}, {"machine_base_image", base}} + for _, tc := range []struct { + name string + extra [][2]string + ok bool + }{ + {name: "inherit", ok: true}, + {name: "policy", extra: [][2]string{{"cpus", "4"}, {"memory_mb", "8192"}, {"timeout_seconds", "1200"}}, ok: true}, + {name: "isolated network", extra: [][2]string{{"machine_network_mode", "isolated"}}, ok: true}, + {name: "egress network", extra: [][2]string{{"machine_network_mode", "egress"}}, ok: true}, + {name: "invalid network", extra: [][2]string{{"machine_network_mode", "bridge"}}}, + {name: "empty network", extra: [][2]string{{"machine_network_mode", ""}}}, + {name: "duplicate", extra: [][2]string{{"source", "second"}}}, + {name: "linux", extra: [][2]string{{"dockerfile", "FROM scratch"}}}, + {name: "secret", extra: [][2]string{{"secrets", "[]"}}}, + {name: "invalid number", extra: [][2]string{{"cpus", "nan"}}}, + {name: "negative", extra: [][2]string{{"memory_mb", "-1"}}}, + {name: "timeout", extra: [][2]string{{"timeout_seconds", "86401"}}}, + {name: "metadata bound", extra: [][2]string{{"tags", strings.Repeat("x", (64<<10)+1)}}}, + } { + t.Run(tc.name, func(t *testing.T) { + req, source, err := parseMachineBuildMultipart(machineMultipart(t, append(append([][2]string{}, valid...), tc.extra...)...)) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, base, req.MachineBaseImage) + require.Equal(t, []byte("synthetic"), source) + if tc.name == "egress network" { + require.Equal(t, "egress", req.BuildPolicy.NetworkMode) + } + }) + } + _, _, err := parseMachineBuildMultipart(machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", "example.test/base:latest"})) + require.Error(t, err) +} + +type machineHTTPManager struct { + builds.Manager + req builds.CreateBuildRequest + calls int +} + +func (m *machineHTTPManager) CreateBuild(_ context.Context, req builds.CreateBuildRequest, source []byte) (*builds.Build, error) { + m.calls++ + m.req = req + return &builds.Build{ID: "synthetic-build", Status: builds.StatusQueued}, nil +} + +func TestMachineBuildHTTPGateAndDispatch(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + for _, tc := range []struct { + name string + cfg *config.Config + accepted bool + }{ + {"disabled", &config.Config{MacOSOnly: true}, false}, + {"linux", &config.Config{}, false}, + {"enabled", &config.Config{MacOSOnly: true, MacOSBuilds: true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := &machineHTTPManager{} + service := &ApiService{Config: tc.cfg, BuildManager: manager} + response, err := service.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{Body: machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", base})}) + require.NoError(t, err) + if tc.accepted { + require.IsType(t, oapi.CreateBuild202JSONResponse{}, response) + require.Equal(t, 1, manager.calls) + require.Equal(t, base, manager.req.MachineBaseImage) + } else { + // Disabled machine builds are unsupported (501) on macOS-only servers, and + // a Linux server rejects the machine selector as a bad request (400). + if tc.cfg.MacOSOnly { + require.IsType(t, oapi.CreateBuild501JSONResponse{}, response) + } else { + require.IsType(t, oapi.CreateBuild400JSONResponse{}, response) + } + require.Zero(t, manager.calls) + } + }) + } +} + +const machineBuildTestSecret = "machine-build-test-secret" + +func signedMachineBuildToken(t *testing.T, claims jwt.MapClaims) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(machineBuildTestSecret)) + require.NoError(t, err) + return token +} + +func authorizedMachineBuildRequest(t *testing.T, path string, body io.Reader) *http.Request { + t.Helper() + token := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + }) + r := httptest.NewRequest(http.MethodPost, path, body) + r.Header.Set("Authorization", "Bearer "+token) + return r +} + +func TestMachineBuildUploadsRequireAuthBeforeTakingASlot(t *testing.T) { + readOnly := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + "permissions": []string{"build:read"}, + }) + for _, tc := range []struct { + name string + header string + status int + }{ + {"unauthenticated", "", http.StatusUnauthorized}, + {"read-only token", "Bearer " + readOnly, http.StatusForbidden}, + } { + t.Run(tc.name, func(t *testing.T) { + reached := false + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + reached = true + })) + for range maxConcurrentMachineBuildUploads + 1 { + r := httptest.NewRequest(http.MethodPost, "/builds", nil) + if tc.header != "" { + r.Header.Set("Authorization", tc.header) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + require.Equal(t, tc.status, w.Code) + } + require.False(t, reached, "rejected uploads must not reach processing") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "rejected requests hold no upload slot") + require.True(t, reached) + }) + } +} diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index ce19cce6b..fc19ca81f 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -84,6 +84,38 @@ func TestLinuxCreateDefaultsMatchSchema(t *testing.T) { require.EqualValues(t, linuxDefaultVcpus, props["vcpus"].Value.Default) } +func TestMacOSCreateDefaultsToDarwinPlatform(t *testing.T) { + useMacOSHost(t) + darwin := "darwin/arm64" + for _, tc := range []struct { + name string + cfg *config.Config + images images.Manager + body string + platform *string + }{ + {"macOS-only server", &config.Config{MacOSOnly: true}, nil, `{"name":"macos","image":"registry.example/macos/app:latest"}`, nil}, + {"uncached darwin image", &config.Config{}, missingMacOSImageManager{}, `{"name":"macos","image":"registry.example/macos/app:latest","platform":"darwin/arm64"}`, &darwin}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, ImageManager: tc.images, Config: tc.cfg} + ctx, status := recordCreateBody(t, tc.body) + require.Equal(t, http.StatusOK, status) + size, overlay, vcpus := linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Platform: tc.platform, + Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) + }) + } +} + func TestMacOSCreateIgnoresInjectedLinuxDefaults(t *testing.T) { useMacOSHost(t) m := newCaptureCreateManager(nil) @@ -172,3 +204,9 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } + +type missingMacOSImageManager struct{ images.Manager } + +func (missingMacOSImageManager) GetImage(context.Context, string) (*images.Image, error) { + return nil, images.ErrNotFound +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 0bffb655b..544a5d494 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -285,7 +285,8 @@ type Config struct { JwtSecret string `koanf:"jwt_secret"` Env string `koanf:"env"` Version string `koanf:"version"` - MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. + MacOSBuilds bool `koanf:"macos_builds"` // Experimental: enable machine builds on the macOS-only server. + MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. Network NetworkConfig `koanf:"network"` Caddy CaddyConfig `koanf:"caddy"` @@ -606,6 +607,9 @@ func (c *Config) Validate() error { if err := c.validateAPIListenPorts(); err != nil { return err } + if err := c.validateMacOSBuilds(); err != nil { + return err + } if strings.TrimSpace(c.Metrics.ListenAddress) == "" { return fmt.Errorf("metrics.listen_address must not be empty") } @@ -852,3 +856,21 @@ func (c *Config) validateFirecrackerUFFDGraduation() error { func intPtr(v int) *int { return &v } + +// validateMacOSBuilds checks the experimental machine-build gate. The publisher +// validates the registry URL itself, since the base publication tool shares it. +func (c *Config) validateMacOSBuilds() error { + if !c.MacOSBuilds { + return nil + } + if !c.MacOSOnly { + return fmt.Errorf("macos_builds requires macos_only") + } + if c.Registry.URL == "" { + return fmt.Errorf("macos_builds requires an explicit registry URL") + } + if c.Registry.CACertFile != "" || c.Registry.Insecure { + return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") + } + return nil +} diff --git a/cmd/api/config/macos_builds_test.go b/cmd/api/config/macos_builds_test.go new file mode 100644 index 000000000..6a4cd08d9 --- /dev/null +++ b/cmd/api/config/macos_builds_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildsFailClosedConfiguration(t *testing.T) { + require.False(t, (&Config{}).MacOSBuilds) + c := *defaultConfig() + c.MacOSBuilds = true + require.ErrorContains(t, c.validateMacOSBuilds(), "requires macos_only") + c.MacOSOnly = true + c.Hypervisor.Default = "vz" + c.Registry.URL = "" + require.ErrorContains(t, c.validateMacOSBuilds(), "explicit registry URL") + c.Registry.URL = "http://localhost:4973" + c.Registry.CACertFile = "custom.pem" + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.CACertFile = "" + c.Registry.Insecure = true + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.Insecure = false + require.NoError(t, c.validateMacOSBuilds()) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 347b395ff..1c8241323 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -535,6 +535,11 @@ func run() error { r.Use(api.RecordCreateFields) } + // Bound machine multipart input before OpenAPI validation reads the body. + if app.Config.MacOSBuilds { + r.Use(api.LimitMachineBuildBody(app.Config.JwtSecret)) + } + // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ Options: openapi3filter.Options{ @@ -632,10 +637,10 @@ func run() error { logger.Error("failed to start builders manager", "error", err) return err } - if err := app.BuildManager.Start(gctx); err != nil { - logger.Error("failed to start build manager", "error", err) - return err - } + } + if err := app.BuildManager.Start(gctx); err != nil { + logger.Error("failed to start build manager", "error", err) + return err } grp.Go(func() error { diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index e7745fb1a..d838b9887 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,6 +27,14 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" +# Native macOS guests/builds are opt-in; these defaults retain Linux guests on VZ. +macos_only: false +macos_builds: false +# For a dedicated native Mac builder, set both true after the release gates in +# docs/macos-builds.md pass, and replace registry.url below with an explicit +# system-trusted HTTPS address reachable by the build and destination hosts. +# Machine publication runs on the host, not at the Linux guest NAT gateway. + # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) # ============================================================================= diff --git a/docs/macos-builds.md b/docs/macos-builds.md new file mode 100644 index 000000000..81f2eb217 --- /dev/null +++ b/docs/macos-builds.md @@ -0,0 +1,125 @@ +# macOS image builds (experimental, opt-in) + +A build job can provision a digest-pinned installed macOS base in an isolated VZ +VM, run the source's `build.sh` in it, and publish the result as an OCI machine +image. This is not macOS installation, Linux rootfs conversion, memory forking or +identity rekeying. It depends on the shared system GuestService. + +The normal build manager owns the queue, persisted request, deadline, source +staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` +runs the guest and `StreamingMachinePublisher` writes the registry. + +## Enabling + +Off by default. Set `macos_only: true` and `macos_builds: true` on an Apple-silicon +VZ server, with `registry.url` set to an address the host can reach (not a guest +gateway), including its scheme: `http://localhost:4973` for loopback, or a +system-trusted `https://` endpoint. Credentials, paths, queries, fragments, custom +CAs and insecure HTTPS are rejected. Linux servers reject the machine selector. + +```sh +curl -H "Authorization: Bearer $TOKEN" \ + -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ + -F "source=@source.tar.gz;type=application/gzip" \ + -F "timeout_seconds=1200" "$API/builds" +``` + +The base must already be Ready. A 202 queues an ordinary job; status, cancellation +and provenance use the shared `/builds` APIs. Only `source`, `machine_base_image`, +`machine_network_mode`, `timeout_seconds`, `memory_mb`, `cpus` and `tags` are +accepted. Duplicate or unknown fields, malformed numbers and mutable bases return +400. Limits: 64 KiB per metadata field, 64 MiB source, 65 MiB total body. Linux +source parsing is unchanged. + +## Using the result + +`GET /builds/{id}` returns a digest-pinned `image_ref` for instance create. Other +Mac hosts need `registry.url` reachable from them; loopback only works on the build +host. A host that hasn't cached the image pulls on create, which can exceed the +readiness wait for large images, so retry or pre-pull. Distribute by digest with +`POST /pushes`. Consumers should pin the digest, not a tag. + +## Base and policy + +The base must be pinned by canonical SHA256, be `darwin/arm64`, and declare a +provisioned system agent on port 2222. The driver also checks guest UID 0 and never +installs an agent or falls back to host SSH. CPU and memory match the base exactly; +zero request values inherit them. Timeout defaults to 600 s, capped at 24 h. + +Networking defaults to isolated. `machine_network_mode=egress` gives unrestricted +VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, +cache, build-arg, secret and admin options are rejected. + +Jobs on the same base digest are serialized. Identity is preserved, so concurrent +guests with the same identity are never allowed. + +## Source + +The source tar.gz is staged privately and SHA256-verified, including on recovery, +and `provenance.source_hash` pins exactly what ran. Input is capped at 64 MiB +compressed, 128 MiB uncompressed, 1024 entries and 64 MiB per file. Only regular +files and directories are accepted; links, devices, duplicates and path escapes fail +before transfer. The archive must contain a non-empty `build.sh` at its root. It is +copied to `/var/root/hypeman-build-{id}` in the guest and run as root with +`/bin/sh build.sh`. + +Before stopping, the guest's build directory is removed and its absence verified, +and the host copy is removed. No registry token, host credential or secret is +injected. Guest stdout and stderr are discarded, so build logs never contain guest +output. Failures report the phase and, for the script, its exit code. + +## Stop, export and cleanup + +Publication is two instance operations. `StopMacOSBuildInstance` does a strict +graceful stop and rejects forced fallback, an instance already Stopped, non-macOS +guests, and any stop without shutdown acknowledgement and confirmed VMM exit. +`ExportStoppedMacOSBuildInstance` then copies the closed storage of a still-Stopped +instance under the instance lock, so a concurrent Start makes export fail. The output +is a private 0700 directory holding only `disk.img`, `aux.img` and `config.json`. + +Failure cleanup uses a forced stop and exports nothing. Storage is deleted only after +confirmed VMM exit; a VM with no confirmed exit stays quarantined for an operator. +Cleanup has its own 2-minute budget. A re-enqueued attempt first reaps the builder +left by the interrupted attempt, and fails rather than start a second one if that +delete fails. Builders orphaned outside this path have no general reaper. + +The builder is destroyed **before** publication. Source stays staged until a terminal +status is persisted, so a job interrupted while pushing is rebuilt on recovery. +Startup on a macOS-only server recovers machine jobs without preparing Linux builder +state, reaps cancelled builders and removes terminal-job source. Cancellation +interrupts the running job and keeps Cancelled terminal. Guest errors that may carry +secrets are masked in strings and logs. + +## Publication + +The publisher uses a server-configured registry and a token scoped to `builds/{id}`. +Plaintext is allowed only on loopback; redirects are never followed and proxies are +disabled. The layer is never written to disk: a first pass computes its digest, size +and DiffID, the upload regenerates the same bytes, and the registry rejects any +mismatch. Blobs use monolithic POST uploads, every persisted blob is re-read and +hashed, the manifest is committed last, and the digest content is verified. Ambiguous +errors never return success. + +A derivative built from a base in the same registry publishes a delta: the base's +layers are mounted by digest and only a layer of changed 64 KiB disk chunks is +uploaded. See `lib/images/macos_delta.go`. Bases outside the registry publish +standalone. There is no garbage-collection rule yet; deleting a base that a derivative +references breaks a later re-pull of that derivative. + +## Status and gates + +Race tests cover admission, the shared queue, cancellation without publication, phase +failures and quarantine, export validation, and authenticated publication with +manifest-last ordering. Test disks are synthetic and prove nothing about bootability. +A development base has been run end to end through the HTTP API on Apple silicon, +including a cold boot of the result; this trimmed branch has not been re-run natively. + +Before production use: a reviewed fresh base with approved artifact pins (base +publication tooling is a separate PR), cross-host publish and pull with sequential +cold boot, disk-space monitoring (this change has no disk admission, so run machine +builds serially on a dedicated host), SDK and CLI release, and Apple licence review. +Keep `macos_builds` off until then. + +**Rollback:** stop new submissions, let or cancel running jobs through the normal +APIs, repoint workloads to the prior validated digest, then set `macos_builds: false`. +Disabling does not delete persisted jobs or images. diff --git a/docs/macos-support.md b/docs/macos-support.md index 2ee42b466..13afe299e 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Partial | Opt-in `macos_builds`. Runs the source's build.sh as root in the guest. Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go new file mode 100644 index 000000000..6eb8e9ef9 --- /dev/null +++ b/lib/builds/macos_build.go @@ -0,0 +1,300 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// MachineBuildRequest is the internal backend contract for one machine build. +// The normal build manager owns ID, source staging and publish authorization. A concrete driver must require a provisioned system agent. +type MachineBuildRequest struct { + ID string + BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. + SourceHash string // Optional expected lowercase SHA256 of the source archive. + Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. + onStarted func(string) error // Normal manager records the owned builder before provisioning. + onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. + deltaBase *MachineDeltaBase // Set when the base is in the publisher's registry; publishes a derivative. +} + +type MachineStopReceipt = instances.MacOSBuildExportReceipt +type MachinePublication struct{ Reference, Digest string } +type MachineBuildResult struct { + Publication MachinePublication + Provenance BuildProvenance +} + +type MachineBuildDriver interface { + // Start owns an isolated instance and identity lease, named name. A partially + // created session must be returned even with an error so cleanup can stop it. + Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) +} + +type MachineBuildSession interface { + ID() string + Provision(context.Context, string) error // Build-owned staged source, never a client host path. + Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. + // Stop must await actual VMM exit. Only a strict graceful stop may produce a + // receipt that permits export. With force, a failed strict stop falls back to + // a forced stop, which is failure-only cleanup: nothing is exported after it. + Stop(ctx context.Context, force bool) (MachineStopReceipt, error) + Export(context.Context, string) error // Stopped disk.img, aux.img, config.json only. + // Destroy removes instance storage only after proven VMM exit, not the export. + Destroy(context.Context) error +} + +type MachineBuildPublisher interface { + // Publisher owns registry address, scoped token and builds/{id} destination. + // It must stream blobs and commit/verify the manifest last. An ambiguous + // network error is not a verified successful publication. + Publish(context.Context, string, string) (MachinePublication, error) +} + +type MachineBuildBackend struct { + Driver MachineBuildDriver + Publisher MachineBuildPublisher +} + +// machinePhaseError keeps potentially secret-bearing guest/command errors out of +// ordinary log/error text, while retaining errors.Is/As for internal handling. +type machinePhaseError struct { + phase string + cause error +} + +func (e *machinePhaseError) Error() string { + if exit := (*machineExitError)(nil); errors.As(e.cause, &exit) { + return "machine build " + e.phase + " failed: " + exit.Error() + } + // Host admission errors, such as a template identity already in use, carry + // no guest output. + if errors.Is(e.cause, instances.ErrInsufficientResources) || errors.Is(e.cause, instances.ErrInvalidState) { + return "machine build " + e.phase + " failed: " + e.cause.Error() + } + return "machine build " + e.phase + " failed" +} +func (e *machinePhaseError) Unwrap() error { return e.cause } +func machineFailure(phase string, err error) error { return &machinePhaseError{phase, err} } + +// machineExitError is the entrypoint's nonzero exit code. It holds no guest +// output, so it is safe to show. +type machineExitError struct{ code int } + +func (e *machineExitError) Error() string { + return fmt.Sprintf("%s exited with code %d", machineEntrypoint, e.code) +} + +// normalizeMachinePolicy applies machine build policy defaults and rejects what +// machine builds do not support. Admission, the runner and the guest driver share it. +func normalizeMachinePolicy(policy *BuildPolicy) error { + if policy.TimeoutSeconds == 0 { + policy.TimeoutSeconds = 600 + } + if policy.TimeoutSeconds < 1 || policy.TimeoutSeconds > 86400 { + return fmt.Errorf("invalid machine build timeout") + } + if policy.NetworkMode == "" { + policy.NetworkMode = "isolated" + } + if len(policy.AllowedDomains) != 0 || (policy.NetworkMode != "isolated" && policy.NetworkMode != "egress") { + return fmt.Errorf("unsupported machine network policy") + } + return nil +} + +// runPrepared executes only machine-specific phases. The shared build manager +// owns resolution, source staging/hash verification, deadline, queue and status. +func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath, workDir string) (result *MachineBuildResult, err error) { + if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(workDir) || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { + return nil, fmt.Errorf("machine driver, publisher, private workspace and verified source required") + } + if _, bounded := ctx.Deadline(); !bounded { + return nil, fmt.Errorf("prepared machine job requires a deadline") + } + if len(req.ID) == 0 || len(req.ID) > 128 || strings.Trim(req.ID, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" { + return nil, fmt.Errorf("invalid machine build ID") + } + ref, e := images.ParseNormalizedRef(req.BaseImage) + if e != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := req.Policy + if e = normalizeMachinePolicy(&policy); e != nil { + return nil, e + } + if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { + return nil, fmt.Errorf("base is not the pinned ready macOS machine") + } + if base.MacOS.Validate() != nil { + return nil, fmt.Errorf("base has invalid machine identity") + } + memory := int(base.MacOS.Memory >> 20) + cpus := int(base.MacOS.CPUs) + if (policy.MemoryMB != 0 && policy.MemoryMB != memory) || (policy.CPUs != 0 && policy.CPUs != cpus) { + return nil, fmt.Errorf("machine resources must match installed base") + } + policy.MemoryMB = memory + policy.CPUs = cpus + if e = policy.Validate(); e != nil { + return nil, e + } + workspace, e := os.MkdirTemp(workDir, "machine-build-") + if e != nil { + return nil, machineFailure("workspace", e) + } + defer func() { + if cleanupErr := os.RemoveAll(workspace); cleanupErr != nil { + err = errors.Join(err, machineFailure("cleanup_workspace", cleanupErr)) + result = nil + } + }() + if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { + return nil, machineFailure("verify_source", e) + } + if e = ctx.Err(); e != nil { + return nil, e + } + session, e := r.Driver.Start(ctx, machineBuilderName(req.ID), base, policy) + if session == nil { + if e == nil { + e = fmt.Errorf("driver returned no session") + } + return nil, machineFailure("start", e) + } + stopped, destroyed := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), machineCleanupTimeout) + defer cancel() + if !stopped { + receipt, stopErr := session.Stop(cleanup, true) + stopped = receipt.VMMExited + if !stopped { + err = errors.Join(err, machineFailure("quarantine_unconfirmed_vmm_exit", stopErr)) + result = nil + return + } + } + if !destroyed { + if destroyErr := session.Destroy(cleanup); destroyErr != nil { + err = errors.Join(err, machineFailure("cleanup", destroyErr)) + result = nil + } + } + }() + instanceID := session.ID() + if req.onStarted != nil { + if recordErr := req.onStarted(instanceID); recordErr != nil { + return nil, machineFailure("record_builder", recordErr) + } + } + if e != nil { + return nil, machineFailure("start", e) + } + if e = session.Provision(ctx, sourcePath); e != nil { + return nil, machineFailure("provision", e) + } + if e = session.Sanitize(ctx); e != nil { + return nil, machineFailure("sanitize", e) + } + receipt, e := session.Stop(ctx, false) + stopped = receipt.VMMExited + if e != nil || !receipt.Graceful || !stopped { + return nil, machineFailure("graceful_stop", e) + } + bundle := filepath.Join(workspace, "bundle") + if e = os.Mkdir(bundle, 0700); e != nil { + return nil, machineFailure("export", e) + } + if e = session.Export(ctx, bundle); e != nil { + return nil, machineFailure("export", e) + } + if e = validateMachineExport(bundle, base.MacOS); e != nil { + return nil, machineFailure("validate_export", e) + } + if e = session.Destroy(ctx); e != nil { + return nil, machineFailure("cleanup", e) + } + destroyed = true + if e = ctx.Err(); e != nil { + return nil, e + } + publication, e := r.publishBuild(ctx, req, bundle) + if e != nil { + return nil, machineFailure("publish", e) + } + output, e := images.ParseNormalizedRef(publication.Reference) + if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { + return nil, fmt.Errorf("machine publication receipt is unverified") + } + if req.onPublished != nil { + if e = req.onPublished(ctx, publication); e != nil { + return nil, machineFailure("image_ready", e) + } + } + provenance := BuildProvenance{BaseImageDigest: base.Digest, SourceHash: req.SourceHash, Timestamp: time.Now().UTC()} + return &MachineBuildResult{Publication: publication, Provenance: provenance}, nil +} + +// machineCleanupTimeout covers a strict graceful stop attempt (30s power-off +// budget) plus forced fallback and storage deletion. Export is outside cleanup. +const machineCleanupTimeout = 2 * time.Minute + +// ValidMachineDigest reports whether s is a canonical lowercase sha256 digest. +// It is the single validation rule for machine base and output digests. +func ValidMachineDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +func validateMachineExport(root string, base *images.MacOSImage) error { + directory, err := os.Lstat(root) + if err != nil || !directory.IsDir() { + return fmt.Errorf("export root must be a real directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return err + } + if len(entries) != 3 { + return fmt.Errorf("export must contain only machine payload files") + } + for _, entry := range entries { + if entry.Name() != "disk.img" && entry.Name() != "aux.img" && entry.Name() != "config.json" { + return fmt.Errorf("unexpected export file") + } + info, err := os.Lstat(filepath.Join(root, entry.Name())) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("export files must be regular") + } + } + platform, err := images.ValidateMacOSBundle(root) + if err != nil { + return err + } + if base != nil && (!bytes.Equal(platform.HardwareModel, base.HardwareModel) || !bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || !strings.EqualFold(platform.MAC, base.MAC) || platform.CPUs != base.CPUs || platform.Memory != base.Memory) { + return fmt.Errorf("machine export changed preserved identity/resources") + } + for _, entry := range entries { + if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { + return err + } + } + return nil +} + +// machineBuilderName names a build's builder instance after the build, so a retry +// can find and reap a builder that an interrupted attempt created but never recorded. +func machineBuilderName(buildID string) string { + return "macos-build-" + strings.ToLower(buildID) +} diff --git a/lib/builds/macos_build_cancel.go b/lib/builds/macos_build_cancel.go new file mode 100644 index 000000000..09b984000 --- /dev/null +++ b/lib/builds/macos_build_cancel.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "fmt" + "sync" + "time" +) + +type machineBuildControl struct { + mu sync.Mutex + cancel context.CancelFunc +} + +func (m *manager) recordMachineBuilder(id, instanceID string) error { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status == StatusCancelled { + return context.Canceled + } + meta.BuilderInstance = &instanceID + return writeMetadata(m.paths, meta) +} + +func (m *manager) cancelQueuedMachineBuild(meta *buildMetadata) error { + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + m.removeMachineSource(meta.ID) + m.notifyStatusChange(meta.ID, StatusCancelled) + return nil +} + +func (m *manager) cancelMachineBuild(id string) error { + active, ok := m.machineBuilds.Load(id) + if !ok { + return ErrBuildInProgress + } + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status != StatusBuilding && meta.Status != StatusPushing { + return fmt.Errorf("machine build no longer running") + } + // Persist cancellation before interrupting phases. The backend performs bounded + // graceful cleanup or quarantine; Cancel must not force-delete mutable storage. + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + control.cancel() + m.notifyStatusChange(id, StatusCancelled) + return nil +} diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go new file mode 100644 index 000000000..c0e4e6e0f --- /dev/null +++ b/lib/builds/macos_build_failure_test.go @@ -0,0 +1,198 @@ +package builds + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func runMachineBuildFailure(t *testing.T, f *machineFixture, ctx context.Context, req MachineBuildRequest, source io.Reader) (*machineRunnerFixture, error) { + t.Helper() + runner := machineTestRunner(t, f) + result, err := runner.Run(ctx, req, source) + if err == nil || result != nil { + t.Fatal("failed build returned a result") + } + return runner, err +} + +type failingMachineSource struct{} + +func (failingMachineSource) Read(p []byte) (int, error) { + return copy(p, "partial"), errors.New("secret-source-error") +} + +type cancelMachineSource struct{ cancel context.CancelFunc } + +func (s cancelMachineSource) Read(p []byte) (int, error) { + s.cancel() + return copy(p, "source"), io.EOF +} + +func TestMachineBuildSourceFailureDoesNotStart(t *testing.T) { + for _, tc := range []struct { + name string + source func(context.CancelFunc) io.Reader + wantCancel bool + }{ + {name: "read failure", source: func(context.CancelFunc) io.Reader { return failingMachineSource{} }}, + {name: "canceled", source: func(cancel context.CancelFunc) io.Reader { return cancelMachineSource{cancel} }, wantCancel: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + r, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), tc.source(cancel)) + if tc.wantCancel { + require.ErrorIs(t, err, context.Canceled) + } else { + require.NotContains(t, err.Error(), "secret-source-error", "source failure was not masked") + } + require.Empty(t, f.calls, "source failure started guest") + entries, err := os.ReadDir(r.WorkDir) + require.NoError(t, err) + require.Empty(t, entries, "partial source retained") + }) + } +} + +// Cancellation and timeout both interrupt a running build. Cleanup must still +// run to completion on a live context, and nothing may publish. +func TestMachineBuildInterruptedCleanup(t *testing.T) { + assertCleanup := func(t *testing.T, f *machineFixture, err, want error) { + t.Helper() + require.ErrorIs(t, err, want) + require.False(t, f.publisherCalled, "interrupted build published") + require.True(t, f.cleanupContextLive, "cleanup ran on a cancelled context") + require.Equal(t, []string{"start", "abort", "destroy"}, f.calls) + } + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + f.cancel = cancel + _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), strings.NewReader("source")) + assertCleanup(t, f, err, context.Canceled) + }) + t.Run("timeout", func(t *testing.T) { + f := newMachineFixture() + f.waitForCancel = true + req := machineTestRequest() + req.Policy.TimeoutSeconds = 1 + parent, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := runMachineBuildFailure(t, f, parent, req, strings.NewReader("source")) + assertCleanup(t, f, err, context.DeadlineExceeded) + }) +} + +func TestMachineBuildPublicationReceipt(t *testing.T) { + for _, kind := range []string{"ambiguous error", "tag receipt", "digest mismatch", "invalid digest"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + switch kind { + case "ambiguous error": + f.fail = "publish" + case "tag receipt": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test:latest", Digest: machineTestDigest} + case "digest mismatch": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "sha256:" + strings.Repeat("b", 64)} + case "invalid digest": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "not-a-digest"} + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if !f.publisherCalled { + t.Fatal("unverified receipt became ready") + } + if strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("publisher error leaked") + } + count := 0 + for _, phase := range f.calls { + if phase == "publish" { + count++ + } + } + if count != 1 { + t.Fatal("ambiguous publication was retried") + } + }) + } +} + +func TestMachineBuildMalformedExport(t *testing.T) { + for _, kind := range []string{"empty disk", "empty aux", "symlink disk", "hardlink payloads", "truncated config", "oversized config", "resource change", "bad MAC", "identity", "extra file", "unknown metadata"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + f.mutateExport = func(root string) error { + disk, aux, config := filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"), filepath.Join(root, "config.json") + switch kind { + case "empty disk": + return os.Truncate(disk, 0) + case "empty aux": + return os.Truncate(aux, 0) + case "symlink disk": + if err := os.Remove(disk); err != nil { + return err + } + return os.Symlink(aux, disk) + case "hardlink payloads": + if err := os.Remove(aux); err != nil { + return err + } + return os.Link(disk, aux) + case "truncated config": + return os.WriteFile(config, []byte(`{"hardware_model":`), 0600) + case "oversized config": + return os.WriteFile(config, []byte(strings.Repeat(" ", 65<<10)), 0600) + case "resource change": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), `"cpus":4`, `"cpus":2`, 1)), 0600) + case "bad MAC": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:01", "02:00:00:00:00:01:02:03", 1)), 0600) + case "identity", "unknown metadata": + var raw map[string]any + data, err := os.ReadFile(config) + if err == nil { + err = json.Unmarshal(data, &raw) + } + if err != nil { + return err + } + if kind == "identity" { + raw["machine_identifier"] = []byte{9} + } else { + raw["password"] = "must-not-publish" + } + data, _ = json.Marshal(raw) + return os.WriteFile(config, data, 0600) + case "extra file": + return os.WriteFile(filepath.Join(root, "secret.txt"), []byte("must-not-publish"), 0600) + } + return nil + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("malformed export published") + } + if strings.Contains(err.Error(), "must-not-publish") { + t.Fatal("export secret leaked in failure") + } + }) + } +} diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go new file mode 100644 index 000000000..ebf341318 --- /dev/null +++ b/lib/builds/macos_build_guest.go @@ -0,0 +1,185 @@ +package builds + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +type machineExportManager interface { + StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) + ExportStoppedMacOSBuildInstance(context.Context, string, string) error +} + +// GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system +// GuestService. It never installs an agent or silently upgrades an unmanaged base. +// The backend validates the base and policy before Start. +type GuestMachineBuildDriver struct { + Instances instances.Manager + WorkDir string +} + +func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { + exporter, ok := d.Instances.(machineExportManager) + if !ok || !filepath.IsAbs(d.WorkDir) { + return nil, fmt.Errorf("strict machine export and private workspace required") + } + if _, ok = ctx.Deadline(); !ok { + return nil, fmt.Errorf("machine start requires a deadline") + } + ref, err := images.ParseNormalizedRef(base.Name) + if err != nil { + return nil, err + } + // Workspaces are named per builder, so one left by an interrupted attempt + // of this job is removed here. + prefix := "guest-machine-" + name + "-" + stale, _ := filepath.Glob(filepath.Join(d.WorkDir, prefix+"*")) + for _, dir := range stale { + _ = os.RemoveAll(dir) + } + dir, err := os.MkdirTemp(d.WorkDir, prefix) + if err != nil { + return nil, err + } + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + if inst == nil { + _ = os.RemoveAll(dir) + if err == nil { + err = fmt.Errorf("instance creation returned no instance") + } + return nil, err + } + session := &guestMachineSession{manager: d.Instances, exporter: exporter, instance: inst, workspace: dir, guestDir: "/var/root/hypeman-build-" + inst.Id} + return session, err +} + +type guestMachineSession struct { + manager instances.Manager + exporter machineExportManager + instance *instances.Instance + workspace, guestDir string + receipt MachineStopReceipt + sanitized bool +} + +func (s *guestMachineSession) ID() string { return s.instance.Id } +func (s *guestMachineSession) dialer() (hypervisor.VsockDialer, error) { + return hypervisor.NewVsockDialer(s.instance.HypervisorType, s.instance.VsockSocket, s.instance.VsockCID) +} +func (s *guestMachineSession) exec(ctx context.Context, command []string, cwd string, env map[string]string, wait time.Duration) error { + dialer, err := s.dialer() + if err != nil { + return err + } + // Drain output without retaining or publishing it. Redaction cannot reliably + // make arbitrary guest output safe: even successful commands may print secrets. + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{Command: command, Cwd: cwd, Env: env, WaitForAgent: wait, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + return err + } + if exit == nil { + return fmt.Errorf("machine guest command did not succeed") + } + if exit.Code != 0 { + return &machineExitError{code: exit.Code} + } + return nil +} + +func (s *guestMachineSession) Provision(ctx context.Context, source string) error { + root := filepath.Join(s.workspace, "source") + if err := extractMachineSource(ctx, source, root); err != nil { + return err + } + err := s.exec(ctx, []string{"/bin/sh", "-c", `test "$(/usr/bin/id -u)" -eq 0`}, "", nil, 30*time.Second) + if err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 30*time.Second); err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/mkdir", "-m", "700", s.guestDir}, "", nil, 0); err != nil { + return err + } + dialer, err := s.dialer() + if err != nil { + return err + } + if err = guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{SrcPath: root, DstPath: s.guestDir}); err != nil { + return err + } + return s.exec(ctx, []string{"/bin/sh", machineEntrypoint}, s.guestDir, nil, 0) +} + +func (s *guestMachineSession) Sanitize(ctx context.Context) error { + if err := s.exec(ctx, []string{"/bin/rm", "-rf", "--", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(s.workspace, "source")); err != nil { + return err + } + s.sanitized = true + return nil +} + +// Stop is the publication path: only an acknowledged graceful power-off with +// confirmed VMM exit may produce a receipt that permits export. With force, a +// failed strict stop falls back to the ordinary forced hypervisor stop. Nothing +// from a failed build is exported, and Destroy still requires confirmed VMM exit, +// so a non-exited instance remains quarantined. +func (s *guestMachineSession) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + if s.receipt.VMMExited { + return s.receipt, nil + } + receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()) + if !force { + s.receipt = receipt + return s.receipt, err + } + if err == nil && receipt.VMMExited { + s.receipt = receipt + return s.receipt, nil + } + if inst, err := s.manager.GetInstance(ctx, s.ID()); err != nil || inst.State != instances.StateStopped { + inst, err = s.manager.StopInstance(ctx, s.ID()) + if err != nil { + return s.receipt, err + } + if inst == nil || inst.State != instances.StateStopped { + return s.receipt, fmt.Errorf("forced stop did not confirm VMM exit") + } + } + s.receipt = MachineStopReceipt{VMMExited: true} + return s.receipt, nil +} + +// Export copies only from closed storage of a Stopped instance, under the +// instance lock, after a strict graceful receipt and sanitation. +func (s *guestMachineSession) Export(ctx context.Context, destination string) error { + if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { + return fmt.Errorf("sanitized strict stop receipt required") + } + return s.exporter.ExportStoppedMacOSBuildInstance(ctx, s.ID(), destination) +} + +func (s *guestMachineSession) Destroy(ctx context.Context) error { + if !s.receipt.VMMExited { + return fmt.Errorf("unconfirmed VMM exit: instance quarantined") + } + if err := s.manager.DeleteInstance(ctx, s.ID()); err != nil { + return err + } + return os.RemoveAll(s.workspace) +} diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go new file mode 100644 index 000000000..12394b213 --- /dev/null +++ b/lib/builds/macos_build_job_test.go @@ -0,0 +1,37 @@ +package builds + +import ( + "context" + "io" + "os" + "path/filepath" + "time" +) + +type machineRunnerFixture struct { + MachineBuildBackend + WorkDir string +} + +func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, source io.Reader) (*MachineBuildResult, error) { + timeout := req.Policy.TimeoutSeconds + if timeout == 0 { + timeout = 600 + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + job, err := os.MkdirTemp(r.WorkDir, "prepared-job-") + if err != nil { + return nil, err + } + defer os.RemoveAll(job) + sourcePath := filepath.Join(job, "source.tar.gz") + hash, err := stageBuildSource(ctx, source, sourcePath, maxBuildSourceBytes) + if err != nil { + return nil, machineFailure("stage_source", err) + } + if req.SourceHash == "" { + req.SourceHash = hash + } + return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath, r.WorkDir) +} diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go new file mode 100644 index 000000000..6798c5582 --- /dev/null +++ b/lib/builds/macos_build_layer_test.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { + parent := t.TempDir() + bundle := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + disk := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(disk[1<<20:], "data") + for name, data := range map[string][]byte{"disk.img": disk, "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) + for range 2 { + body, err := layer.Compressed() + require.NoError(t, err) + sum := sha256.New() + n, err := io.Copy(sum, body) + require.NoError(t, err) + require.NoError(t, body.Close()) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + } + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing is spooled beside the bundle") +} + +func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { + bundle := t.TempDir() + for name, data := range map[string][]byte{"disk.img": make([]byte, 8<<20), "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + before := runtime.NumGoroutine() + body, err := layer.Compressed() + require.NoError(t, err) + _, err = io.CopyN(io.Discard, body, 16) + require.NoError(t, err) + require.NoError(t, body.Close()) + // Polled directly: require.Eventually runs its condition in another goroutine. + deadline := time.Now().Add(5 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), before, "the layer generator exits when the upload is abandoned") +} diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go new file mode 100644 index 000000000..f7359e95c --- /dev/null +++ b/lib/builds/macos_build_lifecycle_test.go @@ -0,0 +1,135 @@ +package builds + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineLifecycleInstances struct { + instances.Manager + lookedUp string + state, stoppedState instances.State + missing, stopped bool + deleted bool + stopErr, deleteErr error +} + +func (m *machineLifecycleInstances) GetInstance(_ context.Context, id string) (*instances.Instance, error) { + m.lookedUp = id + if m.missing { + return nil, instances.ErrNotFound + } + return &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "prior"}, State: m.state}, nil +} +func (m *machineLifecycleInstances) StopInstance(context.Context, string) (*instances.Instance, error) { + m.stopped = true + if m.stopErr != nil { + return nil, m.stopErr + } + if m.stoppedState != "" { + m.state = m.stoppedState + } + return &instances.Instance{State: m.state}, nil +} +func (m *machineLifecycleInstances) DeleteInstance(context.Context, string) error { + m.deleted = true + return m.deleteErr +} + +type abortExporter struct { + receipt instances.MacOSBuildExportReceipt + err error +} + +func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { + return e.receipt, e.err +} +func (abortExporter) ExportStoppedMacOSBuildInstance(context.Context, string, string) error { + return errors.New("abort must not export") +} + +func TestMachineForcedStopCleanup(t *testing.T) { + for _, tc := range []struct { + name string + state, stoppedState instances.State + graceful, forceError, wantExit bool + }{ + {"graceful exit", instances.StateRunning, "", true, false, true}, + {"already stopped", instances.StateStopped, "", false, false, true}, + {"force running VM", instances.StateRunning, instances.StateStopped, false, false, true}, + {"unconfirmed stop", instances.StateRunning, "", false, false, false}, + {"failed force stop", instances.StateRunning, "", false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := &machineLifecycleInstances{state: tc.state, stoppedState: tc.stoppedState} + stop := MachineStopReceipt{Graceful: tc.graceful, VMMExited: tc.graceful} + var stopErr error + if !tc.graceful { + stopErr = errors.New("agent never became ready") + } + if tc.forceError { + m.stopErr = errors.New("hypervisor still alive") + } + s := &guestMachineSession{manager: m, exporter: abortExporter{receipt: stop, err: stopErr}, instance: &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "builder"}}} + receipt, err := s.Stop(context.Background(), true) + if tc.wantExit { + require.NoError(t, err) + require.Equal(t, MachineStopReceipt{Graceful: tc.graceful, VMMExited: true}, receipt) + } else { + wantErr := "forced stop did not confirm VMM exit" + if tc.forceError { + wantErr = "hypervisor still alive" + } + require.EqualError(t, err, wantErr) + require.False(t, receipt.VMMExited) + require.Error(t, s.Destroy(context.Background()), "unconfirmed exit must remain quarantined") + } + require.Equal(t, !tc.graceful && tc.state != instances.StateStopped, m.stopped) + }) + } +} + +func TestReapPriorMachineBuilder(t *testing.T) { + for _, tc := range []struct { + name string + state instances.State + missing bool + unrecorded bool + deleteErr error + wantStop bool + wantDelete bool + wantError bool + }{ + {"stop and delete", instances.StateRunning, false, false, nil, true, true, false}, + {"delete failure blocks retry", instances.StateStopped, false, false, errors.New("storage still open"), false, true, true}, + {"missing builder", "", true, false, nil, false, false, false}, + {"find unrecorded builder", instances.StateRunning, false, true, nil, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := paths.New(t.TempDir()) + meta := &buildMetadata{ID: "job", Status: StatusQueued} + if !tc.unrecorded { + prior := "prior" + meta.BuilderInstance = &prior + } + require.NoError(t, writeMetadata(p, meta)) + fake := &machineLifecycleInstances{state: tc.state, missing: tc.missing, deleteErr: tc.deleteErr} + m := &manager{paths: p, instanceManager: fake, logger: slog.Default()} + err := m.reapPriorMachineBuilder(context.Background(), "job") + if tc.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.unrecorded, fake.lookedUp == machineBuilderName("job")) + require.Equal(t, tc.wantStop, fake.stopped) + require.Equal(t, tc.wantDelete, fake.deleted) + }) + } +} diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go new file mode 100644 index 000000000..dcd5be191 --- /dev/null +++ b/lib/builds/macos_build_manager.go @@ -0,0 +1,199 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// machineBuild reports whether a request selects the native macOS build path. +func machineBuild(req *CreateBuildRequest) bool { + return req != nil && req.MachineBaseImage != "" +} + +func (m *manager) skipMachineBuildRecovery(meta *buildMetadata) bool { + if !m.config.MachineBuildOnly { + return false + } + if machineBuild(meta.Request) && m.config.MachineBuild != nil { + return false + } + m.logger.Warn("leaving unsupported build pending in machine-build-only mode", "id", meta.ID) + return true +} + +// executeMachineBuild is a backend of the same queued build job. runBuild owns +// deadline/status/log completion, and CreateBuild already staged verified source. +func (m *manager) executeMachineBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + backend := m.config.MachineBuild + if backend == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if err := m.reapPriorMachineBuilder(ctx, id); err != nil { + return nil, machineFailure("reap_prior_builder", err) + } + base, err := m.imageManager.GetImage(ctx, req.MachineBaseImage) + if err != nil { + return nil, machineFailure("resolve_base", err) + } + sourcePath, err := filepath.Abs(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")) + if err != nil { + return nil, err + } + // An interrupted attempt skips its own cleanup; drop its workspace first. + removeMachineWorkspaces(m.paths.BuildDir(id)) + defer removeMachineWorkspaces(m.paths.BuildDir(id)) + job := MachineBuildRequest{ + ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, + deltaBase: m.machineDeltaBase(backend.Publisher, req.MachineBaseImage, base), + onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, + onPublished: func(ctx context.Context, publication MachinePublication) error { + return m.waitMachinePublication(ctx, id, publication.Digest) + }, + } + result, err := backend.runPrepared(ctx, job, base, sourcePath, filepath.Dir(filepath.Dir(sourcePath))) + if err != nil { + return nil, err + } + ref, err := images.ParseNormalizedRef(result.Publication.Reference) + if err != nil || ref.Repository() != strings.TrimSuffix(stripRegistryScheme(m.config.RegistryURL), "/")+"/builds/"+id { + return nil, fmt.Errorf("machine publication is outside the authorized job repository") + } + return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil +} + +func (m *manager) waitMachinePublication(ctx context.Context, id, digest string) error { + if !ValidMachineDigest(digest) { + return fmt.Errorf("canonical machine publication digest required") + } + if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+digest); err != nil { + return fmt.Errorf("machine image import: %w", err) + } + return nil +} + +// reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of +// this same job before a re-enqueued attempt starts. Nothing from that builder is +// published, so a forced stop is acceptable; a failed delete fails this attempt +// rather than leaving two builders with storage in play. +func (m *manager) reapPriorMachineBuilder(ctx context.Context, id string) error { + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + // An attempt interrupted before it recorded its builder still named it after the build. + prior := machineBuilderName(id) + if meta.BuilderInstance != nil && *meta.BuilderInstance != "" { + prior = *meta.BuilderInstance + } + inst, err := m.instanceManager.GetInstance(ctx, prior) + if errors.Is(err, instances.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if inst.State != instances.StateStopped { + if _, err := m.instanceManager.StopInstance(ctx, inst.Id); err != nil { + return fmt.Errorf("stop prior builder: %w", err) + } + } + if err := m.instanceManager.DeleteInstance(ctx, inst.Id); err != nil && !errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("delete prior builder: %w", err) + } + m.logger.Info("reaped prior machine builder", "build_id", id, "instance_id", inst.Id) + return nil +} + +// storeMachineSource stages machine source privately and records its SHA256 +// provenance. A caller-supplied source hash must match. +func (m *manager) storeMachineSource(ctx context.Context, meta *buildMetadata, data []byte) error { + sourceDir := m.paths.BuildSourceDir(meta.ID) + if err := os.MkdirAll(sourceDir, 0700); err != nil { + return fmt.Errorf("store source: %w", err) + } + hash, err := stageBuildSource(ctx, bytes.NewReader(data), filepath.Join(sourceDir, "source.tar.gz"), int64(len(data))) + if err != nil { + return fmt.Errorf("store source: %w", err) + } + if meta.Request.SourceHash != "" && meta.Request.SourceHash != hash { + return ErrSourceHashMismatch + } + meta.Request.SourceHash = hash + if err := writeMetadata(m.paths, meta); err != nil { + return fmt.Errorf("write source provenance: %w", err) + } + return nil +} + +// admitMachineBuild validates a machine build request and returns its policy. +// Machine resources inherit the installed base, not Linux builder defaults. +func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) (*BuildPolicy, error) { + if m.config.MachineBuild == nil || m.config.MachineBuild.Driver == nil || m.config.MachineBuild.Publisher == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if req.Dockerfile != "" || req.BuilderID != "" || len(req.BuildArgs) != 0 || len(req.Secrets) != 0 || req.CacheScope != "" || req.GlobalCacheKey != "" || req.IsAdminBuild { + return nil, fmt.Errorf("machine builds do not support Linux builder/cache/secret options") + } + if len(sourceData) > maxBuildSourceBytes { + return nil, fmt.Errorf("machine build source too large") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := &BuildPolicy{} + if req.BuildPolicy != nil { + requested := *req.BuildPolicy + policy = &requested + } + if err := normalizeMachinePolicy(policy); err != nil { + return nil, err + } + req.BuildPolicy = policy + return policy, nil +} + +// enqueueMachineBuild queues a machine build whose source is already staged. +func machineBuildSerialKey(req CreateBuildRequest) string { + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() { + return "machine:" + req.MachineBaseImage + } + return "machine:" + ref.Digest() +} + +func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { + queuePos := m.queue.EnqueueSerial(meta.ID, req, machineBuildSerialKey(req), func() { + m.runBuild(context.Background(), meta.ID, req, policy) + }) + build := meta.toBuild() + if queuePos > 0 { + build.QueuePosition = &queuePos + } + m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) + return build +} + +// removeMachineWorkspaces deletes export workspaces under a build directory. Each +// holds a full exported disk. +func removeMachineWorkspaces(buildDir string) { + matches, _ := filepath.Glob(filepath.Join(buildDir, "machine-build-*")) + for _, dir := range matches { + _ = os.RemoveAll(dir) + } +} + +// removeMachineSource deletes a machine build's staged source after the job is terminal. +func (m *manager) removeMachineSource(id string) { + if err := os.Remove(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")); err != nil && !errors.Is(err, os.ErrNotExist) { + m.logger.Warn("failed to remove machine build source", "id", id, "error", err) + } +} diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go new file mode 100644 index 000000000..fdc2cab7d --- /dev/null +++ b/lib/builds/macos_build_manager_test.go @@ -0,0 +1,228 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineJobImages struct { + images.Manager + fixture *machineFixture + resolveErr error +} + +func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Image, error) { + if m.resolveErr != nil { + return nil, m.resolveErr + } + if ref == "registry/base@"+machineTestDigest { + return m.fixture.ResolveBase(ctx, ref) + } + return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil +} + +func (m machineJobImages) WaitForReady(context.Context, string) error { return nil } + +func newMachineTestManager(t *testing.T, f *machineFixture, publisher MachineBuildPublisher, imageManager images.Manager, machineOnly bool) *manager { + t.Helper() + config := DefaultConfig() + config.RegistrySecret = "synthetic-test-secret" + config.MachineBuildOnly = machineOnly + if f != nil && publisher != nil { + config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: publisher} + } + if imageManager == nil && f != nil { + imageManager = machineJobImages{fixture: f} + } + api, err := NewManager(paths.New(t.TempDir()), config, noPriorBuilder{}, nil, nil, imageManager, nil, nil, nil) + require.NoError(t, err) + return api.(*manager) +} + +type machineJobPublisher struct { + fixture *machineFixture + publishedID string + // sourcePath, when set, must still exist at publish time. + sourcePath func(id string) string +} + +func (p *machineJobPublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + p.publishedID = id + if p.sourcePath != nil { + if _, err := os.Stat(p.sourcePath(id)); err != nil { + return MachinePublication{}, errors.New("source removed before publish") + } + } + // Reuse the fixture's export assertions with its fixed ID. + receipt, err := p.fixture.Publish(ctx, "build-test", bundle) + if err == nil { + receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest + } + return receipt, err +} +func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { + f := newMachineFixture() + publisher := &machineJobPublisher{fixture: f} + m := newMachineTestManager(t, f, publisher, nil, false) + publisher.sourcePath = func(id string) string { return filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz") } + source := []byte("synthetic compressed-source bytes") + sum := sha256.Sum256(source) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: hex.EncodeToString(sum[:])} + build, err := m.CreateBuild(context.Background(), req, source) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && (b.Status == StatusReady || b.Status == StatusFailed) + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, done.Status, "failure: %v", done.Error) + require.NotNil(t, done.ImageRef) + require.Equal(t, "localhost:4973/builds/"+build.ID+"@"+machineTestDigest, *done.ImageRef) + require.Equal(t, build.ID, publisher.publishedID) + require.Equal(t, req.SourceHash, done.Provenance.SourceHash) + require.Equal(t, machineTestDigest, done.Provenance.BaseImageDigest) + require.Equal(t, f.ID(), *done.BuilderInstanceID) + meta, err := readMetadata(m.paths, build.ID) + require.NoError(t, err) + require.Equal(t, req.MachineBaseImage, meta.Request.MachineBaseImage) + require.Equal(t, "isolated", meta.Request.BuildPolicy.NetworkMode) + require.Equal(t, 0, meta.Request.BuildPolicy.CPUs, "machine defaults must not use Linux CPU defaults") + _, err = readBuildConfig(m.paths, build.ID) + require.Error(t, err, "machine builds must not persist a builder config or registry token") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) +} +func TestMachineBuildSerializesByBaseDigest(t *testing.T) { + first := CreateBuildRequest{MachineBaseImage: "registry-a/base@" + machineTestDigest} + second := CreateBuildRequest{MachineBaseImage: "registry-b/another-name@" + machineTestDigest} + require.Equal(t, machineBuildSerialKey(first), machineBuildSerialKey(second)) +} + +func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, false) + _, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.ErrorContains(t, err, "not configured") + f := newMachineFixture() + m.config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: f} + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base:mutable"}, []byte("source")) + require.ErrorContains(t, err, "pinned") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, Secrets: []SecretRef{{ID: "unsupported"}}}, []byte("source")) + require.ErrorContains(t, err, "secret options") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: "wrong"}, []byte("source")) + require.True(t, errors.Is(err, ErrSourceHashMismatch)) + _, err = m.admitMachineBuild(&CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, make([]byte, maxBuildSourceBytes+1)) + require.ErrorContains(t, err, "source too large") + list, err := m.ListBuilds(context.Background()) + require.NoError(t, err) + require.Empty(t, list) +} + +func TestQueuedMachineCancellationRemovesSource(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) + require.NoError(t, err) + source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") + require.FileExists(t, source) + require.NoError(t, m.CancelBuild(context.Background(), queued.ID)) + done, err := m.GetBuild(context.Background(), queued.ID) + require.NoError(t, err) + require.Equal(t, StatusCancelled, done.Status) + require.NotNil(t, done.CompletedAt) + require.NoFileExists(t, source) +} + +func TestMachineCancellationInterruptsPhasesWithoutPublishing(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + build, e := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("synthetic source")) + require.NoError(t, e) + require.Eventually(t, func() bool { + meta, e := readMetadata(m.paths, build.ID) + return e == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), build.ID)) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(build.ID); return !active }, 5*time.Second, 10*time.Millisecond) + done, e := m.GetBuild(context.Background(), build.ID) + require.NoError(t, e) + require.Equal(t, StatusCancelled, done.Status) + require.False(t, fixture.publisherCalled) + require.Contains(t, fixture.calls, "abort") + require.Contains(t, fixture.calls, "destroy") +} + +func TestMachineFailureUsesSharedFailedStatusWithoutPublication(t *testing.T) { + for _, tc := range []struct { + name string + phase string + resolveErr error + wantError string + }{ + {name: "base resolution", resolveErr: errors.New("sensitive-resolution-error"), wantError: "machine build resolve_base failed"}, + {name: "provision", phase: "provision", wantError: "machine build provision failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + m := newMachineTestManager(t, f, f, machineJobImages{fixture: f, resolveErr: tc.resolveErr}, false) + build, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && b.Status == StatusFailed + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusFailed, done.Status) + require.Nil(t, done.ImageDigest) + require.NotContains(t, f.calls, "publish") + require.NotContains(t, *done.Error, "sensitive-resolution-error") + require.Equal(t, tc.wantError, *done.Error) + if tc.phase == "provision" { + require.Contains(t, f.calls, "destroy") + } else { + require.Empty(t, f.calls) + } + }) + } +} + +func TestMachineFailureShowsOnlyExitCode(t *testing.T) { + err := machineFailure("provision", fmt.Errorf("guest said secret-token: %w", &machineExitError{code: 7})) + require.Equal(t, "machine build provision failed: build.sh exited with code 7", err.Error()) + require.Equal(t, "machine build provision failed", machineFailure("provision", fmt.Errorf("secret-token")).Error()) + admission := fmt.Errorf("%w: macOS template identity is already in use", instances.ErrInsufficientResources) + require.Contains(t, machineFailure("start", admission).Error(), "machine build start failed: insufficient resources") +} + +// noPriorBuilder reports that no earlier attempt left a builder instance behind. +type noPriorBuilder struct{ instances.Manager } + +func (noPriorBuilder) GetInstance(context.Context, string) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} diff --git a/lib/builds/macos_build_policy_test.go b/lib/builds/macos_build_policy_test.go new file mode 100644 index 000000000..32c1592f6 --- /dev/null +++ b/lib/builds/macos_build_policy_test.go @@ -0,0 +1,44 @@ +package builds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachinePolicyNormalization(t *testing.T) { + for _, tc := range []struct { + name string + policy BuildPolicy + err string + }{ + {name: "defaults"}, + {name: "minimum timeout", policy: BuildPolicy{TimeoutSeconds: 1}}, + {name: "maximum timeout", policy: BuildPolicy{TimeoutSeconds: 86400}}, + {name: "egress", policy: BuildPolicy{NetworkMode: "egress"}}, + {name: "negative timeout", policy: BuildPolicy{TimeoutSeconds: -1}, err: "invalid machine build timeout"}, + {name: "excessive timeout", policy: BuildPolicy{TimeoutSeconds: 86401}, err: "invalid machine build timeout"}, + {name: "unsupported mode", policy: BuildPolicy{NetworkMode: "host"}, err: "unsupported machine network policy"}, + {name: "domains", policy: BuildPolicy{NetworkMode: "egress", AllowedDomains: []string{"example.test"}}, err: "unsupported machine network policy"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := tc.policy + err := normalizeMachinePolicy(&policy) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + require.NoError(t, err) + want := tc.policy + if want.TimeoutSeconds == 0 { + want.TimeoutSeconds = 600 + } + if want.NetworkMode == "" { + want.NetworkMode = "isolated" + } + require.Equal(t, want, policy) + require.NoError(t, normalizeMachinePolicy(&policy)) + require.Equal(t, want, policy) + }) + } +} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go new file mode 100644 index 000000000..4945020d3 --- /dev/null +++ b/lib/builds/macos_build_publisher.go @@ -0,0 +1,194 @@ +package builds + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// StreamingMachinePublisher targets only the configured internal registry and a +// build-scoped repository. go-containerregistry hashes and compresses the layer +// from a reopenable tar stream, so it is never written to disk. Blobs use the +// registry's disk-streaming monolithic upload and are independently verified before +// the manifest is committed last. +// No credentials, redirect destinations or arbitrary output repositories come from the source. +type StreamingMachinePublisher struct { + RegistryURL string + Tokens *RegistryTokenGenerator +} + +type machineBlob struct { + open func() (io.ReadCloser, error) + descriptor v1.Descriptor + mountFrom string // set for a base layer mounted from this repository instead of uploaded +} + +// Publish commits bundle as a standalone machine image in builds/{id}. +func (p *StreamingMachinePublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + return p.publish(ctx, id, bundle, nil) +} + +// PublishDerived commits bundle as a delta over base. The base layers are mounted +// from base.Repo rather than uploaded; only the delta layer and config are uploaded. +func (p *StreamingMachinePublisher) PublishDerived(ctx context.Context, id, bundle string, base MachineDeltaBase) (MachinePublication, error) { + if err := base.validate(); err != nil { + return MachinePublication{}, err + } + return p.publish(ctx, id, bundle, &base) +} + +func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle string, source *MachineDeltaBase) (MachinePublication, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return MachinePublication{}, fmt.Errorf("bounded publication required") + } + if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { + return MachinePublication{}, fmt.Errorf("scoped machine publisher required") + } + address := p.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return MachinePublication{}, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") + } + if err := validateMachineExport(bundle, nil); err != nil { + return MachinePublication{}, err + } + repo := "builds/" + id + access := []RepoPermission{{Repo: repo, Scope: "push"}} + if source != nil { + access = append(access, RepoPermission{Repo: source.Repo, Scope: "pull"}) + } + token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) + if err != nil { + return MachinePublication{}, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + base := strings.TrimRight(address, "/") + "/v2/" + repo + layers, diffIDs, err := machineLayers(ctx, client, token, address, bundle, source) + if err != nil { + return MachinePublication{}, err + } + config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + }}} + configBytes, err := json.Marshal(config) + if err != nil { + return MachinePublication{}, err + } + configBlob := machineBlob{open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(configBytes)), nil }, descriptor: v1.Descriptor{MediaType: types.OCIConfigJSON, Digest: machineHash(configBytes), Size: int64(len(configBytes))}} + descriptors := make([]v1.Descriptor, 0, len(layers)) + for _, layer := range layers { + descriptors = append(descriptors, layer.descriptor) + } + manifest := v1.Manifest{SchemaVersion: 2, MediaType: types.OCIManifestSchema1, Config: configBlob.descriptor, Layers: descriptors} + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return MachinePublication{}, err + } + digest := machineHash(manifestBytes).String() + for _, blob := range append([]machineBlob{configBlob}, layers...) { + if blob.mountFrom != "" { + if err := mountMachineBlob(ctx, client, token, base, blob); err != nil { + return MachinePublication{}, err + } + continue + } + body, e := blob.open() + if e != nil { + return MachinePublication{}, e + } + response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, base+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) + closeErr := body.Close() + if e != nil { + return MachinePublication{}, e + } + if closeErr != nil { + response.Body.Close() + return MachinePublication{}, closeErr + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") + } + if _, e = machineFetch(ctx, client, token, base+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { + return MachinePublication{}, fmt.Errorf("machine blob verification failed: %w", e) + } + } + target := base + "/manifests/latest" + response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target, string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + if err != nil { + return MachinePublication{}, err + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") + } + // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. + if _, err = machineFetch(ctx, client, token, base+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { + return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) + } + return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil +} + +func machineTarballLayer(ctx context.Context, bundle string) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(writeMachineTar(ctx, bundle, writer)) }() + return reader, nil + }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) +} + +// writeMachineTar writes a deterministic tar of the exported bundle. +func writeMachineTar(ctx context.Context, bundle string, w io.Writer) error { + archive := tar.NewWriter(w) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + source, e := os.Open(filepath.Join(bundle, name)) + if e != nil { + return e + } + info, e := source.Stat() + if e != nil { + source.Close() + return e + } + e = archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) + if e == nil { + _, e = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + } + closeErr := source.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + } + return archive.Close() +} diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go new file mode 100644 index 000000000..c710a49e7 --- /dev/null +++ b/lib/builds/macos_build_publisher_test.go @@ -0,0 +1,96 @@ +package builds_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +type machineImportStub struct { + images.Manager + imported chan string +} + +func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, digest string) (*images.Image, error) { + m.imported <- repo + return &images.Image{Status: images.StatusReady, Digest: digest}, nil +} + +func machinePublisherBundle(t *testing.T) string { + t.Helper() + root := t.TempDir() + platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} + data, e := json.Marshal(platform) + require.NoError(t, e) + for name, data := range map[string][]byte{"config.json": data, "disk.img": []byte("synthetic disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), data, 0600)) + } + return root +} + +func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { + // Tiny storage is intentional: this proves auth/streaming/commit ordering, not bootability. + imports := &machineImportStub{imported: make(chan string, 1)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads, verified atomic.Int32 + handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") + if r.Method == http.MethodPost { + require.NotEmpty(t, r.URL.Query().Get("digest")) + require.Greater(t, r.ContentLength, int64(0)) + uploads.Add(1) + } + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { + verified.Add(1) + } + if r.Method == http.MethodPut { + require.Equal(t, int32(2), uploads.Load(), "manifest must be last") + } + reg.Handler().ServeHTTP(w, r) + })) + server := httptest.NewServer(handler) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + result, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.NoError(t, e) + require.Contains(t, result.Reference, "/builds/qa-build@sha256:") + select { + case repo := <-imports.imported: + require.Equal(t, "builds/qa-build", repo) + case <-ctx.Done(): + t.Fatal("manifest did not reach production conversion boundary") + } +} + +func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing.T) { + var visited atomic.Bool + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.Error(t, e) + require.False(t, visited.Load()) +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go new file mode 100644 index 000000000..411c12102 --- /dev/null +++ b/lib/builds/macos_build_recovery_test.go @@ -0,0 +1,88 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { + for _, status := range []string{StatusQueued, StatusBuilding, StatusPushing} { + t.Run(status, func(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, true) + m.config.BuilderImage = "linux-builder-must-not-start" + policy := &BuildPolicy{} + require.NoError(t, normalizeMachinePolicy(policy)) + meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) + // A workspace left by the interrupted attempt. + stale := filepath.Join(m.paths.BuildDir(meta.ID), "machine-build-interrupted") + require.NoError(t, os.MkdirAll(filepath.Join(stale, "bundle"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(stale, "bundle", "disk.img"), []byte("disk"), 0600)) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) + t.Cleanup(func() { + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) + }) + + // Hold the recovered build active so the second recovery sees it running. + f.startGate = make(chan struct{}) + require.NoError(t, m.Start(context.Background())) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 1 }, 5*time.Second, 10*time.Millisecond) + m.RecoverPendingBuilds() + close(f.startGate) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 && m.queue.PendingCount() == 0 }, 5*time.Second, 10*time.Millisecond) + build, err := m.GetBuild(context.Background(), meta.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) + require.NoDirExists(t, stale, "the interrupted attempt's workspace is removed") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") + require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") + var starts int + for _, call := range f.calls { + if call == "start" { + starts++ + } + } + require.Equal(t, 1, starts, "recovery must not duplicate an active build") + linux, err := readMetadata(m.paths, "linux-job") + require.NoError(t, err) + require.Equal(t, StatusQueued, linux.Status, "Linux jobs are preserved for Linux-mode recovery") + require.True(t, m.BuilderHasBuilds("linux-builder"), "unrecovered disk jobs still protect their builders") + }) + } +} + +func TestRecoveryCleansCancelledMachineBuildSource(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, false) + meta := &buildMetadata{ + ID: "cancelled-build", + Status: StatusCancelled, + Request: &CreateBuildRequest{ + MachineBaseImage: "registry/base@" + machineTestDigest, + }, + } + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("source"))) + meta.Status = StatusCancelled + require.NoError(t, writeMetadata(m.paths, meta)) + + m.RecoverPendingBuilds() + + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz")) +} + +func TestMachineOnlyStartPreservesBuildsWhenBackendDisabled(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, true) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "disabled", Status: StatusBuilding, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}})) + require.NoError(t, m.Start(context.Background())) + require.Equal(t, 0, m.queue.ActiveCount()) + meta, err := readMetadata(m.paths, "disabled") + require.NoError(t, err) + require.Equal(t, StatusBuilding, meta.Status) + require.False(t, m.ReadyForBuilds()) +} diff --git a/lib/builds/macos_build_source.go b/lib/builds/macos_build_source.go new file mode 100644 index 000000000..37b7ee2f5 --- /dev/null +++ b/lib/builds/macos_build_source.go @@ -0,0 +1,119 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "unicode/utf8" +) + +// machineEntrypoint is the script a machine build source must contain at its +// root. It runs as root with /bin/sh in the guest's private build directory. +const machineEntrypoint = "build.sh" +const maxMachineExtractBytes = 128 << 20 + +// extractMachineSource unpacks a build-owned tar.gz on the host into an exclusive +// directory, never with guest tar or a caller-supplied destination. Links, +// devices, duplicates and decompression bombs fail before anything is copied to +// the VM. The archive must contain the entrypoint as a regular file. +func extractMachineSource(ctx context.Context, source, destination string) error { + file, err := os.Open(source) + if err != nil { + return err + } + defer file.Close() + compressed, err := gzip.NewReader(file) + if err != nil { + return err + } + defer compressed.Close() + compressed.Multistream(false) + if err = os.Mkdir(destination, 0700); err != nil { + return err + } + complete := false + defer func() { + if !complete { + _ = os.RemoveAll(destination) + } + }() + limited := &io.LimitedReader{R: compressed, N: maxMachineExtractBytes + 1} + reader := tar.NewReader(limited) + seen := map[string]bool{} + var total int64 + for count := 0; ; count++ { + if err = ctx.Err(); err != nil { + return err + } + header, e := reader.Next() + if e == io.EOF { + break + } + if e != nil { + return e + } + if count >= 1024 { + return fmt.Errorf("too many machine source entries") + } + name := path.Clean(header.Name) + if name == "." && header.Typeflag == tar.TypeDir { + continue + } + if !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || !filepath.IsLocal(name) || seen[name] { + return fmt.Errorf("invalid or duplicate machine source path") + } + seen[name] = true + if header.Size < 0 || header.Size > 64<<20 || total+header.Size > maxMachineExtractBytes { + return fmt.Errorf("machine source expands beyond limit") + } + total += header.Size + target := filepath.Join(destination, name) + switch header.Typeflag { + case tar.TypeDir: + if header.Size != 0 { + return fmt.Errorf("directory has data") + } + if err = os.MkdirAll(target, 0700); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + output, e := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600|os.FileMode(header.Mode)&0100) + if e != nil { + return e + } + _, copyErr := io.Copy(output, reader) + closeErr := output.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("machine source must contain only regular files and directories") + } + } + // Consume bounded trailing tar padding to verify gzip integrity without allowing + // an unbounded post-tar decompression stream. + if _, err = io.Copy(io.Discard, limited); err != nil { + return err + } + if limited.N <= 0 { + return fmt.Errorf("machine source expands beyond limit") + } + info, err := os.Lstat(filepath.Join(destination, machineEntrypoint)) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine source must contain %s", machineEntrypoint) + } + complete = true + return nil +} diff --git a/lib/builds/macos_build_source_test.go b/lib/builds/macos_build_source_test.go new file mode 100644 index 000000000..00f0d5f4d --- /dev/null +++ b/lib/builds/macos_build_source_test.go @@ -0,0 +1,67 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeMachineSource(t *testing.T, headers []*tar.Header, bodies []string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "source.tar.gz") + f, e := os.Create(name) + require.NoError(t, e) + gz := gzip.NewWriter(f) + writer := tar.NewWriter(gz) + for i, h := range headers { + require.NoError(t, writer.WriteHeader(h)) + if h.Typeflag == tar.TypeReg { + _, e = writer.Write([]byte(bodies[i])) + require.NoError(t, e) + } + } + require.NoError(t, writer.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + return name +} + +func TestMachineSourceExtraction(t *testing.T) { + source := writeMachineSource(t, []*tar.Header{{Name: machineEntrypoint, Size: 4, Typeflag: tar.TypeReg, Mode: 0755}, {Name: "lib/data", Size: 1, Typeflag: tar.TypeReg}}, []string{"true", "x"}) + root := filepath.Join(t.TempDir(), "source") + require.NoError(t, extractMachineSource(context.Background(), source, root)) + info, e := os.Stat(filepath.Join(root, machineEntrypoint)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0700), info.Mode().Perm()) + info, e = os.Stat(filepath.Join(root, "lib", "data")) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestMachineSourceRejectsUnsafeArchives(t *testing.T) { + entry := &tar.Header{Name: machineEntrypoint, Typeflag: tar.TypeReg, Size: 4} + for _, tc := range []struct { + name string + headers []*tar.Header + bodies []string + }{ + {"missing entrypoint", []*tar.Header{{Name: "other", Typeflag: tar.TypeReg, Size: 1}}, []string{"x"}}, + {"empty entrypoint", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeReg}}, []string{""}}, + {"entrypoint directory", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeDir}}, []string{""}}, + {"traversal", []*tar.Header{entry, {Name: "../outside", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x"}}, + {"symlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc"}}, []string{"true", ""}}, + {"hardlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeLink, Linkname: "other"}}, []string{"true", ""}}, + {"duplicate", []*tar.Header{entry, {Name: "a", Typeflag: tar.TypeReg, Size: 1}, {Name: "a", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x", "y"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := filepath.Join(t.TempDir(), "source") + require.Error(t, extractMachineSource(context.Background(), writeMachineSource(t, tc.headers, tc.bodies), root)) + require.NoDirExists(t, root) + }) + } +} diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go new file mode 100644 index 000000000..da848ce78 --- /dev/null +++ b/lib/builds/macos_build_test.go @@ -0,0 +1,226 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type machineFixture struct { + image *images.Image + calls []string + fail string + stop MachineStopReceipt + cancel context.CancelFunc + waitForCancel bool + mutateExport func(string) error + publicationOverride *MachinePublication + publisherCalled bool + cleanupContextLive bool + startGate chan struct{} +} + +func newMachineFixture() *machineFixture { + return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} +} +func (f *machineFixture) record(phase string) error { + f.calls = append(f.calls, phase) + if f.fail == phase { + return errors.New("sensitive-command-argument") + } + return nil +} +func (f *machineFixture) ResolveBase(context.Context, string) (*images.Image, error) { + if err := f.record("resolve"); err != nil { + return nil, err + } + return f.image, nil +} +func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { + if p.CPUs != 4 || p.MemoryMB != 8192 { + return nil, errors.New("wrong resource defaults") + } + if f.startGate != nil { + <-f.startGate + } + return f, f.record("start") +} +func (f *machineFixture) ID() string { return "isolated-test-instance" } +func (f *machineFixture) Provision(ctx context.Context, source string) error { + if _, err := os.ReadFile(source); err != nil { + return err + } + if f.cancel != nil { + f.cancel() + return ctx.Err() + } + if f.waitForCancel { + <-ctx.Done() + return ctx.Err() + } + return f.record("provision") +} +func (f *machineFixture) Sanitize(context.Context) error { return f.record("sanitize") } +func (f *machineFixture) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + f.cleanupContextLive = ctx.Err() == nil + phase := "stop" + if force { + phase = "abort" + } + return f.stop, f.record(phase) +} +func (f *machineFixture) Export(_ context.Context, root string) error { + if err := f.record("export"); err != nil { + return err + } + data, _ := json.Marshal(*f.image.MacOS) + for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { + if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { + return err + } + } + if f.mutateExport != nil { + return f.mutateExport(root) + } + return nil +} +func (f *machineFixture) Destroy(ctx context.Context) error { + f.cleanupContextLive = ctx.Err() == nil + return f.record("destroy") +} +func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachinePublication, error) { + f.publisherCalled = true + if err := ctx.Err(); err != nil { + return MachinePublication{}, err + } + if id != "build-test" { + return MachinePublication{}, errors.New("wrong job namespace") + } + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, err := os.Stat(filepath.Join(root, name)) + if err != nil || info.Mode().Perm() != 0600 { + return MachinePublication{}, errors.New("unsafe export mode") + } + } + publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} + if f.publicationOverride != nil { + publication = *f.publicationOverride + } + return publication, f.record("publish") +} +func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { + t.Helper() + return &machineRunnerFixture{MachineBuildBackend: MachineBuildBackend{Driver: f, Publisher: f}, WorkDir: t.TempDir()} +} +func machineTestRequest() MachineBuildRequest { + return MachineBuildRequest{ID: "build-test", BaseImage: "localhost/macos@" + machineTestDigest} +} + +func TestMachineBuildPublicationOrder(t *testing.T) { + f := newMachineFixture() + r := machineTestRunner(t, f) + result, err := r.Run(context.Background(), machineTestRequest(), strings.NewReader("source")) + if err != nil { + t.Fatal(err) + } + expected := []string{"start", "provision", "sanitize", "stop", "export", "destroy", "publish"} + if !reflect.DeepEqual(f.calls, expected) { + t.Fatalf("wrong phase order: %v", f.calls) + } + hash := sha256.Sum256([]byte("source")) + if result.Provenance.SourceHash != hex.EncodeToString(hash[:]) || result.Provenance.BaseImageDigest != machineTestDigest { + t.Fatal("unverified provenance") + } + entries, err := os.ReadDir(r.WorkDir) + if err != nil || len(entries) != 0 { + t.Fatal("workspace not cleaned") + } +} + +func TestMachineBuildFailuresDoNotPublish(t *testing.T) { + for _, tc := range []struct { + name string + phase string + receipt *MachineStopReceipt + }{ + {name: "start", phase: "start"}, + {name: "provision", phase: "provision"}, + {name: "sanitize", phase: "sanitize"}, + {name: "stop", phase: "stop"}, + {name: "export", phase: "export"}, + {name: "destroy", phase: "destroy"}, + {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, + {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + if tc.receipt != nil { + f.stop = *tc.receipt + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("failed build reached publication") + } + if tc.phase != "" && strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("raw guest error leaked") + } + for _, phase := range f.calls { + if phase == "export" && tc.receipt != nil { + t.Fatal("unconfirmed stop exported") + } + if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { + t.Fatal("destroyed unconfirmed live storage") + } + } + }) + } +} + +func TestMachineBuildInputAdmission(t *testing.T) { + for _, kind := range []string{"floating base", "wrong digest", "linux", "pending", "shaping", "source hash", "CPU overflow", "memory precision"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + req := machineTestRequest() + switch kind { + case "floating base": + req.BaseImage = "localhost/macos:latest" + case "wrong digest": + f.image.Digest = "sha256:" + strings.Repeat("b", 64) + case "linux": + f.image.Platform = "linux/arm64" + case "pending": + f.image.Status = images.StatusPending + case "shaping": + req.Policy.MemoryMB = 4096 + case "source hash": + req.SourceHash = strings.Repeat("0", 64) + case "CPU overflow": + f.image.MacOS.CPUs = ^uint(0) + case "memory precision": + f.image.MacOS.Memory++ + } + runMachineBuildFailure(t, f, context.Background(), req, strings.NewReader("source")) + for _, phase := range f.calls { + if phase == "start" { + t.Fatal("bad input started a VM") + } + } + }) + } +} + +var _ MachineBuildDriver = (*machineFixture)(nil) +var _ MachineBuildSession = (*machineFixture)(nil) +var _ MachineBuildPublisher = (*machineFixture)(nil) diff --git a/lib/builds/macos_delta_publish.go b/lib/builds/macos_delta_publish.go new file mode 100644 index 000000000..f0590eb47 --- /dev/null +++ b/lib/builds/macos_delta_publish.go @@ -0,0 +1,172 @@ +package builds + +import ( + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// maxMachineMetadataBytes bounds a base manifest or config fetched for a delta. +const maxMachineMetadataBytes = 4 << 20 + +// MachineDeltaBase is a pinned base that a derivative is published against. Its +// layers are mounted from Repo, so the base must already be in the registry the +// publisher writes to. +type MachineDeltaBase struct { + Repo string // builds/{id} repository holding the base manifest and layers + Digest string // canonical sha256 digest of the base manifest + DiskPath string // absolute path of the base boot disk on this host +} + +func (m MachineDeltaBase) validate() error { + if !strings.HasPrefix(m.Repo, "builds/") || strings.Contains(m.Repo, "..") || len(m.Repo) > 256 { + return fmt.Errorf("delta base repository must be under builds/") + } + if !ValidMachineDigest(m.Digest) { + return fmt.Errorf("delta base digest must be a canonical sha256 digest") + } + if !filepath.IsAbs(m.DiskPath) { + return fmt.Errorf("delta base disk path must be absolute") + } + return nil +} + +// machineLayers returns a machine image's layer blobs and their diff IDs. A +// standalone image has one uploaded layer. A derivative mounts the base's layers +// and uploads one delta layer that patches the base disk. +func machineLayers(ctx context.Context, client *http.Client, token, registry, bundle string, source *MachineDeltaBase) ([]machineBlob, []v1.Hash, error) { + if source == nil { + layer, err := machineTarballLayer(ctx, bundle) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blob := machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}} + return []machineBlob{blob}, []v1.Hash{diffID}, nil + } + baseLayers, baseDiffIDs, err := fetchMachineSource(ctx, client, token, registry, source) + if err != nil { + return nil, nil, err + } + blobs := make([]machineBlob, 0, len(baseLayers)+1) + for _, descriptor := range baseLayers { + blobs = append(blobs, machineBlob{descriptor: descriptor, mountFrom: source.Repo}) + } + diskPath := filepath.Join(bundle, "disk.img") + delta, err := images.ComputeMacOSDelta(source.DiskPath, diskPath) + if err != nil { + return nil, nil, fmt.Errorf("compute disk delta: %w", err) + } + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(images.WriteMacOSDeltaTar(writer, delta, diskPath, bundle)) }() + return reader, nil + }, tarball.WithMediaType(images.MacOSDeltaMediaType), tarball.WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blobs = append(blobs, machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: images.MacOSDeltaMediaType, Digest: digest, Size: size}}) + return blobs, append(baseDiffIDs, diffID), nil +} + +// fetchMachineSource reads the base manifest and config from the registry, checking +// each against its digest. It returns the base's layer descriptors and diff IDs. +func fetchMachineSource(ctx context.Context, client *http.Client, token, registry string, source *MachineDeltaBase) ([]v1.Descriptor, []v1.Hash, error) { + root := strings.TrimRight(registry, "/") + "/v2/" + source.Repo + manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+source.Digest, source.Digest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + var manifest v1.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + configDigest := manifest.Config.Digest.String() + configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + var config v1.ConfigFile + if err := json.Unmarshal(configBytes, &config); err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") + } + return manifest.Layers, config.RootFS.DiffIDs, nil +} + +// machineDeltaBase returns the base as a delta source when the publisher writes to +// the registry that stores the base. Any other base publishes standalone. +func (m *manager) machineDeltaBase(publisher MachineBuildPublisher, baseImage string, base *images.Image) *MachineDeltaBase { + publisherWithRegistry, ok := publisher.(*StreamingMachinePublisher) + if !ok || base == nil { + return nil + } + address := publisherWithRegistry.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" { + return nil + } + ref, err := images.ParseNormalizedRef(baseImage) + if err != nil || !ref.IsDigest() { + return nil + } + prefix := registry.Host + "/" + if !strings.HasPrefix(ref.Repository(), prefix) { + return nil + } + disk, err := images.GetDiskPath(m.paths, baseImage, base.Digest) + if err != nil { + return nil + } + return &MachineDeltaBase{Repo: strings.TrimPrefix(ref.Repository(), prefix), Digest: base.Digest, DiskPath: disk} +} + +// publishBuild publishes bundle as a derivative when the base is a registry source, +// and as a standalone image otherwise. +func (r *MachineBuildBackend) publishBuild(ctx context.Context, req MachineBuildRequest, bundle string) (MachinePublication, error) { + if derived, ok := r.Publisher.(interface { + PublishDerived(context.Context, string, string, MachineDeltaBase) (MachinePublication, error) + }); ok && req.deltaBase != nil { + return derived.PublishDerived(ctx, req.ID, bundle, *req.deltaBase) + } + return r.Publisher.Publish(ctx, req.ID, bundle) +} diff --git a/lib/builds/macos_delta_publish_test.go b/lib/builds/macos_delta_publish_test.go new file mode 100644 index 000000000..034cff270 --- /dev/null +++ b/lib/builds/macos_delta_publish_test.go @@ -0,0 +1,202 @@ +package builds_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +const deltaTestSecret = "synthetic registry secret" + +type deltaTestRegistry struct { + server *httptest.Server + tokens *builds.RegistryTokenGenerator + mounts atomic.Int32 + uploads atomic.Int32 + puts atomic.Int32 +} + +func newDeltaTestRegistry(t *testing.T) *deltaTestRegistry { + t.Helper() + imports := &machineImportStub{imported: make(chan string, 16)} + reg, err := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, err) + dr := &deltaTestRegistry{tokens: builds.NewRegistryTokenGenerator(deltaTestSecret)} + handler := middleware.JwtAuth(deltaTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Query().Get("mount") != "" { + dr.mounts.Add(1) + } else if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { + dr.uploads.Add(1) + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + dr.puts.Add(1) + } + reg.Handler().ServeHTTP(w, r) + })) + dr.server = httptest.NewServer(handler) + t.Cleanup(dr.server.Close) + // Each manifest PUT starts a conversion goroutine that writes the OCI layout + // after the response. Its import call is the last step, so wait for every one + // before the TempDir cleanup removes the registry's data. + t.Cleanup(func() { + for range dr.puts.Load() { + select { + case <-imports.imported: + case <-time.After(10 * time.Second): + return + } + } + }) + return dr +} + +// deltaTestDisk is a disk of eight 64 KiB chunks, each a distinct byte. +func deltaTestDisk() []byte { + disk := make([]byte, 8*64<<10) + for i := range 8 { + for j := range 64 << 10 { + disk[i*64<<10+j] = byte(i + 1) + } + } + return disk +} + +func deltaTestBundle(t *testing.T, disk []byte, aux string) string { + t.Helper() + bundle := machinePublisherBundle(t) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "disk.img"), disk, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "aux.img"), []byte(aux), 0600)) + return bundle +} + +// pullManifest fetches a manifest from the registry with a pull-scoped token. +func (dr *deltaTestRegistry) pullManifest(t *testing.T, repo, digest string) v1.Manifest { + t.Helper() + token, err := dr.tokens.GenerateToken("reader", []builds.RepoPermission{{Repo: repo, Scope: "pull"}}, time.Minute) + require.NoError(t, err) + request, err := http.NewRequest(http.MethodGet, dr.server.URL+"/v2/"+repo+"/manifests/"+digest, nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + var manifest v1.Manifest + require.NoError(t, json.NewDecoder(response.Body).Decode(&manifest)) + return manifest +} + +func TestStreamingMachinePublisherDerivedMountsBaseLayers(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + baseBundle := deltaTestBundle(t, deltaTestDisk(), "base aux") + base, err := publisher.Publish(ctx, "base-x", baseBundle) + require.NoError(t, err) + baseManifest := dr.pullManifest(t, "builds/base-x", base.Digest) + + derived := deltaTestDisk() + for j := range 64 << 10 { + derived[3*64<<10+j] = 0xAA + } + derivedBundle := deltaTestBundle(t, derived, "derived aux") + dr.mounts.Store(0) + dr.uploads.Store(0) + result, err := publisher.PublishDerived(ctx, "derived-x", derivedBundle, builds.MachineDeltaBase{ + Repo: "builds/base-x", + Digest: base.Digest, + DiskPath: filepath.Join(baseBundle, "disk.img"), + }) + require.NoError(t, err) + require.Equal(t, int32(1), dr.mounts.Load(), "the base layer is mounted, not uploaded") + require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded") + + derivedManifest := dr.pullManifest(t, "builds/derived-x", result.Digest) + require.Len(t, derivedManifest.Layers, 2) + require.Equal(t, baseManifest.Layers[0].Digest, derivedManifest.Layers[0].Digest) + require.Equal(t, images.MacOSDeltaMediaType, derivedManifest.Layers[1].MediaType) +} + +func TestStreamingMachinePublisherDerivedRejectsMissingBase(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + missing := fmt.Sprintf("sha256:%064x", 1) + _, err := publisher.PublishDerived(ctx, "derived-y", bundle, builds.MachineDeltaBase{ + Repo: "builds/never-published", + Digest: missing, + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.Error(t, err) +} + +func TestStreamingMachinePublisherDerivedRejectsNonBaseRepo(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + _, err := publisher.PublishDerived(context.Background(), "derived-z", bundle, builds.MachineDeltaBase{ + Repo: "library/alpine", + Digest: fmt.Sprintf("sha256:%064x", 2), + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.ErrorContains(t, err, "under builds/") +} + +// mountRequest posts a blob mount with the given token and reports the status. +func (dr *deltaTestRegistry) mountRequest(t *testing.T, token, repo, digest, from string) int { + t.Helper() + target := dr.server.URL + "/v2/" + repo + "/blobs/uploads/?mount=" + digest + "&from=" + from + request, err := http.NewRequest(http.MethodPost, target, bytes.NewReader(nil)) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+token) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + _, _ = io.Copy(io.Discard, response.Body) + return response.StatusCode +} + +func TestBlobMountFallsThroughToUploadForMissingBlob(t *testing.T) { + dr := newDeltaTestRegistry(t) + token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ + {Repo: "builds/target-x", Scope: "push"}, + {Repo: "builds/source-x", Scope: "pull"}, + }, time.Minute) + require.NoError(t, err) + status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 3), "builds/source-x") + require.Equal(t, http.StatusAccepted, status, "a missing blob opens an upload session") +} + +func TestBlobMountRefusedFromUnauthorizedSource(t *testing.T) { + dr := newDeltaTestRegistry(t) + token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ + {Repo: "builds/target-x", Scope: "push"}, + }, time.Minute) + require.NoError(t, err) + status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 4), "builds/other-private") + require.Equal(t, http.StatusForbidden, status, "a token without pull on the source cannot mount from it") +} diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go new file mode 100644 index 000000000..a9d2f35cd --- /dev/null +++ b/lib/builds/macos_registry.go @@ -0,0 +1,105 @@ +package builds + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// machineRegistryRequest sends one authenticated registry request. The caller +// closes the response body. +func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { + request, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return nil, err + } + request.ContentLength = size + request.Header.Set("Authorization", "Bearer "+token) + if media != "" { + request.Header.Set("Content-Type", media) + } + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("machine registry request failed") + } + return response, nil +} + +func machineHash(data []byte) v1.Hash { + sum := sha256.Sum256(data) + return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} +} + +// machineFetch GETs a registry object and checks it against digest. A size of 0 +// marks metadata: the body is buffered up to maxMachineMetadataBytes and returned. +// Any other size must match exactly and is only hashed, never kept, so layers of +// tens of gigabytes are not held in memory. +func machineFetch(ctx context.Context, client *http.Client, token, target, digest string, size int64) ([]byte, error) { + response, err := machineRegistryRequest(ctx, client, token, http.MethodGet, target, "", nil, 0) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d", response.StatusCode) + } + metadata := size == 0 + limit := size + 1 + if metadata { + limit = maxMachineMetadataBytes + 1 + } + sum := sha256.New() + var kept bytes.Buffer + out := io.Writer(sum) + if metadata { + out = io.MultiWriter(sum, &kept) + } + n, err := io.Copy(out, io.LimitReader(response.Body, limit)) + if err != nil { + return nil, err + } + if n >= limit || (!metadata && n != size) { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if (v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum.Sum(nil))}).String() != digest { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if !metadata { + return nil, nil + } + return kept.Bytes(), nil +} + +// mountMachineBlob links a base layer into the build repository from its source. The +// registry answers 201 only when it already stores the blob; the size is then checked +// with a HEAD request. The bytes are not downloaded or hashed again. +func mountMachineBlob(ctx context.Context, client *http.Client, token, base string, blob machineBlob) error { + digest := blob.descriptor.Digest.String() + target := base + "/blobs/uploads/?mount=" + digest + "&from=" + url.QueryEscape(blob.mountFrom) + response, err := machineRegistryRequest(ctx, client, token, http.MethodPost, target, "", nil, 0) + if err != nil { + return err + } + response.Body.Close() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("base layer %s was not mounted", digest) + } + head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+digest, "", nil, 0) + if err != nil { + return err + } + head.Body.Close() + if head.StatusCode != http.StatusOK || head.ContentLength != blob.descriptor.Size { + return fmt.Errorf("mounted base layer %s is not verified", digest) + } + return nil +} diff --git a/lib/builds/macos_registry_test.go b/lib/builds/macos_registry_test.go new file mode 100644 index 000000000..77ccbac79 --- /dev/null +++ b/lib/builds/macos_registry_test.go @@ -0,0 +1,54 @@ +package builds + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachineFetchVerifiesSizeAndDigest(t *testing.T) { + body := []byte("synthetic object") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + })) + defer server.Close() + digest := machineHash(body).String() + oversize := strings.Repeat("x", maxMachineMetadataBytes+1) + bigServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(oversize)) + })) + defer bigServer.Close() + + for _, tc := range []struct { + name string + target string + digest string + size int64 + ok bool + }{ + {"metadata matches digest", server.URL, digest, 0, true}, + {"sized object matches", server.URL, digest, int64(len(body)), true}, + {"wrong digest", server.URL, machineHash([]byte("other")).String(), 0, false}, + {"size shorter than body", server.URL, digest, int64(len(body)) - 1, false}, + {"size longer than body", server.URL, digest, int64(len(body)) + 1, false}, + {"metadata over cap", bigServer.URL, machineHash([]byte(oversize)).String(), 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest, tc.size) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.size == 0 { + require.Equal(t, body, got, "metadata is returned") + } else { + require.Nil(t, got, "sized objects are verified, never buffered") + } + }) + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 806ae5622..a4327e358 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -87,6 +87,12 @@ type Manager interface { // Config holds configuration for the build manager type Config struct { + // MachineBuild is the opt-in native macOS build backend. + MachineBuild *MachineBuildBackend + + // MachineBuildOnly disables Linux builder preparation and recovery. + MachineBuildOnly bool + // MaxConcurrentBuilds is the maximum number of concurrent builds MaxConcurrentBuilds int @@ -149,6 +155,7 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex + machineBuilds sync.Map // build ID -> *machineBuildControl builderReady atomic.Bool // pendingRecovered is set once RecoverPendingBuilds has re-enqueued // persisted pending builds; until then BuilderHasBuilds also scans @@ -204,6 +211,10 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { + if m.config.MachineBuildOnly { + m.RecoverPendingBuilds() + return nil + } go func() { for { m.ensureBuilderImage(ctx) @@ -444,7 +455,12 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc // Apply defaults to build policy policy := req.BuildPolicy - if policy == nil { + if machineBuild(&req) { + var err error + if policy, err = m.admitMachineBuild(&req, sourceData); err != nil { + return nil, err + } + } else if policy == nil { defaultPolicy := DefaultBuildPolicy() policy = &defaultPolicy } else { @@ -481,6 +497,15 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc CreatedAt: time.Now(), } + if machineBuild(&req) { + // Stage verified input before making the job visible to recovery. + if err := m.storeMachineSource(ctx, meta, sourceData); err != nil { + deleteBuild(m.paths, id) + return nil, err + } + return m.enqueueMachineBuild(meta, req, policy), nil + } + // Write initial metadata if err := writeMetadata(m.paths, meta); err != nil { return nil, fmt.Errorf("write metadata: %w", err) @@ -678,18 +703,29 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques start := time.Now() m.logger.Info("starting build", "id", id) - // Update status to building - m.updateStatus(id, StatusBuilding, nil) - - // Create timeout context + // Register machine cancellation before exposing the running status. Ordinary + // Linux builder cancellation retains its existing instance-delete behavior. buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + if machineBuild(&req) { + m.machineBuilds.Store(id, &machineBuildControl{cancel: cancel}) + defer m.machineBuilds.Delete(id) + defer func() { + meta, err := readMetadata(m.paths, id) + if err == nil && (meta.Status == StatusReady || meta.Status == StatusFailed || meta.Status == StatusCancelled) { + m.removeMachineSource(id) + } + }() + } + m.updateStatus(id, StatusBuilding, nil) // Mirror base images to the local registry before launching the VM. // BuildKit is configured with our registry as a mirror for docker.io, // so pre-cached images will be served locally without pulling from Docker Hub. - if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { - m.logger.Warn("failed to mirror base images", "id", id, "error", err) + if !machineBuild(&req) { + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } } // Run the build in a builder VM @@ -754,6 +790,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // If image_name is set, re-tag the image so it's accessible by that name. // The builder pushed to builds/{id} but the user wants it as image_name. imageRef := buildRepo + if machineBuild(&req) && result.ImageRef != "" { + imageRef = result.ImageRef + } if req.ImageName != "" { ref, err := images.ParseNormalizedRef(req.ImageName) if err != nil { @@ -798,6 +837,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + if machineBuild(&req) { + return m.executeMachineBuild(ctx, id, req, policy) + } if !m.builderReady.Load() { return nil, fmt.Errorf("builder image is being prepared, please retry shortly") } @@ -1106,6 +1148,11 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1246,12 +1293,18 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { case StatusQueued: // Remove from queue if m.queue.Cancel(id) { + if machineBuild(meta.Request) { + return m.cancelQueuedMachineBuild(meta) + } m.updateStatus(id, StatusCancelled, nil) return nil } return ErrBuildInProgress // Was already picked up case StatusBuilding, StatusPushing: + if machineBuild(meta.Request) { + return m.cancelMachineBuild(id) + } // Can't cancel a running build easily // Would need to terminate the builder instance if meta.BuilderInstance != nil { @@ -1449,15 +1502,36 @@ func (m *manager) StreamBuildEvents(ctx context.Context, id string, follow bool) // RecoverPendingBuilds recovers builds that were interrupted on restart func (m *manager) RecoverPendingBuilds() { - pending, err := listPendingBuilds(m.paths) + all, err := listAllBuilds(m.paths) if err != nil { - m.logger.Error("list pending builds for recovery", "error", err) + m.logger.Error("list builds for machine cleanup", "error", err) return } + for _, meta := range all { + if !machineBuild(meta.Request) { + continue + } + switch meta.Status { + case StatusReady, StatusFailed: + m.removeMachineSource(meta.ID) + case StatusCancelled: + if m.config.MachineBuild != nil { + if err := m.reapPriorMachineBuilder(context.Background(), meta.ID); err != nil { + m.logger.Error("failed to reap cancelled machine build", "id", meta.ID, "error", err) + continue + } + } + m.removeMachineSource(meta.ID) + } + } + pending := pendingBuilds(all) for _, meta := range pending { meta := meta // Shadow loop variable for closure capture m.logger.Info("recovering build", "id", meta.ID, "status", meta.Status) + if m.skipMachineBuildRecovery(meta) { + continue + } // Re-enqueue the build if meta.Request != nil { @@ -1472,7 +1546,11 @@ func (m *manager) RecoverPendingBuilds() { continue } - m.queue.EnqueueSerial(meta.ID, *meta.Request, meta.Request.BuilderID, func() { + serialKey := meta.Request.BuilderID + if machineBuild(meta.Request) { + serialKey = machineBuildSerialKey(*meta.Request) + } + m.queue.EnqueueSerial(meta.ID, *meta.Request, serialKey, func() { policy := DefaultBuildPolicy() if meta.Request.BuildPolicy != nil { policy = *meta.Request.BuildPolicy @@ -1485,11 +1563,16 @@ func (m *manager) RecoverPendingBuilds() { if len(pending) > 0 { m.logger.Info("recovered pending builds", "count", len(pending)) } - m.pendingRecovered.Store(true) + if !m.config.MachineBuildOnly { + m.pendingRecovered.Store(true) + } } // refreshBuildToken regenerates the registry token for a build and updates the config file func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) error { + if machineBuild(req) { + return nil // Machine builds have no builder config or registry token. + } // Read existing build config config, err := readBuildConfig(m.paths, buildID) if err != nil { diff --git a/lib/builds/source.go b/lib/builds/source.go new file mode 100644 index 000000000..b2075607f --- /dev/null +++ b/lib/builds/source.go @@ -0,0 +1,75 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +const maxBuildSourceBytes = 64 << 20 + +type buildContextReader struct { + ctx context.Context + r io.Reader +} + +func (r buildContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +// stageBuildSource streams compressed source once into a private, exclusive job +// file. Both backends consume this file and use its hash for input verification. +func stageBuildSource(ctx context.Context, source io.Reader, path string, limit int64) (hash string, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return "", err + } + defer func() { + if closeErr := file.Close(); err == nil { + err = closeErr + } + }() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(file, h), io.LimitReader(buildContextReader{ctx, source}, limit+1)) + if err != nil { + return "", err + } + if n > limit { + return "", fmt.Errorf("build source too large") + } + if err = file.Sync(); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// verifyBuildSource rejects missing/replaced source on recovery before a machine +// is launched. It never writes a second source copy. +func verifyBuildSource(ctx context.Context, path, expected string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxBuildSourceBytes { + return fmt.Errorf("invalid staged source") + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + h := sha256.New() + if _, err = io.Copy(h, io.LimitReader(buildContextReader{ctx, file}, maxBuildSourceBytes+1)); err != nil { + return err + } + if hex.EncodeToString(h.Sum(nil)) != expected { + return ErrSourceHashMismatch + } + return nil +} diff --git a/lib/builds/source_test.go b/lib/builds/source_test.go new file mode 100644 index 000000000..1bb7e8a84 --- /dev/null +++ b/lib/builds/source_test.go @@ -0,0 +1,36 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedBuildSourceStagingAndRecoveryVerification(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.tar.gz") + hash, err := stageBuildSource(ctx, strings.NewReader("source"), path, maxBuildSourceBytes) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.NoError(t, verifyBuildSource(ctx, path, hash)) + _, err = stageBuildSource(ctx, strings.NewReader("replacement"), path, maxBuildSourceBytes) + require.Error(t, err, "exclusive staging must preserve existing source") + require.NoError(t, verifyBuildSource(ctx, path, hash)) + canceled, cancel := context.WithCancel(ctx) + cancel() + require.ErrorIs(t, verifyBuildSource(canceled, path, hash), context.Canceled) + require.NoError(t, os.WriteFile(path, []byte("changed"), 0600)) + require.ErrorIs(t, verifyBuildSource(ctx, path, hash), ErrSourceHashMismatch) + require.NoError(t, os.Remove(path)) + require.Error(t, verifyBuildSource(ctx, path, hash)) + target := filepath.Join(filepath.Dir(path), "target") + require.NoError(t, os.WriteFile(target, []byte("source"), 0600)) + require.NoError(t, os.Symlink(target, path)) + require.ErrorContains(t, verifyBuildSource(ctx, path, hash), "invalid staged source") +} diff --git a/lib/builds/storage.go b/lib/builds/storage.go index ce555aac9..1add7a3c1 100644 --- a/lib/builds/storage.go +++ b/lib/builds/storage.go @@ -144,7 +144,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { if err != nil { return nil, err } + return pendingBuilds(all), nil +} +func pendingBuilds(all []*buildMetadata) []*buildMetadata { var pending []*buildMetadata for _, meta := range all { switch meta.Status { @@ -152,13 +155,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { pending = append(pending, meta) } } - - // Sort by created_at ascending (oldest first for FIFO recovery) sort.Slice(pending, func(i, j int) bool { return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) - - return pending, nil + return pending } // deleteBuild removes a build's data from disk diff --git a/lib/builds/types.go b/lib/builds/types.go index 4ad6008e9..571259328 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -43,6 +43,10 @@ type CreateBuildRequest struct { // The Dockerfile specifies the runtime (e.g., FROM node:20-alpine). Dockerfile string `json:"dockerfile,omitempty"` + // MachineBaseImage selects the installed-machine backend. The HTTP multipart + // field machine_base_image maps here; it is enabled only with macos_builds. + MachineBaseImage string `json:"machine_base_image,omitempty"` + // BaseImageDigest optionally pins the base image by digest for reproducibility BaseImageDigest string `json:"base_image_digest,omitempty"` @@ -219,6 +223,9 @@ type BuildResult struct { // ImageDigest is the digest of the pushed image (only on success) ImageDigest string `json:"image_digest,omitempty"` + // ImageRef is the pullable output reference for machine builds. + ImageRef string `json:"image_ref,omitempty"` + // Error is the error message (only on failure) Error string `json:"error,omitempty"` diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 5e41bb8e7..73b8a4e0a 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -2,10 +2,12 @@ package images import ( "bytes" + "context" "encoding/json" "fmt" "io" "os" + "os/exec" "path/filepath" ) @@ -19,6 +21,19 @@ type macOSMachinePayload struct { Platform *MacOSImage } +// RequireClosedFile fails unless no process holds path open. lsof exits 1 when +// nothing holds the file; any other failure leaves the storage unverified. +func RequireClosedFile(ctx context.Context, path string) error { + err := exec.CommandContext(ctx, "/usr/sbin/lsof", "-t", path).Run() + if err == nil { + return fmt.Errorf("storage is open: stop its VM first") + } + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + return fmt.Errorf("cannot verify storage is closed: %w", err) + } + return nil +} + // ValidateMacOSBundle validates a fixed-layout export, rejecting unknown metadata // fields so an accidental credential field cannot become part of an image. func ValidateMacOSBundle(root string) (*MacOSImage, error) { diff --git a/lib/images/macos_delta.go b/lib/images/macos_delta.go new file mode 100644 index 000000000..17255ac55 --- /dev/null +++ b/lib/images/macos_delta.go @@ -0,0 +1,261 @@ +package images + +import ( + "archive/tar" + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// MacOSDeltaMediaType marks a machine layer that patches the boot disk unpacked +// from the layers beneath it. Its tar holds, in this order, delta.json, the changed +// chunks of disk.img, and aux.img and config.json in full. +const MacOSDeltaMediaType types.MediaType = "application/vnd.hypeman.machine.disk-delta.v1.tar+gzip" + +// MacOSDeltaChunkSize is the granularity at which a delta records changed bytes. +const MacOSDeltaChunkSize = 64 << 10 + +const ( + macOSDeltaVersion = 1 + macOSDeltaManifestEntry = "delta.json" + macOSDeltaPatchEntry = "disk.img.patch" + macOSDeltaDisk = "disk.img" + macOSDeltaAux = "aux.img" + macOSDeltaConfig = "config.json" + maxMacOSDeltaManifestBytes = 1 << 20 + macOSDeltaEntries = 4 +) + +// MacOSDelta lists the chunks of a boot disk that differ from the disk beneath it. +// Size is the disk's logical size, which the base disk must match. +type MacOSDelta struct { + Version int `json:"version"` + ChunkSize int64 `json:"chunk_size"` + Size int64 `json:"size"` + Chunks []int64 `json:"chunks"` +} + +// Validate checks a delta before any chunk is applied: a supported version and +// chunk size, a whole number of chunks, and changed chunks in ascending order. +func (d *MacOSDelta) Validate() error { + if d.Version != macOSDeltaVersion || d.ChunkSize != MacOSDeltaChunkSize { + return fmt.Errorf("unsupported machine disk delta version %d with chunk size %d", d.Version, d.ChunkSize) + } + if d.Size <= 0 || d.Size%d.ChunkSize != 0 { + return fmt.Errorf("machine disk delta size %d is not a positive multiple of %d", d.Size, d.ChunkSize) + } + count := d.Size / d.ChunkSize + previous := int64(-1) + for _, index := range d.Chunks { + if index <= previous || index >= count { + return fmt.Errorf("machine disk delta chunk %d is out of order or out of range", index) + } + previous = index + } + return nil +} + +// ComputeMacOSDelta compares a derived boot disk with its base chunk by chunk. Both +// files must be the same size and a whole number of chunks. +func ComputeMacOSDelta(basePath, derivedPath string) (MacOSDelta, error) { + base, err := os.Open(basePath) + if err != nil { + return MacOSDelta{}, err + } + defer base.Close() + derived, err := os.Open(derivedPath) + if err != nil { + return MacOSDelta{}, err + } + defer derived.Close() + baseInfo, err := base.Stat() + if err != nil { + return MacOSDelta{}, err + } + derivedInfo, err := derived.Stat() + if err != nil { + return MacOSDelta{}, err + } + if baseInfo.Size() != derivedInfo.Size() { + return MacOSDelta{}, fmt.Errorf("derived disk is %d bytes, base disk is %d", derivedInfo.Size(), baseInfo.Size()) + } + delta := MacOSDelta{Version: macOSDeltaVersion, ChunkSize: MacOSDeltaChunkSize, Size: derivedInfo.Size(), Chunks: []int64{}} + if err := delta.Validate(); err != nil { + return MacOSDelta{}, err + } + baseReader := bufio.NewReaderSize(base, 4<<20) + derivedReader := bufio.NewReaderSize(derived, 4<<20) + baseChunk := make([]byte, MacOSDeltaChunkSize) + derivedChunk := make([]byte, MacOSDeltaChunkSize) + for index := int64(0); index*MacOSDeltaChunkSize < delta.Size; index++ { + if _, err := io.ReadFull(baseReader, baseChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read base disk: %w", err) + } + if _, err := io.ReadFull(derivedReader, derivedChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read derived disk: %w", err) + } + if !bytes.Equal(baseChunk, derivedChunk) { + delta.Chunks = append(delta.Chunks, index) + } + } + return delta, nil +} + +// WriteMacOSDeltaTar writes a delta layer's tar. The changed chunks are read from +// derived, which must be the disk the delta was computed against. aux.img and +// config.json are copied in full from bundle. +func WriteMacOSDeltaTar(w io.Writer, delta MacOSDelta, derived, bundle string) error { + if err := delta.Validate(); err != nil { + return err + } + manifest, err := json.Marshal(delta) + if err != nil { + return err + } + disk, err := os.Open(derived) + if err != nil { + return err + } + defer disk.Close() + archive := tar.NewWriter(w) + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaManifestEntry, Mode: 0600, Size: int64(len(manifest)), Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := archive.Write(manifest); err != nil { + return err + } + patchSize := int64(len(delta.Chunks)) * delta.ChunkSize + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaPatchEntry, Mode: 0600, Size: patchSize, Typeflag: tar.TypeReg}); err != nil { + return err + } + chunk := make([]byte, delta.ChunkSize) + for _, index := range delta.Chunks { + if _, err := disk.ReadAt(chunk, index*delta.ChunkSize); err != nil { + return fmt.Errorf("read changed chunk %d: %w", index, err) + } + if _, err := archive.Write(chunk); err != nil { + return err + } + } + for _, name := range []string{macOSDeltaAux, macOSDeltaConfig} { + if err := writeMacOSDeltaFile(archive, filepath.Join(bundle, name), name); err != nil { + return err + } + } + return archive.Close() +} + +func writeMacOSDeltaFile(archive *tar.Writer, path, name string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if err := archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}); err != nil { + return err + } + _, err = io.Copy(archive, file) + return err +} + +// macOSDeltaApplier applies the entries of a delta layer to the boot disk already +// unpacked into targetDir. The entries must arrive in the order WriteMacOSDeltaTar +// writes them, so a layer cannot name files outside the format. +type macOSDeltaApplier struct { + targetDir string + step int + manifest *MacOSDelta +} + +func (a *macOSDeltaApplier) entry(name string, size int64, archive io.Reader) error { + switch a.step { + case 0: + if name != macOSDeltaManifestEntry { + return fmt.Errorf("delta layer must start with %s, got %q", macOSDeltaManifestEntry, name) + } + if size > maxMacOSDeltaManifestBytes { + return fmt.Errorf("machine disk delta manifest is %d bytes", size) + } + decoder := json.NewDecoder(io.LimitReader(archive, size)) + decoder.DisallowUnknownFields() + var delta MacOSDelta + if err := decoder.Decode(&delta); err != nil { + return fmt.Errorf("decode machine disk delta: %w", err) + } + if err := delta.Validate(); err != nil { + return err + } + a.manifest = &delta + case 1: + if name != macOSDeltaPatchEntry { + return fmt.Errorf("delta layer entry %q must be %s", name, macOSDeltaPatchEntry) + } + if size != int64(len(a.manifest.Chunks))*a.manifest.ChunkSize { + return fmt.Errorf("machine disk patch is %d bytes, want %d", size, int64(len(a.manifest.Chunks))*a.manifest.ChunkSize) + } + if err := a.applyChunks(archive); err != nil { + return err + } + case 2, 3: + want := macOSDeltaAux + if a.step == 3 { + want = macOSDeltaConfig + } + if name != want { + return fmt.Errorf("delta layer entry %q must be %s", name, want) + } + if err := writeSparseFile(filepath.Join(a.targetDir, name), archive, size); err != nil { + return err + } + default: + return fmt.Errorf("delta layer has more than %d entries", macOSDeltaEntries) + } + a.step++ + return nil +} + +// applyChunks writes each changed chunk over the base disk in place. The base must +// be a regular file of the delta's size, so the offsets land where the diff computed. +func (a *macOSDeltaApplier) applyChunks(r io.Reader) error { + path := filepath.Join(a.targetDir, macOSDeltaDisk) + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("delta has no base disk: %w", err) + } + if !info.Mode().IsRegular() || info.Size() != a.manifest.Size { + return fmt.Errorf("base disk does not match the delta's %d-byte size", a.manifest.Size) + } + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return err + } + buf := make([]byte, a.manifest.ChunkSize) + for _, index := range a.manifest.Chunks { + if _, err = io.ReadFull(r, buf); err != nil { + break + } + if _, err = file.WriteAt(buf, index*a.manifest.ChunkSize); err != nil { + break + } + } + return errors.Join(err, file.Close()) +} + +// complete reports an error if the layer ended before its last entry. +func (a *macOSDeltaApplier) complete() error { + if a.step != macOSDeltaEntries { + return fmt.Errorf("delta layer has %d of %d entries", a.step, macOSDeltaEntries) + } + return nil +} diff --git a/lib/images/macos_delta_test.go b/lib/images/macos_delta_test.go new file mode 100644 index 000000000..4e0227e85 --- /dev/null +++ b/lib/images/macos_delta_test.go @@ -0,0 +1,203 @@ +package images + +import ( + "archive/tar" + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/require" +) + +type machineTestFile struct { + name string + data []byte +} + +// machineFilesLayer builds a layer of regular files with the given media type. +func machineFilesLayer(t *testing.T, mediaType types.MediaType, files ...machineTestFile) gcr.Layer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, f := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0600, Size: int64(len(f.data)), Typeflag: tar.TypeReg})) + _, err := tw.Write(f.data) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(mediaType)) + require.NoError(t, err) + return layer +} + +// deltaTestDisk returns a disk of n chunks, each filled with a distinct byte. +func deltaTestDisk(n int) []byte { + disk := make([]byte, n*MacOSDeltaChunkSize) + for i := range n { + for j := range MacOSDeltaChunkSize { + disk[i*MacOSDeltaChunkSize+j] = byte(i + 1) + } + } + return disk +} + +func writeDeltaTestFile(t *testing.T, path string, data []byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, data, 0600)) +} + +// deltaTestLayer builds a delta layer from a manifest and a bundle directory. +func deltaTestLayer(t *testing.T, delta MacOSDelta, derived, bundle string) gcr.Layer { + t.Helper() + var buf bytes.Buffer + require.NoError(t, WriteMacOSDeltaTar(&buf, delta, derived, bundle)) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(MacOSDeltaMediaType)) + require.NoError(t, err) + return layer +} + +func TestMacOSDeltaRoundTripAppliesOnlyChangedChunks(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + base := deltaTestDisk(8) + derived := append([]byte(nil), base...) + for j := range MacOSDeltaChunkSize { + derived[2*MacOSDeltaChunkSize+j] = 0xAA + derived[5*MacOSDeltaChunkSize+j] = 0 + } + basePath := filepath.Join(dir, "base-disk.img") + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, basePath, base) + writeDeltaTestFile(t, derivedPath, derived) + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("derived aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte(`{"derived":true}`)) + + delta, err := ComputeMacOSDelta(basePath, derivedPath) + require.NoError(t, err) + require.Equal(t, []int64{2, 5}, delta.Chunks) + + baseLayer := machineFilesLayer(t, types.OCILayer, + machineTestFile{name: "disk.img", data: base}, + machineTestFile{name: "aux.img", data: []byte("base aux")}, + machineTestFile{name: "config.json", data: []byte(`{"base":true}`)}, + ) + deltaLayer := deltaTestLayer(t, delta, derivedPath, bundle) + target := t.TempDir() + require.NoError(t, unpackMachineLayers2(ctx, target, baseLayer, deltaLayer)) + + got, err := os.ReadFile(filepath.Join(target, "disk.img")) + require.NoError(t, err) + require.Equal(t, derived, got) + aux, err := os.ReadFile(filepath.Join(target, "aux.img")) + require.NoError(t, err) + require.Equal(t, "derived aux", string(aux)) + config, err := os.ReadFile(filepath.Join(target, "config.json")) + require.NoError(t, err) + require.Equal(t, `{"derived":true}`, string(config)) +} + +func TestMacOSDeltaRejectsBaseOfDifferentSize(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + baseLayer := machineFilesLayer(t, types.OCILayer, machineTestFile{name: "disk.img", data: deltaTestDisk(4)}) + err := unpackMachineLayers2(context.Background(), t.TempDir(), baseLayer, deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "base disk does not match") +} + +func TestMacOSDeltaRejectsLayerWithoutBase(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + err := unpackMachineLayers2(context.Background(), t.TempDir(), deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "delta has no base disk") +} + +func TestMacOSDeltaRejectsIncompleteAndMisorderedLayers(t *testing.T) { + ctx := context.Background() + manifest := []byte(`{"version":1,"chunk_size":65536,"size":524288,"chunks":[]}`) + cases := map[string][]machineTestFile{ + "missing entries": {{name: "delta.json", data: manifest}}, + "wrong first entry": { + {name: "aux.img", data: []byte("x")}, + }, + "patch skipped": { + {name: "delta.json", data: manifest}, + {name: "aux.img", data: []byte("x")}, + }, + "extra entry": { + {name: "delta.json", data: manifest}, + {name: "disk.img.patch"}, + {name: "aux.img", data: []byte("x")}, + {name: "config.json", data: []byte("{}")}, + {name: "extra", data: []byte("x")}, + }, + } + for name, files := range cases { + t.Run(name, func(t *testing.T) { + target := t.TempDir() + writeDeltaTestFile(t, filepath.Join(target, "disk.img"), deltaTestDisk(8)) + layer := machineFilesLayer(t, MacOSDeltaMediaType, files...) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.Error(t, unpackMachineLayer(ctx, layer, diffID, target, unlimitedMachineBudget())) + }) + } +} + +func TestMacOSDeltaValidateRejectsBadManifests(t *testing.T) { + size := int64(8 * MacOSDeltaChunkSize) + cases := map[string]MacOSDelta{ + "version": {Version: 2, ChunkSize: MacOSDeltaChunkSize, Size: size}, + "chunk size": {Version: 1, ChunkSize: 4096, Size: size}, + "partial size": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size + 1}, + "unordered": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 1}}, + "duplicate": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 3}}, + "out of range": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{8}}, + } + for name, delta := range cases { + t.Run(name, func(t *testing.T) { + require.Error(t, delta.Validate()) + }) + } + valid := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{0, 7}} + require.NoError(t, valid.Validate()) +} + +// writeDeltaBundle writes the aux and config files a delta layer copies in full. +func writeDeltaBundle(t *testing.T, dir string) string { + t.Helper() + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte("{}")) + return bundle +} + +// unpackMachineLayers2 unpacks layers in order into one target directory with an +// unlimited budget, as the bundle unpack does. +func unpackMachineLayers2(ctx context.Context, target string, layers ...gcr.Layer) error { + budget := unlimitedMachineBudget() + for _, l := range layers { + diffID, err := l.DiffID() + if err != nil { + return err + } + if err := unpackMachineLayer(ctx, l, diffID, target, budget); err != nil { + return err + } + } + return nil +} diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index d0de8ee4d..0fa90c9e6 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "time" @@ -44,11 +43,8 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if !info.Mode().IsRegular() { return nil, fmt.Errorf("bundle file must be regular: %s", file) } - if e = exec.CommandContext(ctx, "lsof", "-t", file).Run(); e == nil { - return nil, fmt.Errorf("bundle storage is open: stop its VM first") - } - if exit, ok := e.(*exec.ExitError); !ok || exit.ExitCode() != 1 { - return nil, fmt.Errorf("cannot verify storage is closed: %v", e) + if e = RequireClosedFile(ctx, file); e != nil { + return nil, e } } root := p.ImageRepositoryDir(ref.Repository()) diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index ad08493ec..7b71ec481 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -70,7 +70,8 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } // unpackMachineLayer extracts one layer, spending its decompressed bytes from the -// bundle's remaining budget. +// bundle's remaining budget. A delta layer patches the boot disk unpacked beneath it +// instead of writing files. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { rc, err := l.Uncompressed() if err != nil { @@ -90,6 +91,12 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return err } archive := tar.NewReader(stream) + mediaType, err := l.MediaType() + if err != nil { + return err + } + delta := mediaType == MacOSDeltaMediaType + applier := &macOSDeltaApplier{targetDir: targetDir} for { header, err := archive.Next() if errors.Is(err, io.EOF) { @@ -102,6 +109,15 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe if !filepath.IsLocal(name) || strings.HasPrefix(filepath.Base(name), ".wh.") { return fmt.Errorf("machine layer entry %q is not a plain local path", header.Name) } + if delta { + if header.Typeflag != tar.TypeReg || isGNUSparse(header) { + return fmt.Errorf("delta layer entry %q must be a regular file", header.Name) + } + if err := applier.entry(name, header.Size, archive); err != nil { + return overLimit(err) + } + continue + } target := filepath.Join(targetDir, name) switch header.Typeflag { case tar.TypeDir: @@ -127,6 +143,11 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return fmt.Errorf("machine layer entry %q has unsupported type %q", header.Name, header.Typeflag) } } + if delta { + if err := applier.complete(); err != nil { + return overLimit(err) + } + } // Hash the trailing tar padding too, so the digest covers the whole stream. if _, err := io.Copy(io.Discard, stream); err != nil { return overLimit(err) diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..e7ecdde66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,6 +99,9 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } + if m.macOSOnly && strings.TrimSpace(req.Platform) == "" { + req.Platform = "darwin/arm64" + } if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index cd605597f..825196d8b 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -20,8 +20,22 @@ type createImageResolver interface { func resolveImageForCreate(ctx context.Context, imageManager createImageResolver, imageName, platform string, log *slog.Logger) (*images.Image, error) { if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { img, err := imageManager.GetImage(ctx, imageName) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return nil, fmt.Errorf("get macOS image: %w", err) + } + if err != nil || img.MacOS == nil { + img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}) + if err != nil { + return nil, fmt.Errorf("pull macOS image: %w", err) + } + } + pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { - return nil, fmt.Errorf("get imported macOS image: %w", err) + return nil, err + } + img, err = awaitReady(ctx, imageManager, img, pinned, log) + if err != nil { + return nil, err } if img.MacOS == nil { return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..fea162437 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "strings" "testing" "github.com/kernel/hypeman/lib/images" @@ -30,6 +31,55 @@ func (f createImageResolverFake) WaitForReady(ctx context.Context, name string) return f.waitReady(ctx, name) } +func TestResolveImageForCreatePullsUncachedMacOSImage(t *testing.T) { + t.Parallel() + + const imageName = "registry.example/macos/app:latest" + digest := "sha256:" + strings.Repeat("a", 64) + pinnedName := "registry.example/macos/app@" + digest + createPlatform := "" + waitName := "" + resolver := createImageResolverFake{ + getImage: func(_ context.Context, name string) (*images.Image, error) { + if name == imageName { + return nil, images.ErrNotFound + } + if name != pinnedName { + t.Fatalf("unexpected image lookup %q", name) + } + return &images.Image{ + Name: imageName, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusReady, MacOS: &images.MacOSImage{}, + }, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{ + Name: req.Name, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusPending, MacOS: &images.MacOSImage{}, + }, nil + }, + waitReady: func(_ context.Context, name string) error { + waitName = name + return nil + }, + } + + img, err := resolveImageForCreate(context.Background(), resolver, imageName, "darwin/arm64", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != "darwin/arm64" { + t.Fatalf("expected Darwin pull, got %q", createPlatform) + } + if waitName != pinnedName { + t.Fatalf("expected readiness wait on %q, got %q", pinnedName, waitName) + } + if img.MacOS == nil || img.Platform != "darwin/arm64" { + t.Fatalf("expected ready macOS image, got %+v", img) + } +} + func TestResolveImageForCreateWithPlatformForcesPlatformResolve(t *testing.T) { t.Parallel() diff --git a/lib/instances/delete.go b/lib/instances/delete.go index d9ea69a40..5cc1ce936 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -114,7 +114,7 @@ func (m *manager) deleteInstanceWithOptions( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown = m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, false) if gracefulShutdown { gracefulSpanEnd(nil) } else { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index b7a10a72e..6fec8d6b1 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -551,7 +551,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) return returnWithReadiness(inst, err, false) case StateStopped: - inst, err := m.stopInstance(ctx, forkID) + inst, err := m.stopInstance(ctx, forkID, false) return returnWithReadiness(inst, err, false) } } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9e8e4d901..909972aef 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -24,20 +24,17 @@ import ( // macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. const macOSAuxStorageFile = "mac-aux.img" -// requireMacOSImageName refuses a create on a macOS-only server unless the image -// is an already imported macOS image, so a Linux image is never pulled there. +// requireMacOSImageName checks lookup errors before resolving the reference as +// darwin/arm64. The resolved image is validated by checkMacOSCreate. func (m *manager) requireMacOSImageName(ctx context.Context, name string) error { if !m.macOSOnly { return nil } - cached, err := m.imageManager.GetImage(ctx, name) - if err != nil && !errors.Is(err, images.ErrNotFound) { - return err - } - if err != nil || cached.MacOS == nil { - return fmt.Errorf("%w: macOS-only server requires an already imported macOS image", ErrInvalidRequest) + _, err := m.imageManager.GetImage(ctx, name) + if errors.Is(err, images.ErrNotFound) { + return nil } - return nil + return err } // checkMacOSCreate applies macOS machine defaults to a create from a macOS image. diff --git a/lib/instances/macos_agent_startup.go b/lib/instances/macos_agent_startup.go index 05451f57f..74025e0cb 100644 --- a/lib/instances/macos_agent_startup.go +++ b/lib/instances/macos_agent_startup.go @@ -83,7 +83,7 @@ func (m *manager) stopOverdueMacOSAgentStartup(ctx context.Context, id string, l reason += ": " + last.(string) } log.WarnContext(ctx, "stopping macOS guest whose agent did not start", "instance_id", id, "reason", reason) - if _, err := m.stopInstance(ctx, id); err != nil { + if _, err := m.stopInstance(ctx, id, false); err != nil { return err } meta, err = m.loadMetadata(id) diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go new file mode 100644 index 000000000..13539cd15 --- /dev/null +++ b/lib/instances/macos_build_export.go @@ -0,0 +1,5 @@ +package instances + +// MacOSBuildExportReceipt describes one stop/export operation, not persisted state. +// A failed export may still confirm that instance cleanup is safe. +type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go new file mode 100644 index 000000000..c37a45424 --- /dev/null +++ b/lib/instances/macos_build_export_darwin.go @@ -0,0 +1,133 @@ +//go:build darwin + +package instances + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" +) + +// StopMacOSBuildInstance performs only the strict graceful stop used by build +// export. It copies no storage and never falls back to a forced hypervisor stop. +// Success requires the shutdown acknowledgement and confirmed VMM exit. +func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receipt MacOSBuildExportReceipt, err error) { + if err = ctx.Err(); err != nil { + return + } + if _, bounded := ctx.Deadline(); !bounded { + return receipt, fmt.Errorf("machine stop requires a deadline") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return receipt, err + } + if current.MacOS == nil || !current.GuestAgentEnabled() || (current.State != StateRunning && current.State != StateInitializing) { + return receipt, fmt.Errorf("strict stop requires an active macOS guest with a system agent") + } + if err = m.markRestartManualStopLocked(ctx, id); err != nil { + return + } + stopped, err := m.stopInstance(ctx, id, true) + if err != nil { + return receipt, err + } + receipt = MacOSBuildExportReceipt{Graceful: true, VMMExited: true} + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + return receipt, nil +} + +// ExportStoppedMacOSBuildInstance copies closed storage for an instance that is +// still Stopped when the copy starts. The instance lock excludes Start during the +// check and copy; a concurrently restarted instance fails export instead. +func (m *manager) ExportStoppedMacOSBuildInstance(ctx context.Context, id, destination string) error { + if err := ctx.Err(); err != nil { + return err + } + if _, bounded := ctx.Deadline(); !bounded { + return fmt.Errorf("machine export requires a deadline") + } + if !filepath.IsAbs(destination) { + return fmt.Errorf("export destination must be absolute") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateStopped || current.MacOS == nil { + return fmt.Errorf("export requires a stopped macOS instance") + } + if rel, e := filepath.Rel(current.DataDir, destination); e != nil || filepath.IsLocal(rel) { + return fmt.Errorf("export must be outside instance storage") + } + return writeStoppedMacOSBuildBundle(ctx, current, m.paths.InstanceMacOSBootDisk(id), destination) +} + +func writeStoppedMacOSBuildBundle(ctx context.Context, inst *Instance, disk, destination string) error { + if inst.State != StateStopped || inst.MacOS == nil { + return fmt.Errorf("machine must be stopped") + } + aux := filepath.Join(inst.DataDir, "mac-aux.img") + var diskInfo os.FileInfo + for _, source := range []string{disk, aux} { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + if diskInfo != nil && os.SameFile(diskInfo, info) { + return fmt.Errorf("machine storage files must be distinct") + } + diskInfo = info + if err = images.RequireClosedFile(ctx, source); err != nil { + return err + } + } + for _, file := range []struct{ source, name string }{{disk, "disk.img"}, {aux, "aux.img"}} { + if err := ctx.Err(); err != nil { + return err + } + path := filepath.Join(destination, file.name) + if err := forkvm.CopyRegularFile(file.source, path); err != nil { + return err + } + if err := os.Chmod(path, 0600); err != nil { + return err + } + } + data, err := json.Marshal(inst.MacOS) + if err != nil { + return err + } + path := filepath.Join(destination, "config.json") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + if err = ctx.Err(); err != nil { + return err + } + _, err = images.ValidateMacOSBundle(destination) + return err +} diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go new file mode 100644 index 000000000..0d357bed1 --- /dev/null +++ b/lib/instances/macos_build_export_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildBundleRequiresClosedDistinctStorage(t *testing.T) { + root := t.TempDir() + disk := filepath.Join(root, paths.MacOSBootDiskFile) + aux := filepath.Join(root, "mac-aux.img") + require.NoError(t, os.WriteFile(disk, []byte("synthetic disk"), 0600)) + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + inst := &Instance{State: StateStopped, StoredMetadata: StoredMetadata{DataDir: root, MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}} + file, e := os.Open(disk) + require.NoError(t, e) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + require.NoError(t, file.Close()) + out := t.TempDir() + require.NoError(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, out)) + _, e = images.ValidateMacOSBundle(out) + require.NoError(t, e) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, e := os.Stat(filepath.Join(out, name)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + } + require.NoError(t, os.Remove(aux)) + require.NoError(t, os.Link(disk, aux)) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + inst.State = StateRunning + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) +} + +func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { + m := &manager{} + _, e := m.StopMacOSBuildInstance(context.Background(), "unused") + require.Error(t, e) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, e = m.StopMacOSBuildInstance(ctx, "unused") + require.ErrorIs(t, e, context.Canceled) + require.Error(t, m.ExportStoppedMacOSBuildInstance(context.Background(), "unused", t.TempDir())) + require.Error(t, m.ExportStoppedMacOSBuildInstance(ctx, "unused", t.TempDir())) +} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 7e97f93cd..797468f83 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -186,7 +186,7 @@ func (m *manager) rekeyMacOSCloneOnStart(ctx context.Context, stored *StoredMeta // Teardown runs even if saving the pending flag fails. Otherwise the clone would keep // running, networked, with its source's SSH host keys. saveErr := m.saveMetadata(&metadata{StoredMetadata: *stored}) - _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id) + _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id, false) if saveErr != nil || stopErr != nil { return fmt.Errorf("macOS clone rekey failed and teardown needs attention: %w", errors.Join(err, saveErr, stopErr)) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index b64d1c1c9..aa234f69a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -736,7 +736,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - inst, err := m.stopInstance(ctx, id) + inst, err := m.stopInstance(ctx, id, false) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStop, inst) } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 0bf65d7af..88775bf14 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -255,7 +255,7 @@ func TestGracefulShutdownWaitsForSocketOwnerInsteadOfExitedStoredPID(t *testing. }} m := &manager{} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "stop and delete must fall back to the hardened kill path while the socket owner is alive") assert.True(t, ProcessExists(owner.Process.Pid)) } @@ -297,7 +297,7 @@ func TestGracefulShutdownFallbackKillsCapturedPIDAfterListenerCloses(t *testing. VsockSocket: filepath.Join(t.TempDir(), "missing-vsock.sock"), }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 0)) + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 0, false)) require.NoError(t, m.killHypervisor(context.Background(), inst)) assert.False(t, ProcessExists(owner.Process.Pid)) } @@ -752,7 +752,7 @@ func TestMacOSGracefulShutdownWithoutLiveVMMIsNotGraceful(t *testing.T) { sent = true return nil }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "a macOS stop with no VMM to signal must not produce a graceful receipt") assert.False(t, sent) } @@ -800,7 +800,7 @@ func TestMacOSGracefulShutdownRequiresGuestStopReceipt(t *testing.T) { return err }} - assert.Equal(t, guestStopped, m.tryGracefulGuestShutdown(context.Background(), inst, 5)) + assert.Equal(t, guestStopped, m.tryGuestShutdown(context.Background(), inst, 5, false)) assert.NoFileExists(t, receipt, "the receipt is consumed") }) } diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 227b6c76c..056bb076b 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -173,7 +173,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err return err } - stopped, err := m.stopInstance(ctx, id) + stopped, err := m.stopInstance(ctx, id, false) if err != nil { _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2d8ee7a39..4e4f23340 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -39,9 +39,10 @@ func hasVFIODevices(stored *StoredMetadata) bool { return storedVGPUDevicePath(stored) != "" || len(stored.Devices) > 0 } -// tryGracefulGuestShutdown asks guest init to shut down and waits for the -// hypervisor process to exit. Returns true if the process exited in time. -func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int) bool { +// tryGuestShutdown asks guest init to shut down and waits for the hypervisor +// process to exit. Returns true if the process exited in time. With strict, only +// an acknowledged shutdown within the context deadline counts. +func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int, strict bool) bool { log := logger.FromContext(ctx) if !inst.GuestAgentEnabled() { @@ -67,7 +68,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, if pid == 0 { // Nothing was signalled. Linux treats an already-gone VMM as stopped, but // macOS only counts a stop as graceful after a shutdown attempt. - return inst.MacOS == nil + return !strict && inst.MacOS == nil } inst.HypervisorProcessIdentity.Set(pid) if inst.MacOS != nil { @@ -104,6 +105,14 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // If we couldn't signal the guest, don't burn the full graceful timeout. waitTimeout = shutdownFailureFallbackWait } + if strict { + if ctx.Err() != nil { + return false + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < waitTimeout { + waitTimeout = max(0, time.Until(deadline)) + } + } if WaitForProcessExit(pid, waitTimeout) { // The shim exits after a forced stop too, so macOS also needs its receipt that @@ -113,7 +122,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, return false } log.DebugContext(ctx, "VM shut down gracefully", "instance_id", inst.Id) - return true + return shutdownSent || !strict } log.WarnContext(ctx, "graceful shutdown timed out, falling back to hypervisor shutdown", "instance_id", inst.Id) @@ -124,10 +133,9 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // Flow: send Shutdown RPC -> wait for VM to power off -> // fall back to hypervisor shutdown -> final SIGKILL if still alive. // Multi-hop orchestration: Running/Initializing → Shutdown → Stopped -func (m *manager) stopInstance( - ctx context.Context, - id string, -) (_ *Instance, retErr error) { +// stopInstance stops an instance. With strict, a failed graceful shutdown is an +// error instead of falling back to a forced hypervisor stop. +func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) log.InfoContext(ctx, "stopping instance", "instance_id", id) @@ -180,7 +188,7 @@ func (m *manager) stopInstance( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown := m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, strict) if gracefulShutdown { gracefulSpanEnd(nil) } else { @@ -189,6 +197,10 @@ func (m *manager) stopInstance( // 5. Fallback hypervisor shutdown if guest graceful shutdown didn't work if !gracefulShutdown { + // Export callers must never turn a forced stop into a graceful receipt. + if strict { + return nil, errGracefulShutdownFailed + } log.DebugContext(ctx, "shutting down hypervisor (fallback)", "instance_id", id) shutdownCtx, shutdownSpanEnd := m.startLifecycleStep(ctx, "shutdown_hypervisor", attribute.String("instance_id", id), diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 148967567..5aa8c610e 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -203,6 +203,26 @@ func extractRepoFromPath(path string) string { } // isWriteOperation returns true if the HTTP method implies a write operation +// authorizeBlobMount checks the source of a cross-repository blob mount. The blob +// store is shared by digest, so a mount is the only way a build could link another +// repository's blob into its own; the token must allow pulling from that source. +func authorizeBlobMount(claims *RegistryTokenClaims, r *http.Request) error { + query := r.URL.Query() + from := query.Get("from") + if query.Get("mount") == "" && from == "" { + return nil + } + if r.Method != http.MethodPost || query.Get("mount") == "" || from == "" { + return fmt.Errorf("blob mount requires POST with mount and from") + } + for _, perm := range claims.RepoAccess { + if perm.Repo == from && (perm.Scope == "pull" || perm.Scope == "push") { + return nil + } + } + return fmt.Errorf("%w: mount source %s", errRepoNotAllowed, from) +} + func isWriteOperation(method string) bool { return method == http.MethodPut || method == http.MethodPost || method == http.MethodPatch || method == http.MethodDelete } @@ -394,6 +414,10 @@ func JwtAuth(jwtSecret string) func(http.Handler) http.Handler { // Try to validate as a registry-scoped token registryClaims, err := validateRegistryToken(token, jwtSecret, r.URL.Path, r.Method) if err == nil { + if err := authorizeBlobMount(registryClaims, r); err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return + } // Valid registry token - set build ID as user for audit trail log.DebugContext(r.Context(), "registry token validated", "build_id", registryClaims.BuildID, diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4c604c755..eb421b5b1 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -274,6 +274,12 @@ const ( SnapshotTargetStateStopped SnapshotTargetState = "Stopped" ) +// Defines values for CreateBuildMultipartBodyMachineNetworkMode. +const ( + Egress CreateBuildMultipartBodyMachineNetworkMode = "egress" + Isolated CreateBuildMultipartBodyMachineNetworkMode = "isolated" +) + // Defines values for GetInstanceLogsParamsSource. const ( App GetInstanceLogsParamsSource = "app" @@ -2002,6 +2008,21 @@ type CreateBuildMultipartBody struct { // Admin builds can populate the shared global cache that all tenant builds read from. IsAdminBuild *string `json:"is_admin_build,omitempty"` + // MachineBaseImage Experimental installed darwin/arm64 base pinned by sha256 digest. + // Requires server macos_only and macos_builds. The source must contain + // build.sh at its root, which runs as root in the guest. Machine source + // is limited to 64 MiB compressed; only machine_network_mode, + // timeout_seconds, memory_mb, cpus and tags are accepted alongside + // source and this selector. CPU/memory zero inherit the base. + // Linux/cache/secrets/admin options are rejected. Output retains the + // base identity and cannot run concurrently with it. + MachineBaseImage *string `json:"machine_base_image,omitempty"` + + // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables + // unrestricted VZ NAT for checksum-verified artifact downloads; it does not + // enforce domain allowlists. Linux builds do not use this selector. + MachineNetworkMode *CreateBuildMultipartBodyMachineNetworkMode `json:"machine_network_mode,omitempty"` + // MemoryMb Memory limit for builder VM in MB (default 2048) MemoryMb *int `json:"memory_mb,omitempty"` @@ -2021,6 +2042,9 @@ type CreateBuildMultipartBody struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty"` } +// CreateBuildMultipartBodyMachineNetworkMode defines parameters for CreateBuild. +type CreateBuildMultipartBodyMachineNetworkMode string + // GetBuildEventsParams defines parameters for GetBuildEvents. type GetBuildEventsParams struct { // Follow Continue streaming new events after initial output @@ -19326,397 +19350,403 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", - "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", - "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", - "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", - "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", - "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", - "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", - "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", - "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", - "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", - "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", - "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", - "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", - "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", - "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", - "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", - "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", - "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", - "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", - "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", - "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", - "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", - "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", - "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", - "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", - "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", - "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", - "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", - "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", - "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", - "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", - "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", - "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", - "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", - "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", - "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", - "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", - "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", - "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", - "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", - "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", - "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", - "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", - "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", - "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", - "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", - "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", - "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", - "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", - "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", - "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", - "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", - "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", - "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", - "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", - "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", - "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", - "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", - "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", - "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", - "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", - "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", - "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", - "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", - "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", - "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", - "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", - "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", - "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", - "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", - "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", - "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", - "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", - "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", - "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", - "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", - "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", - "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", - "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", - "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", - "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", - "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", - "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", - "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", - "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", - "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", - "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", - "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", - "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", - "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", - "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", - "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", - "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", - "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", - "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", - "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", - "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", - "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", - "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", - "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", - "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", - "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", - "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", - "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", - "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", - "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", - "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", - "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", - "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", - "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", - "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", - "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", - "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", - "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", - "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", - "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", - "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", - "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", - "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", - "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", - "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", - "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", - "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", - "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", - "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", - "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", - "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", - "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", - "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", - "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", - "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", - "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", - "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", - "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", - "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", - "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", - "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", - "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", - "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", - "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", - "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", - "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", - "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", - "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", - "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", - "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", - "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", - "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", - "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", - "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", - "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", - "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", - "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", - "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", - "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", - "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", - "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", - "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", - "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", - "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", - "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", - "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", - "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", - "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", - "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", - "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", - "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", - "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", - "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", - "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", - "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", - "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", - "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", - "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", - "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", - "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", - "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", - "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", - "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", - "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", - "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", - "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", - "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", - "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", - "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", - "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", - "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", - "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", - "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", - "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", - "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", - "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", - "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", - "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", - "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", - "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", - "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", - "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", - "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", - "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", - "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", - "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", - "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", - "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", - "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", - "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", - "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", - "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", - "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", - "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", - "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", - "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", - "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", - "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", - "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", - "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", - "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", - "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", - "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", - "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", - "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", - "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", - "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", - "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", - "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", - "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", - "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", - "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", - "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", - "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", - "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", - "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", - "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", - "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", - "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", - "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", - "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", - "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", - "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", - "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", - "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", - "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", - "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", - "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", - "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", - "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", - "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", - "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", - "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", - "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", - "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", - "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", - "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", - "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", - "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", - "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", - "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", - "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", - "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", - "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", - "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", - "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", - "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", - "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", - "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", - "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", - "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", - "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", - "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", - "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", - "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", - "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", - "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", - "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", - "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", - "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", - "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", - "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", - "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", - "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", - "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", - "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", - "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", - "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", - "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", - "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", - "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", - "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", - "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", - "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", - "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", - "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", - "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", - "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", - "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", - "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", - "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", - "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", - "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", - "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", - "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", - "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", - "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", - "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", - "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", - "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", - "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", - "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", - "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", - "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", - "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", - "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", - "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", - "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", - "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", - "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", - "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", - "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", - "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", - "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", - "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", - "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", - "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", - "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", - "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", - "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", - "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", - "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", - "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", - "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", - "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", - "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", - "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", - "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", - "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", - "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", - "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", - "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", - "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", - "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", - "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", - "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", - "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", - "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", - "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", - "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", - "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", - "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", - "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", - "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", - "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", - "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", - "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", - "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", - "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", - "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", - "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", - "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", - "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", - "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", - "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", - "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", - "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", - "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", - "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", - "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", - "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", - "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", - "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", - "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", - "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", - "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", - "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", - "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", - "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", - "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", - "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", - "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", - "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", - "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", - "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", - "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", - "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", - "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", - "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV0s20qlvuPYjqMdK9a2bM+3J8yhwG6QxKgb6ABoyUzK", + "VeeveYCp84TzJKfWAtA3osmWbEn2xDVVE5mNOxYWFtblt/7oRTLNpGDC6N7RHz0dLVhK8c8nxtBo8U4m", + "ecpes99ypg38nCmZMWU4w0KpzIWZZNQs4F8x05HimeFS9I56p9QsyNWCKUYusRWiFzJPYjJlBOuxuNfv", + "sfc0zRLWO+rtpMLsxNTQXr9nlhn8pI3iYt770O8pRmMpkqXtZkbzxPSOZjTRrN/o9gSaJlQTqDLAOkV7", + "UykTRkXvA7b4W84Vi3tHv1Sn8WtRWE7/ziIDnT/JjTwzVMTT5alMeLRcnexLLvL32BuhuZEpNTwi2tYh", + "GVYiU6pZTKQgNDL8khEupjIXMXnz9JREUggWQWN6LORUM3XJYjJTMiVmwchCaoNljKLRBTF0mrDhWPT6", + "jf1gAr7Em1fprwtmFkwFBss1ca2QmVTELLgmXMDXiA2rG2ZUzlZXtt/jccImhqdM5mZ1oX6UVySRYo7T", + "8u2SNNeGLOglI78zJclvOU34bMnFvH2RpmwmFSM/LjOWUkGyhEZME24IF0b62dg1KmnsQRoiLj4XUrFJ", + "zLThgkL7k0wqo1dH/wr/oAmplMWhYXliFtR4KhfSkAvGsvpE6RW9qC/jL3t7/cej0ejXfo8bltpjRd/z", + "NE97R4cPHuw/6PdSLuy/d4vRc2HYnKneh+IXqhRdVqajZa4iNol4rNbNJEo4E4Y8PX72+oYT6O2Ohvi/", + "nUe9fm/38d5w9/AR/nv3sFed1srC10f+Yf3ROzPU5HqVB9nTNHGEMqkQyeqsf87TKVNEzkiUK8WESZYE", + "jxSLOxBdbdqj0FZEUsz4PFcsXu3bH7naci6oJlRYpjFo8IuysU7nLgImFssrMVEspVzAGq8M4rX/ROCE", + "EneIYEiRFEbJJAGmYAxLM6P9KeoDGxeEZlnCI2Q9tUN1kI50r98TeZLQ6coIy91mCZ9zLNBpabiubJKv", + "S4wkTBimihPeZWlqbLGt43K5g7tR8sVOXS5kEk9yYXiy2utb+NmtabUnrsmCJTGRs1mf8BmhBFqBny2N", + "15d9b7R3OBgdDEaHb3YfHo0eH40e/K3X782kSqnpHfViatgAdrnL3iDT1lxE4d0RTRJJ4UJSLLK7U1xY", + "tQ2cskimjEDTa0d+8PEjb71unpabCgWJK1jerIGtHjaujG7UnVBtCiaE+8XNckIDY3rDU6YNTTPgQzCG", + "ymK2cSHfYHMf/MqvXeDdj1pgwd6biVuh4HxC9MHeZyyCG1F6blIIGNCeI++CZd0JjStGtRRWTILL9Jde", + "LnSewdXN4kmWUAPt9vo9SwaTlGsNVYsfYq4tH+n3PJFPhDQTlQthCwpmrqS6qJZ0rUx41uv3FlRPLudZ", + "3uuvu7bqRI1dsIRmGttzO64mTCmpelY0Xk5mUvlN6v1aXcI1Ta2skC6u2MAK9fq92gIU7NzPxY+72NXg", + "4Po999dE2VcFNmcnszrwalurwy2Gtp6x21vECtF+m4mrrOscIOZ0LqQ2PNKd2DwKD7C9qYwDrPNZ0Rzh", + "MROGzzhTTq5mRMH9kDLiGyHQCOGC5LpxDgrRf8IumTB6cnkwMVG2uiiNh0118yqySXkjVm7lYvuLk7KB", + "SOtzDz6cLinHM/mMXfKIrcpubmsmseKXTAXYdyEAWFZoy5EtOOvAQoQUbLu2UuKSx5x2YQcxjmnCA9Rz", + "+vSY2M/k+BnZWrD39U72Hk4f9dqbFDQN0MKPeUrFAA4EDMu3j2Wrbb88CLXMZZrmk7mSebba8vGrk5O3", + "BD8SgRJutcVHeyFJNYv4hMaxYlqH5+8/Vsc2Go1GR3TvaDQajkKjvGQilqp1Se3n8JLujmK2pslOS+ra", + "X1nSn98dPzt+Qp5KlUmFb7aNB6e6PNV5Vcmmvish+v8+50m8SvVT+Jm1rBNTmmvDhCHf22JEMfuUI1cL", + "qRmJaLRgZGofLfg8x/a6kHzRsb+9QiN4xnUmkc+TdyelNIdPQ/aeRbmp9fstibk2XEROqKpMrsOIQP2U", + "MLh/Q5IFrgBxZbgUxHjR6cZCQKQY3dAdlOjU2er5zy15TVLd1rovQrggKU8SrlkkRayrfXBhDg/aJ1M5", + "xfbaXOnqOfxMUqY1nTOyhWopfG5YDk+4JjPKExZvd5Ow2ybzdzmt3Gu1M4dkMKDTaHdvP8jQUjpnk5jP", + "nV6xSYPwO8jH0I4hPG2dCEoY3eaBXSo2W+3vB7xPsBPFZkwxEX10d5mSl0xQ96T6D+y393/tlArXHadt", + "3cHFPC2Lf+j3fstZziaZ1NyOcIVNuC9ARrjUBGuEx4yf4u1OFKUNVevPB5b4BCexFDY3ro1T/YC8Recb", + "q7yBMk2GjuyoEHAqXKCVbz8HSSsgskhhmAiszks5JwkX9hnEhHF7gcLeMmPfJXK+3ftk61As/+rhh3Hf", + "gHnZH1pag2/94lWQyHl1NReMKjNltcVsuVddQ+XoWpf/tHZ8Ghco1WyynoOcciFYjDp3d7BtSZCtg28f", + "PEUX3Ewu4QYOnTkc1k/cEFeitalERhcznrDJguqFHTGNY24Vrqe1mQREyJoxg6KSwDeIog0+qs9+fLL3", + "4JC4DgJr6LS/UGB1JpXa0LwtSwxVU5okQdpoJ7fr39GrFBKmgFLh23b3FBToCdNyup7bTfd4z/XC/oW8", + "G0aFd1+v34uAvBL4e4Vw+733A2h1cEkVrjo0XxnWf/ueKr99X3Za+fW06L/y42s3lMpPP/hRVX57Wg7Q", + "rwpTq+fBPdNw0mGpUnJhBlzgFoAA7jRPU7t5xSvLaTKIFBUJj6ku7HydYPV0RaQiW69/eLq/v/94u7tw", + "xfXFRPPf2WQ+XSs3u0E7QRmqEahGuCBzPqfTpWF6SJ5SIaQBY2S0oGLOYkJnBmq5sdYe4A9Gm4wwrUIS", + "U20ikvn9gD5+9P49NY8P+ZV+/Hs6VfO/79Nem3Yx1y3rixo46nYTihJFRWATb3ZRp/Q9vPwtlYSMnyfW", + "XEWKQm4PdHMIQ1I+6Gf8Pay5Ibu1lQ4ubvj1V5iwhBSDXPDfctztLKHL1TegYTQdIEWEltfyDXuA9Obj", + "o+H82Dp+pleUGzg5hc3UTblPZBLDvTPjSpted2vYtQQDpj6ViFQ7ZWGJaZUimgvYeqkX42w9KwmfsWgZ", + "JWyVuXv2nancqVtjBncO/lkoELuycaYCnNj/eFp0Ufv5Wdlf7ffntvMP/d5TmtEpT7jnzHU+7WzyE6f4", + "27RL1cae2aqvXc0P/d6MUZMrFljLM/QPIFZPPEjYJUuIKw3Ee0SenB4TnasZmsyRWm1ZUNtLzfRYKDan", + "Kk6YRlK/WvBoUagrQfWgCRWlWwB5klzRpSaZYpoJczQW40JDrse9Phnbx5f721KJ/du6hvhCYq6Ydv8A", + "XcO414e2EjnX496Q/Ci1GcBz2clSR2Ts9DF63CNb9k+SUa3NQsl8vhiLlAo6ZykTpk/QSQPVwMCZkuU2", + "oSIm456SmhlDByzNE+T90NiTLEvYWJzxhEdSkJRGr85c3StuFuS1rVS5PHHGcFn3SeZUz5qmbCyoVYPC", + "Ti5JpuSUkYTmIlowTZiYSRWx7SE5ZWrgl9jvLaGKwWZYJS7JBRwRRsvN0L/8Spgwynk6dGcuMJPr0B8s", + "ffGI1tepeWxrfCisI9ep/LOrAqzKzTj4RlHLUpteoWd7Ixaqfn8d1SwB/bHAFcVd5VDqSpDals0SOkdS", + "qRwiS4x6LCKZpvjQc70QzS6ZokmxQ3DHSdQnwN3gbqSxiBKZx4PFMmPqkmup+nBDsAidEVSf/MbSvI+d", + "wl+DlEdKXqZEirFIgIx3aBofHmwPyROScA3E4efPLXE7EkNOgPoADqMd96jXyY97dmJcE7j4h+S5MIpb", + "miPaEtzU3qQN4uq6eRVmtXK34f5cp7EzW2PFwmF/dgTdX2GxFbopCbAg4woXDd1Zaxjwqgzu13W9McqN", + "r9itiAq3UzXqPCLcEJpljCpNuBiLkppEjFTqKmWK4XJobtzWpcyQrZQaZC+gtR0L5BDfaFLb/e0hQbnR", + "+oeVHAmqEMWSpR0QGws/5iueJKg4JDl6MCCHswcNNWMVDwo8TFQUvC8prV3O+OXsorUJ1I6I7fu/n5+8", + "JXqpDUvJlAuqltuE65LXWtJcNcyFRUa3e/hyR1GtPutYotcT1CVS1AXI5nndaEJwImhJF5so7EcZcrGk", + "KlqE3Oi0IU9P3xL4zA2LgIhr46UqPQyakqRuaQ56pSjE2vWuNYdMZ+OcJRwpHPGmyR4X90hYRCrM8avW", + "LqhZseOyhEXAq6w/jX2/MeKMy0SmyNIFW53NTusS+cb1pt412ZJ6Bya8XblZ4EyrXAzJK0FQjCA1KWIs", + "sCgXUZLHTJMKPyel+tbLFzU3qILsx+Jf/+//Z2WJmFxyWkgb5B1XBnxIfrev2JmiKQOmV7/SogWLLsai", + "KYYQaFVLQv2lUiwz19ULRfH5whAhr4ZjcWzHVAx4S8uZuaKK5Rk8M8lg4EY9cHLWtnXiwoa4abho/tLY", + "m8riXNOXsGbOKza0v0phm0j151JsqdPqnBp2RQMP4hdAeoNLrtFlDWnClSXHp0OCn9GUEC3sV+CiIMWS", + "LUsVXMzHAr84kXibOInWkpkzTQ7Jq5SbgviFLK4Xd9ONBTgYThkTaEVMLq0DckmqS9bcAPDm3AUD73A3", + "dDTmMPaJkZN5HtS7+rsu5opFhrw7GRg5eHdCjKKzGY+AjjKm3KjtBdMcdJCfg49B0rLSvqJ3oUiG8MhQ", + "PJ6DkAPL5UR/+5u9nDKmYFxvnpzCBYer/y0Z9wQ1tk7J6QdgyeExi8nPT97A1Rq9Ott2q+aeprbhHlw7", + "ISV4v6fzqWCmbfz2K/rghrcCXWk3MV+7RCtbtIm8P06igYvfNvDNGpmk5GAgnYAQW5DgkLysya+2QuG4", + "AiLoqkRftOAP8bekGO13KNCQlFGhCXWyfmVEhOux8NJHTeAASZu4FYWqKheWS9n7cFDhviiWWHkEJbLC", + "Sf8S/n9wqWV04Vw87Cuz4LSCpni6q0u3Illx00nI2agCqDxYAkvqe68JnvBG0IJmeiENvqr9P3YU00Yq", + "tg2Pc+cvBd8zmmtG/kJSlkq1HAtfvm/PWUKN9WEo6s6AMfXIVpRIwap6BOtAQMdCG5llLHaGiW/JzB1u", + "WjgRSlXoqm2hsXBd6B0rAWhSzAI3oLTAJMtykysz2UZVA86mmBbMOU/tsBfSZEk+H7h59shWwi+ZmzZM", + "kP/ObBtgRZFSDJw/JBQtdSf4iRiq5sw41bPG9pFmxr2x2MJzS/DfYFhJc8EjvM2x3DzLB1UVR49svTh9", + "u1P6LI1F5TNWAaXegMtBwlMO7G0LfiDHO6+Ighsaf+ZibkePhe10iklieTfFIXlHk9ydU6R2WN+x8NI/", + "jnRgVG4Wszw5Ipe/OyGsvh/FThZS0e5+32maEtQwtUoy706IppfMEyTWgie1NnCMaHzJlOHaaWMvyJYj", + "p4GzcyHdaSIYi4mQY1FtCw4ykikMsSnyE6uI0aSxSGNRim0NT3J/tXmTYaGK8IJPd43N+hdNi43h418t", + "13ginxWP+sZFkvF2qyroIvGg0Khcpsr7lZFXGRNQKpZRnjJhCBcz6YvWvchGw702z7SW7s+q10rRvV5I", + "ZcicG6IYiHFSOG427g1irgwcf53PZvw9viJz1AJZmQZb0n3gUONeLi6EvBLjXn2YDw/p3iye7m/cDT/s", + "fm0Ng/uATM9ppVvj9dbb0J6uNZiBqEmMhLdz9eXvKPwTmMvKkCS49NQAzkrCWVyh7b711p0zwYBzxdYH", + "1BAvobWYfAZBUfbWrUrXssS07Kj1nG3d0A1zmCdyiheen0jpdDokx/W16xOKH1Bgrq9yES1RcQwlW2w4", + "H8LtkkV8AJ6hA7o3GI0Goya1JwcD6/KeUWOYggH+P7/Qwe9PBn8bDR7/Wv45GQ5+/ct/BB/kHb1VvV3b", + "zXPLn6U+8YOturA2B7rJvfXGlrXq8NsPL6oWWnc6UgyPAU02jgE8Dp5WirfT+tPjVa83u1KxBD30kMud", + "hE8VVcsdMefi/RGIc9rU12192XUKlkBwjBWMfAFCNXEall92LqniVJhf7RjJuKocAPmN4K0B9/4zHBEZ", + "DHw7jn15z/uaAWBIfpaGUGeRhifdERZa7XeuaJpS1HZe0oTHeDicmU3zmFnlrMBv5NJKSYqZXAlyMBr5", + "L5PKoEBukNpPZdxzilMVLYhTCCmCqhAbIROxDLo0MqaNMKXKUqyeNDhmf9kp/tryfw5//cv2//6PT0rq", + "SGhraNyqM67Jzxp+3lCIbCXyiqmIakYSBrPVfRLzOTfaCm8xukahQu1bEllnD+u6KBVhwquosVydnNPl", + "gGZ84DQv1tz9kom5WfSODveDi7vl/hj8+p/+p5aFVXkSeq+9ljlqXvFzNfDaj6GT8cWvbp6g0SXl4thW", + "210VJz9uh/1E1u20fdS1bjXcMUVU0IaBrEbgg8dRGhIiLplScBQtX3t68gyeMBfMMTR4NZJxPhrtR1gA", + "/2TuFxDi4CmDv23XJR7bXEjY+aXHooVEN8gkkddRUvabTL3NS3CtjSy02iX3X/UpRCO6NYujs7srSKZK", + "XjCFJhlYYs40uWBLqy6Y1xSaTFySS6q8afANhiVgEf8JrUEMn8fwiMXgfIhlZcqxRfdms2FQjCYlt0wp", + "mLvwXakjmTFgqK4YTo2cM3F5TlKa2We9YlZpnlLDFIfnYqlYTFnMgUePBcODQTIKZz6KpAIVK+wt6mDL", + "VfhGk3P7TDzH5s+5ALI+twezoSr9o/fq7ZvvX739+dnk1enzn58cT356/j/ws63UO/rlj54F0ygcr75n", + "VDFF/uMPnO8H6zyLPn0Q6r6Qyr12vYke/VVoxocyY4LyYSTTXr/3n9V//vrhV+/uaSP3LntHwYF9CIqZ", + "TvcZ8MH3AVCaSFV6nlLEBUEW5dUOFXVDQ5VvJb9rHQl8onA5mWY6GJvS1FuUcujuaHTy/Y7zHnng/7E9", + "JM50i8MHHiSVE4/1AsinwOVAe1qSyMhFaMzaQvB9VyEGj7bWTHJhNjKnsugqjxoMyq/XYEU7Uy52NGzD", + "ILreuiPd3NhR+bm45EoKfJ6juDRNmK6flZ9fPXs+ef7zu94RXARxjnGEvX7v9NXrN72j3v5oNOqFCJS9", + "z7hiOuj4+GSqZZIbRrBQ6WM6JCe5RudOxzxmOepBUT7TJM1BsZQsCXsfJbkGPoXSgDFJKP740WBv983u", + "o6PR6Gg0+ltnh1Ug/Q3M+8XpWxuhjued0cQsJmia21TxRyz7FIt+6HvdJD7tA6tU7KnXVsLpdXXI1qIu", + "XdnJESTIcW//xff2QO2+GPcqhOhCgYtWbMON1XvxfWhdKtqoVUmv+GYj3ETsqX4VjMbq63Pt1ERVS/hA", + "UITYgZIxVTGZSqrib+u+NFgTFPnfaILaCpo4O5EVvbGOlSUl0/hW9j4+pbJV9xtLgJqf1KoSqSAMLaWX", + "HFjOIE259PXqbMkpU2o61GHFwBTQ41XchXr9HszM/cdPsNfvXf5ej6nerA50rjGd3oobHoE0ybhga16B", + "n4mwDzrlRNJ4sPuJZf2Kv1sDhsZ+qG938Tj1BL6C7jSlIr7isVlMICIehhy4Id0XUhQursn3MBOa/Osf", + "/3x3UipEdl9MM3dn7u49+Mg7s3FLQtPBWJtiInkWnsbbLDyJdyf/+sc//UzudxJWpLyReO72/7ltoQlQ", + "43C7rDd+i921kMPK+FzZYkvfgGEQunLlJVMJXVauEzem3u4IeXpjVIpbc6OrV1Ehr79coDUvrlWvF2Dz", + "M84S66NlXW/xfaG9hcjaDvG0AE/6RvsIJEamUho7AlgbRlewUcK30seppIbu5oikMJQLptx4ixvDXin2", + "xZIeHgzJ8/cZUxwEJpqQmKorLqzTi6/p3tyECgAESrhgA546C65dEjtJ5MnQMBjavMORtg5Hq1p7VFbH", + "xN2PtrJUDYXYbWqWFEPGPMkKJL11x+e1LV0++gMkGaDI76kuLLNd6LAgw929E/fnniXIKvEVh8ueNa4K", + "4rNdfQtiKDydDXaqLazdlBHoiVFtyAF5wb8nXEAof8LICf+eUHc7cUHe/c3uA/I63UmW0hc8s34eEzov", + "wlLXIf+dXfDMveexhvcysHdQnEPLeIaGY2GRfYBtIO2AR7633RoIItDWcQFuW7zPVmU00A6UN5otbs20", + "Khd9Ms0NPPmlYcQpXxzEAIwFC08ZyUsn0sYD3E1w1TECl+WCKcGSiX1g644rYysRV6l1cXCqM6odEpky", + "eVZfr2c/nZyRrWdLQVMekZ9sqycyzsEb0PKE7frq9cciUwjvAp3AWeCuXzkjMjcDORsYxZgfYoqNFXYY", + "B3Jx+eL0rYNJ0eCk9JrBwjIRO4dXL+wUHq/iG2NN4bVmq/03Fr0NCseb9zue7DNXvDza3TWS/Z4xSQjc", + "LrFsDYKIbBwFGK4K6vOhfX04gS9kicVgucGQvNXADUbg9iIVei2QqTSL6usSryVdfQGVEJplMXh/hp+Y", + "5Wu2cb4PFkG7dZTlddLd67diGl5ahwkQZ3RXznUJhSuMCwGXApxrryOjCoLNuCCf1R2z8LJVvZIdYLln", + "1NQBObqqwm3LCAa70TvUabRxSbprtCs61hXdtlcA/tExaGJ9+8fCD2StRr/UAX5EX2e2keYSubb7fmY3", + "WKXjYk0adoBPszxPdEVn2gm31co8Vp7RZOsc1KyOjkGxet4n5/9Z+wH4qZen4LlwRexqII8W8FO1fb3i", + "2bxBj3tj72aqb74fT3Qr6ha53CVGUaGtn/2CZmxIfsSLkRiWgrQINxWvuhYKefUtkfaN4quOBQxN130t", + "vTZf87kA1zN0mgeGE8dW5e80dt7/Rofgjr0ebkVit6NzoEXIy5y3PTn3qvfzOv9dVcyv6q2cpn5FYYG/", + "W0UFqnLMTpob6N6+XpxNWubGgpi5qTcEvLr6f5NjkBtLoYq8wf6fFeyiiSF9GdBYwOTkrLTXVAxHbfYZ", + "J/yFbUcXbIlb7u1EdMVSVDURhQ05zqXdiTKVqqg7tFcJ1xX7krUUOeMQOh/Xj2jQbLJpK2C9Oi9//eUf", + "jMMZ+MmWFOMe8x6f1HMhmJztD5WOmuHioybhyIZ1nfet6oOhfpEIIJbE++Y3m0e3ZHRiPHe/DF1r50WQ", + "4KfAHC+jP6pbSyo760VpbMaLYobF/bpsAOjUevOkrHvqosXsqdiV4p6ReYDLjiKvi5pJ3cPr4/RAzyuN", + "rXEI69rEapSsXd/KmB09WWBnH0Ri9wPtXzXE4ybwf9x4hlvkmHqX5zRJzsmWK7RNFPs7osK6vQIPu2L7", + "AeXWkUDhbvbupD8WlgucL4zJJvB/egKn+LzZmKvrT3gJyv5ohKLrwcF+PWbDDrjRbF1dHkTDad8a8Lu6", + "Jf+tFqW8R7/Hz8UNBIg3ZMvQORwdB3sE98T2xyrsudAsyhXbfNafoHjkoHdwTAnlYvDjmzenRLE51xhq", + "HAxqsK7xLRhtTllQuqv56RpZ19ru7R88OHz46PEwvlBDFqlhrgfwkBnsDmlKf5eCXmkQu3bSJc2yo8vd", + "jXzebkExvnaW71+27aQgU2DJ3iG5yyv5aVmlNBhecBF3beAnKLvZ6dU/4m/b7JIpNsizuaIx+8RGlxv7", + "NeFqtu/qhkwo6x2ac21AEVFCAG+t92OuL9alTAYxNfQ6bsx2uKsey+nSNmWf5KH2Wt3Dz5o+4TXcnmA+", + "hI91MvNjCW1LG6awVSSweGJkIHzXyw7Hz2DNfdlO4KkIgWzk5HLG5aYgfxhaPXQXq3v1BjQxyCLuNHXe", + "qsELrb9V1r07qfrWDMdigFLYEXlWdFA0WzRJ8YlBY2vl35KqFj+MhoApQJ+QdydD8qYY7TfaGwXsmEgR", + "OZqjQRGFooEVyaoDyDHQhptmdaeWtHjM2+hCJN23YZG1BRWghQIN9WVT3piPjevGjXJ3SRV6pqNCeB06", + "2Wt7H61DKKt5hDwYjHYHuw/e7I6u6xFyC5DTobae1HmL8/2vcp+nb4+f7Tm15/aNQck+OSh1mJU9K4MW", + "yFauWSUgV6pgqEIlIqAlFOHGEQa3FDBQwm+uK2tX4g2UvA207RAeGBbp3wAPu8kwN4KuVia3qtBZZqh+", + "KE9JRZC3O5pFPBh5DS6D3ytGLzCLwOq9jTgUE3ufhf0NESxkukRwLFRyKSnNTJMCQ6ZUfu8ePDx4tH94", + "8Gg0CuA5rxK8jPgELbKdBgAuOAldFtCGW+jGEJNpIqd1Qn+wf/jo4ejx7l7XcVjDfbd1KN7fvhbZcivy", + "F5/pzX+pDWpv7+Hh/v7+6PBw76DTqGxj3QblytZFkof7Dw92H+0ddFqF0LvuucfXbgrwoWQPT2x+JPjX", + "QGcs4jMeEcSlI1ABkQMWYMb3Vuj6mZzS2KfECN8dhvJEr/Xus525klbfmuaJ4VnC7DfckE4mCZz5M2wp", + "mGNMiCIFyfVacqjkG/2+/FyKIijOxGyaz+cO8q9YuhMXzV8KT5wl8REpnmxr+RzuZjmwX9vowM2hIzW8", + "lFcF+F6FCOzVZZEqFCMFndhNq83KhwtxkeVBkmhdyh9yZUE4sFFCpzI31oLvcsaUnVg8G4EW7VzE3dQd", + "P0h1sTG+A27iIjVOR4WBD/SfIRBAI8S/KvQVlnbrj+C+a/La1rCKwvLnLK/nBexjT06hKDw6QVzojV0z", + "XaXLsNyCOnPv1Gf7K3nnHXk0DmbWCezTvrDVnGFKKLNRYgFKsf5TZ1i8c7gYVNyoSOmw7oJd3cWiY8Dj", + "AMh2ACqU21nxdY7Tha6hLESkCzcAfE6YeVyBl2qctDOPuOGWfDgWLt6q+Mk5BUBFuw7WkUAqPuf1jut6", + "1jt0Xb4OZXriujF1Viu2+QzqwgwY4gEWcHpmPYGt92IV79btSa/fOytyZznGVF8a/2NgRcoAg1XoodO3", + "13VAzpSc8RAeEHodua/uoeZdc18ejM4Gu/9tvTuB/FBi48J6KoEdYdjIf4Tlu11EL07fnraNqUg+Raqj", + "W5nTGpSj0rOmjIopFsfFmnPtW64k6qWCGHrBQMOEJ65WzMJ3sXhInlpfG5Rr+Aw12sYXs/AqwIBs35fs", + "qAaVAz5hghWNWlNqnANVVXHeoBiIAtpbPREj7sXp2zpew+OQ+I2AK9McBjZJA/rAH+A7sQUKwIiT7+si", + "+N5BqOkw4z6tERC+9Gc0Au+AzhQS0CE2prEJkfHF6dvXLuFSG1Y1kFORlMnCVA/Jz0VKMlheXeJRDQPK", + "xQYJWtdzKDDRiQw5sPygGCP4rUyFCnSDRm+Lblw4GtAY0a2k8AkZwfffYQ3+llNFheGCxeTdD/pbMnJW", + "9nc/EBt/5HgW11W/y/rj8mFoS1vjCE8XSw26PbssNikKF1XFJnKBzi+T07KiUwEH3ifhdHye45AtSMKI", + "C3j2enD86t1OGrPLfm1M8NF66b44fbtdYcuXHsyiKFvnxpdtGia/VUV2yEmZkbJ5EBBH26k+m9QBezMi", + "UxZR79Td3LyoSOA8RXcNVJSBmdswQRR1emoqao0g6wLH+zxJho5n+iET1EzPkHVXsF/t7eaYmmIZxcdX", + "IXm9N9gpIzqPIsbihsdLc9QVejsiuUip0guaNCd3VAAsRAuqaGSYIt+8/6YXzsCErLTr3VChzuJy6kyW", + "lasoQI+Vc9d2xN/9UD+eVjqwYdVOp84FNwgTh9jgW8UgLZr5kJzQpVO9ZHjNOnopd4zOZhYstbhHFEsY", + "1Zg5XZN3P2z25TTS0KRtDm/go+NTWzAhOEUwzD7JamwAfq+ilFW7PQxeGeV8ApYp7LBp5hiSYxFme9Zh", + "CHEGLWSY1nwu4Dmonc8Q3Kd+JVf2rn53BjVJTVzGXn3hatPpBy6AEMWEbisbB7r6MlrN5Cov6mxKXmzO", + "Hmobae/3qQ9VbZjC3PPTn1uMaPXOS8RWdRzDKlQ1y6iihiVL+x4pqLOeDYKt+qFAtMA1YmWfQ/EPNgNQ", + "rtjELBTT4DNWU1Ds91ezSGuMSrhkLnGenVPFXGckSam6wFPmn98kF3YF6nGx+5vAsRbGZNeYFDhVWJ2c", + "YeqSJs2ILb3il/OMQZTWlJkrxoSfCtWEVr3imxHvuiV5GcTyMMVlfQ17+4F+z2ykBJkrGjFia/kc9AVb", + "g6PXdSl7+y3DggtH65b93V23v67qLE+67XFoWLubdthE19ngN09PfRKuIsm5X+a91VWGRBL2yLkaG7Z2", + "T69PB+e7EhZFe0VmnzLMD+dizapR7N4rEfPfCYzYq0SOV5iDrsY6u37wFNil6ttz/munN2HzuIfcbyDS", + "KJQsHj+4KL05gg1ibILK8eXFYysp2Tu5cj+7SCqQtLgAXOa633eUq6TX7w1mblZHOzvA7xPEWz3Y3320", + "s975e63Xv3NynMR8nVbIlXLOch4CwIJV4qTrJLFDs6yD3hxrbrofkD2tbIJNEA93W+WR5UNcwF7TcF58", + "TyPjU1qiIr3mKEGrxxZR0euCDDSY2ixV8Np9XD2fo6DnCrih1sl/Z4X2YWbWExloxCopG+toifz3oHQq", + "VShvoVTGhStPmX+0F/ehdwT2+N81k/roUXWWhw8e7D/YxIeQ2TSOuTt3gana0nVABlZZbnd+XQPwH12X", + "OdyX9WSF67KBpt48DZDU+gWVGRPXWs8HB/t711vPrhM59s6cDb4UwrMCACuUicrI4pQZGlND60wGNdDA", + "ZTAxFmUpBg7Ovl3PWlq8nqoAVTfN6vepfGZaEny+9nj2KRV8ho8kW7Las17QvQeHRzYNccxmBw8Oh8Ph", + "dWF7nhffum3FjnWtrSD4DPXi4/bhFtB5uszlj97pkzc/AiPLtbKX1o6ecnFU+Xfxz/ID/mH/OeUijOrT", + "JXM1n61krK5tb5Y72HQWH8FMhAvG8pJSB6++FhMSxhQglFgQI7PmW317YJgFjfN6zpNrgGKuCeUHcQU0", + "8q1rc+Mk1KLQhZpK8umqXqFDImpw/Vzn+eE1zljG9Wk1HUWO7lV/mBtlWddrE9Gu5CnMmChSzyaJ/SuS", + "AlHJQ3loa1ek/9YhjSFeIy5fYdFl9cei98qPT6sDqfzu0yBWfnIJaX+9piPbWoH0r6ty6GYu5MXRDYc5", + "rP4vboWuyb8diGYwguOe78KbOHvWe381/6/f/o8+ffj33d9evnv3P5cv/uvZz/x/3iWnrz4KoGk9fuq9", + "gqB+MtxT9HCsgZ92JaUTsMys0hM8/1pW2H2xGgcTLTBdMZmyI3DBfskNUzb5ZSNkGfI1MHwpYS0iBWYp", + "dGgW21D51PohQOU/vBj8odlG7GArlNuQAihJ59NYppSL7bEYC9cW8RPR+C6Av2IS0cxmsOQCFLPg2K0w", + "3agzDJed98kfNMs+bENKFGoIe29TAWQUMzbOCtuE8G4dblTWed0VZwXeso0jHoviti4ARK13wLDMNsJZ", + "0oz0a1mU9e8393J6NApBrWIUGmwkZtTCJ0hB2UBGRXgceTTaXn3PbXhjFDS0hvyc+4wNU4Yw4wAptoQ2", + "P2Mxj5Cv+OheW7ISWG0pzSnxMiXfL3FvXtuQ05jQ3CyYMM4xsk8iKS846+OW9gkYZ9FfC2taL5yFzAbT", + "5WAhswKGpbCFWz+W+iP7/wzcRAfvmOIz11MQTQNIJCB04pFxM8MypXZhZWJvbDIiodEo74rbrFbaJjKy", + "ARgmV8Ln2mGYcRfBhgrqYyGZ/FsSJRy1TnqBNrwFooUaaCYE+NkbFYFudBrFbNb8d90Rae/BIb5g/b/3", + "9zrHmdulW0dleRJ406ae9XXg2JZN4gCseDDxivANroNwA1pOTFBPYST+94z4hsoTVzASa+mzka/apW5J", + "dCXmdTsYnOiOgQMVmtDcdJ3d6imsxTp2aMH6FGG1pAOc0XOcB3nz8owYplKPerEVwe7gKbHYNQOude6y", + "CD55evJ8e9gLosTVTFq4VWtjIeuDDiCk2HPQ6jpV6mhoyvrk+BnGuLtrpXyLYVDSD1KRxN6K5WV0hHA+", + "dXUPtflHj585ATRZlo5KVmwZ97Z9i1nzejsir323hBZDKaKVS9ryTZaXCTbr3FZtxNRK643gdrSPueef", + "u48xPgqTs1pejEi3rfdXd52jW3H42NSQXftCqlRs139V9v5TY8J/ehl9/3oyurNCT7IF1SHqXlStmlho", + "Zd+rhuw6u1ftnn61fieaB8/WX32Kqso1ZBjcc7XqIRDkB4Pd3Te7B9dX310XzruO1FfB+CwQvbtDcd8G", + "pHUAoJqbSWtICIHPLgDE60XenUB0JSCvwceGdmR3/2EXpQT22jWYovIRRGocUsGlPPJeEQRgARAveJJY", + "AUbzuaAJeUy2zo5f/HT88uU2GZBXr06aW7GuRnB/boDsjbcArqN1jgqAr9WAPEgR8fvmzUv4EiUMg6as", + "HH5xc7zvjarFDvjffnAvTt9CjQXVE+9u3R5hTMsoffaea6NXcRc7RS18DN64rVoqxrpM0rbhnCo3gpb/", + "WEPVDqJ4bt8C2rgPOVlZznsA4r7P0N7PDwR8LWz3x2JvOz3DLUFvt15pIdjqBo7Jg7bb7eYg2rcynBoM", + "VIhtVSUcj7twY9Tqfo8HYs6fOD8+cnxapqkrjRG++cacHu8Ndw8fYabl3VEXxp7SaE3fJ0+edu98tGdv", + "mSM6PYriI3yw39Rm5QjbPkFocgWRuWP/SBz37IVZed1Wjq0t0y3mYhUc/GZY4E0xrgWWG8VZ77ikJ+na", + "REkdgoqbUIdp7kBjU54kXLNIiljXZeQF1URnFivZJgwqJPixwAH2SZG3HaUUQqNI5aXq0UnXTt7PM0f3", + "mLQ4k0Izl7XkJ7bUJOVoBC26R9dHTYrYtXgstpSPcywCGjFfcQw/YNRQn7inUB+9hjExElQYC73IDTCx", + "bQhAEZDeWDmtLJlytBhtE53bJy2OF1djCQxT85ipsYBiATTmP4rnydEhICP0e8VLDpK7jEYharpV4+fQ", + "wa5jCxgCQ4VLp4fe7ioXJBcxU+S11MwYSpglh6aL3DUNpx+Jce6rdxOvXPVSrgofzE2Q6J2EqY/GeMah", + "trzP0Tv0Bo/zBzcX0TtFCnr51cUIulqT63gwuJAMePFN4bazCjkWOzWkZi7Wo4i9eWsTB9en7lyPjQSj", + "ulqSdycnNbcHF6HRbeLIJVr2QWbX2oa9DTqSjaO5jnm5goi9GQX745Gpm5JKRUL85DjUVQujhy6wFFpT", + "bNUezuv0a/CqDMZAcWH3CYhmzQQbuaMt0EvpGFltVS/1zKGyOlHax1Q4RPoCtKU06uul3pnmeieL+I79", + "rncQUecRIuocBCEPYnY5yfPQ0wg+uX7I27cAe4O/ICK0DZCpdU/p4e6j0aPHg0fT3cPBQTzaHdDd/cPB", + "3gM6mu1HD/d39/bXBBV1iIG9eVhr8MUc8GMuvNYn3ns+5NTcFrvQkE2cP/YVF7G8ql1/QQfZau/O+XZT", + "96uu9Z2HEAwJSqg2Vn3RwslO8JJnEbSN5XycdZEnLqzoPHwz2v1Y7Q8OruWOeKNyYc2qWKA0IaSVAVc3", + "qz7Om7F8HJAPfNm0WtXOuy/a6OjB46MHH7toPnhj0xib5HSHm9vmEeaR0xvRIT5IuKJH8grKnpOJrFbf", + "Fl/2+r1cVP9GYaDhS1187hTE1XZg+2E2su5aaYE8OK69V9BTxSJnxkcgqfj3COZ8KYA1QAR6CrowUtHF", + "WSBBtMMdV94u0AyaxZyKzgYu22AMjIzk2sXzcQGMGO2P0IjDRzgiL7AsfqKpfda5QdjE3VXTG42X1l8G", + "zpfv2j6y1g/5zL2vsA48tgj8C6cNy+BUtuubsNLZEflZYp3itSdkU/dri+Mza7V4oyzZciD7Hu8GO3Oi", + "5hH5oRAvCwHVCaRbmrk/J45hlTBT2zWwD7fj4EhY2bkKcEW/Z1e01+/5hYI/A1AXb0uqXzl/VVIMOZIx", + "muBZLsP0c8MTB66PM+Ha8Ei74BHY3Daxx8VmsnhiH09tPqk29NWWIUUlL1W9OyFbiJ/6F+IU2/Cv7cJ/", + "tXbX7T0+eHz4cO/xYSeUtHKAm0Xjp4hMsDq4jXJylOUTpxtpmzok0cM1j6xWofB9AWtXSeaZksB6YOa+", + "wWrnj4ePq+BwscyBDxdDckiSFosaNyyIf1jwohY/yN94cslnM/Hb79HF3t8VT3ffH+q96W4L6LXtKKx2", + "O646F6zoqNl0YFN7hvG7kKCUboW4e800zoCcMUOQfgaERvi8KeKpHcnlurbiQcI62N/ff/TwwV4nunKj", + "qxycCSrhApeyG0FZkmBJsvX67IzsVAjOtumRYLIAzEHjnBGXoH1UF0iHu6P9EJW0XNwl1bi2L9PWJX/n", + "no9uUm7RMXi7eFqunPLgau/vjx4ePHj0oNsxdurhiXq/nsO4crY/nz6juvNbKE2+eXJKoHU1oxHksn9i", + "9cGj1oyKuWbk5ydvLLSTkCRjalDeiL6lepir8zW71vzMteaHSWRs8ocNU6x5qjw8hOit3W6ojyH3FYdn", + "Wjv6dS4YOL4B8grsa2ApVpl4v+3eCQlmllRfsyihPH0S+WCNxj1mkzxMlC1WbkKXK8a95VeuwA51O6nJ", + "CuWXDfXBOkQqkositdBws3H3k9ho2/m/vWg23w+hAB8Bq+fgyWy6xhssZabYJZe5/gQNSWOjb2eJlOpa", + "dduePq+ZBgueB8V6d/KNJo7WiDYsq7/GHDWuAXG74eSudZ5rJBIm8rbF6rQbXbZ+3YT7Lae2vw6ao8YN", + "WqETY+BcudjsRvqUJlGOybRosZ8wK7woEFwgy5KljRJIEikFiRZUoLlFudR9Yk4oARSGYdCnFr5MZkH/", + "DHlFEmlB3yHZj8szZQcB1UAYAs3SVjWboyWlRvrnB6llMi6TUJ0aH6ThTLVhdKYiqB7WkxpZwSO3VWo6", + "1UTONT4vDcY/DJtpMDKqbFgDFTZv2mVqX6UB1+3AEBvMPHSj4kc4mPap7IQXDFm3K0kjJXUFWOzdiR52", + "jZ4r4qE3u4bWB9uBdK1pNHCV4RWnO6dXDN2Pgcigj7khkYYx+nCN06VXc6ZU5Jh5qkLITqU/7OxYCXrt", + "SQGYdc3BajPBdDKgCixsYEX8fqFZ8mWC96JnbTdZLufBfKPaK1QVbqptgO08Nbii4dXqFzQYIuNVhLq1", + "oHglVGATFus6YJVl/hGusVVewSAkWxg9U2FLFSzJ7S7uNuHHL/Sz8u51mYjB4tEVo3E9JOMpNYtjMZOr", + "nOA6Nliv03Z+rxlTHiYxZoKz2L9KC2OsU5px7fLQxTlzK4fd1oD00FmC4svJadsgWKjO65sddlE02zGs", + "zzaD/bqCHXaS63Bo7huV41ohAQHxlEG6ndxJuZ6EDWOrDSs2zxOqEDqw45D1Mk24uOjSul6mU5nwiECF", + "poV9JiGEbAKf9Hc4l+1Os4MKk7bUZGd2cC7C0G5Io99yCt/BLLcb8c2o09mx9Xegfic/tKDP+Q88YQ7l", + "8q3g7yuEXk/HcLA3aou7b2m0FnG/CvN6Xc7tSDZ44nMdiFJcK+X4pGosdjkysDQmh8Oj1uFW8iDM3ph4", + "M9tQPQTl40BOnlp+3YA4IVOGEUR+aivNdWGLXaYSTC0D2fb+Lqd11WpXB+JAwsItUYJtKDYLRgrgjq5V", + "bdsSK2tS2d3roFkgW4WJYqVrgkRsSu1Yemq18ZPXK1kOF8wtGfdztBkPO2Tw8Z4kBRCC67U7IkIzH2XA", + "8xnxbrRZYmZnTKe1rCRZhTRlSiECPUg4UvjZIICLVHatPYIWxHUviWIp5WIsuCjUrQjLxohgl0xV4m2l", + "gkfWHII9/lp54sHvLM3M0iWDQDX8N5qAMc6PcSyqg4TGQY0+JE+E1SyqPDO1dLEsdq8+IBSfJF8xozB/", + "KjcLMlNML6pzD+XMBRnvSqq4NRnZkvgiMA2K1lpi5AUTVVZWNBN8GtqGJrbWqj+gTXiNX937k9RyUJNm", + "jun1/eWaqbCQWEypKNLJCaZyVCqPEzyQMACXW9T9ZVl8gafSAT2lbP6/fZPlT6dF4/XfGsUqCCkeM/yJ", + "VdsGVbBRlq+uTsMmt9HpBQPq1uHirFolyJZ3xvYJmvR2S6KkTvdkt5i+ZtiBH82OZlG994NHDx4edsxU", + "9VFmP4sD9qmNfJfpGuNey06ddLEgPXrw6PHj/YMHj/euZWHxESot+9MWpVLdH4ARMXBYk3/945/vTurU", + "svcAvblH1xpUnrUP6W3WYUDvTv71j3/6Ud14QCFGswr33+IB0Orvk1R30rsc1I2B3Yxka973T2pKAlqw", + "GbLFENQb3Hfsug3KwTQARbpJwTSjETfLACOnVz7BhSvSQO3u0HpjsCGR17bt8FWBc+l8WpQAHxZX4D+t", + "lblBC486J7zT+bTNov2q2SuWI6XVouot0cFZwVJEWMFdzOeK6ppreMys3aFfxKqtxu3YEuvxe5tBFTAU", + "hOSttRWFshY05ElXqbr9je2s2C1rap3miv+65hy2H8FraX0DN3JA6RtledeGSrkdLsCb1ZpMq6ko1+b6", + "rOWtLG/d6/fbIQ55NelJcYNdv79K6OV1KjZozdKjG4Nb8rLtfo0kWqipElUTUKDJhA0qaRCwNNG5tQjC", + "mXeA9YFY0OhCzmZ11NwH7SjrCCCEYWO+F2oMvEz6oPV3OosmRLdFCxr3Huhxj0gFIZnpuNcwWwUDMVP6", + "fuI6qKPEjNbBnpeB9I1Baj+DaSKjC5tlEZP3D8mIpIwKTXKBh79hVdsdrbcO9XtZZW/8OuLjbmUp3WaS", + "KVvQS46ZWZxNZV5z6WTvudHoeortHJFYWtyoWoppN0MoZsMkj8pJ46VDxdI1DA1COSm8b2xZFhV8M0xs", + "LX5nSvYd7AFw7FevTvrWgQGdGO3Aap6SfqJ2BL1+r+yikaih/D3siQzGChx3E/g/XV3HanQ7WlYV08xo", + "hwRekkODCCB8SZhmRoC020OuHqC2eiXlAt0GnfsHLKjv3RIIiVmEJ1KvnsU6od+AuBsRCG6lQyEI+yES", + "xkOxJpPPa2cQbg7AKhsq2eFtO1UPcWslnGgjs179VE/Y+4ixuAkdGi7S1eve1Qx63b+kDm2oSNzuSqPn", + "9OrshrcXKoZjbVvtanSAkGIAp67YUns4LXKdR72pE1p19avgGJMQUGuoQIfxC/Z+/Vr/zN4bRFqP84TF", + "q2tbHEnLqtxltGnFbxwj2XagpWIb03PeQtpK67l+o8SVtiq7j9yVrtSt5Ktc2Z0zZnzZM0dGrTtUTxlT", + "M2m5JogvUvexsaTUJ+6CJ7vpdoMEDxZhrYiD9+0Y7Qn6FjC+z/j7NcRjC9iHcR0gpTxInhqIRSrdSul7", + "cvAQk4jpxtgFny9Msqw74BwEUJk+KqmrYoYJryjssvPlbvqKq95ubjurrYeE47MKyNBKchQnkk7WIXA/", + "LYp57XxGl6jFaTUSPtw/GI3290Y3guD2w7rGcj0tq7gUqPV22oLzKvUIq1rMLAeqtlCEa6/m1baJ78rV", + "JNooRtMjDOHJaMRIwmYIt1eExm62LDa7Xj94J1C5eNyC/v1GuX3zNvh68p2iKzGv/MuqPnCudTSj6vcN", + "BtEWNhOtgPMFovf2B6PDN7v7Rw8Oj3Z3bwM2u1iktriRh7/vXj1M9ujsIHm0fPjb7uLhfC/dD77DLriI", + "u9LqT1C21cumvCTrqEg1lka23BwyppoJ0+taI0BzTrhgA13EWm0OeFzDC6z9feP5v56e385grexwVp9k", + "VYSghuggZd0PkpebzFrbRXM2x8/Wz+JGsUzNgYTprTkUKNVxMJjrYrf3kRgPueh4Db2tFOx8Ea2Nr9t0", + "FYUs7HjSg7vcsuIh8m5APFRmve4CX73kArrTuVTcLNL1t0VRrAAkR7/p37WJ68hRQ3I8F1Kx+s+Fm1z1", + "EQWVe/1e8vtB/cy437tjiDks44IA3VZXpYIObmQJu2TJ+lXAIuXDQ1lPdqoYLsR3u4Pdx+iHkPx+8N1o", + "8LjucdC3q1Vdvl1fuvbrqMsaVpMJ+iRUu4+v5XHt13MdBf3EQ6nwynvZoRw7Gi9z0/urw4fu1ja4/Lyy", + "xw1MoFYB9GMlPXe5TapCUwwa2RDKfUVRqxuvxyqRkSmbc6G76G33R4Xi9gEobskTBzVuE8LLop9q80Ar", + "VTrhacpiTg2OSpk1EQx7HXVxzbfE9bKc+FoBaW0YFtcebwZb2BRwtemaHH5EZO9HvX67vXjX4YCgXs0/", + "URGNDAv2CZ+BDrrevk97bR8dGBiJ/mpHHvKtJFnHA3QpB3o9SZ/MpSFlMH5HfVsu2vWCxfghR2e+Fn3D", + "EsTeJ4FWKaDI+Dr2dfyMZErGeVTGjyY46BI7ROVi2FXI3zi0W9VvYIg3KDk26zfaFBrd9JNt+93QTQLB", + "tm/17mjzVt+KUqTfy7N4Mw+zhbpxsGthwG8IQQyoaOrL3pAEK5P5tQNHf11dwdU3r9UlRyAS5Zk3sABN", + "rVJSwNyCJoaQX+8zljDDQo0QmcRllATXJRfdzFJ3Dx8t2kycaJFaHchPjGWEGoukhP2lYM0LDcwnMC3u", + "kq2Rxw3X1uA1sImP3GrVB/dwoyTWulVVFW5bMgbL5Rs6b4u8clFRf/vZ1ESzVSwVz/BrQtrr9mQCrqZ3", + "e2vVH9+GWu4+hbRXzvTQAID18MQFzrrvv/QFBrGuTrwHIfN8iCze0Dlm3dtgR2kllVCGSjRyUGKoS5OF", + "qTrhnwTzkWoP/eIFBKzuI3pd2kLCDaFz2sxB0Zbg0lqiN/sQtaeMeUPna9F9m+GwVfX7k8HfrLqdTIZH", + "O9/95f8e/PqfQbV7Q4GgmRrEbIYeVxdsObD5nEBZMaxj22IyiZ42dO7ODKMpKtMQN95xpep4H4wK7rn8", + "maYrU0BXtUoypt2NE/rLf7Q7elWW8S1eGBvP7kfnWrmNnLRG+nt5K2Vq7p3qfUTd9nAsMEjvgi01qaR4", + "my5rJ/YbXVSpuOKTc0vuQyYuz8mUY85MPRbwvKdRxMCePXRZgziORUlkw4rRpNqOSzXnz4uzzFrHCgif", + "XgFGfvX2zfev3v78bPLq9PnPT44nPz3/n95RT7Crge0hHgDtHTw4dOneqyu5G0w5cv3MGUNy4uIVnM/D", + "LMeXPUKfgY0UQtCSJWHvoyTX4DaBS29McvMcGatRyzfPOfGRoMrGJCH3DAf+DYDeMGZ7rzrvIq49MXKN", + "efKdlocLsiq6WMIZ9/BKMdJLEsEMJbAV4dUuN7a+6IeLbneJhZsNHHYM/YXDFtCivuQagTu8E0ylMNnC", + "EBqfzNdHEG9fD/72SdFg0AXzE+dsGj3+FHlV365NpHopkwG881qSTwTV6nYtgiEE2JQNzei1WV/m08Bj", + "xum453xOAwaXkGHlk+Q/9QPaGDq2sv+tieDCAR3Pmpk57LG0S9XIJNHQlmgzaI/3SEG8b8EyRrBmG8TL", + "K06GteZ3UmF2XJ7iEHBILBEffl3YdnnKPODkACttjkZe+8CszKwykva9OZF5aFvWLRDE4wOPVKyyEViB", + "xTdcMheg1AGOBumtjk7mKqN4jnZ3TbYKFZhfgiLsetUusD6jxQl9X/QAJeA10TTE4jzKjFq7L77H7AOv", + "3S4B+3RN4DAar9ww1n6ditatiaeq1c2oUtXqvG354MFzvGoN92s7Ww3iLPuokWaIHv9KuflBKnwXt4O/", + "3DpkP17+MVMIq9cE5O+EZs9TFk9kbtaffygG0qO98otMs2WmYq8DoEjEUS2uuY0XeHiScgyrKw3LwaJc", + "cbMEpZF7mUwxHtSnB8aFxI7w57JjTMn64QNqz2eByJkXTDDFI0x4a6EGBb6YALavzHtoU2CuQOCiCPTq", + "6bHTu3gUZXxHc4Ok5x1Qn5wegzGeKat76I2G+8MRHuaMCZrx3lFvf7g7HPXwVbXAKe5MIWG/CyR3L+ri", + "BX8cO0noe18IaiqaMoM1fglAIqADpiuOTxA6rzwiM8qVe0VmiYwLguFQGzM4+Av1yN7KfbvsnZXHGG+N", + "YUAse+U291cUlPHs4DT3RiOHFW/c9YtBTDZyYufvzo227LeTVOeWKJDQYOWZ52XLYuk/9HsHo91rjWnd", + "UPDshjp+K6iLZmb4On8wGt1+p8fCxie6aHPnF1Y9cUhI1bP2y6+wZzpPU6qWfsGqq5VJ3SYYM9BrubL2", + "HWc0iYBVYNolALRh9juoRKl14Va5wGzVviJQaP0U2Lb9JhdoTd/LePnJlrDWh9dRfKizMzguH1bo+dPR", + "TkHGqxvpPnnQcku1d0BA39Mi1fq9nZSD0ePb7xR0vwmPIFrVE7BzzOYafZ8ShGD3IExSkd9yaSgp4hq+", + "oCPtZNZpQW798ira+YPHH+zxTljIHnDKVEqFjRKxZTYc+pXjbG0z5XFee6t5wj9+1nM3lUcjshcVCnL1", + "I1q9tprC4Op1dNA7auvTTi++R8I/uIMT7iZbZNu9zyOHOU7BrvwlHSdna5yWQkhQlnvBzOdC86O7vLJc", + "XoY/4Sn6Ugj4BSskvHK3Vi6FnUzlwj6AgxLg6zJy09X7pi78vSm/VNyF0K4BTWOGEGMNzDReDolfU/vo", + "B6WvthD96tK6m9XP1ykM73M5YXt3ccJwxoWl6Os19fWaWnfKLbX4KeDBrJzyDjqIa2kg/nz6h2trH77q", + "HrrrHjppHgS7sqUBD3JInGtuJGNG9ALynYKx2QI/eSccQ9Vw/juhKlrwS4bofpj3Lk8Mz6hCF5uUgJXA", + "2tBbFRNr1RJFczvQ3MA7ZJYL3AT00GyCLjCTNiDO0hWTCwHKWqq910yJq7iSod2e/aCCvWiwvBrJ1UJq", + "VgAbClO5zTHOW9vXMTY7HIs3DvEWFhC9yj2v0SxB3N41+h8pCB0LV+Fbz0K8R5ymacm5qELwRG4hOu22", + "rEwVRzrRkQyBDr1hggoz0BmL+IxHbloXbOkcW4MNdkplBQP244S8MN5Ra287DFyHrkthlOJnxTfiKKlu", + "vxHSuCTfpZHLYylRBbkPghlK5omc0mRi1+eCBWyCL7CEW5TS4FJak4SMmc3Kny3NQgr7dz7Nhcnt31Ml", + "rzRTYHMaC4xIcWsN3gqFgEiuMDdemkllrJ8D9rljh7jzxwVbfhiOxZM45cLW0LYKTbQEMwXUM9KDh1ju", + "1UIP9jSF7eBPc21kWoWA9XRnhylzk+XGhdZoZvoh+NOxMJL84UEuP+z8Ufb4AY3FjMZAJ5UidkooW7eN", + "Wk8ozH6CRQPmdoYLMO7BRWrdPOaKCoNjKlE6yby6pVtFmghMQttc4YgKksnMpthAolpQILlaGwhaQZOE", + "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", + "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", + "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", + "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1LcheYUnhSgGK4gX", + "wVjYm8k/4WAcERXAvFQuYLFLzBnUt3KzgXaqKxRgyNZ9yhX2dInOsLX0K1xLIPB4SJ6/B8mHG8IwP6eL", + "fdNjkQvFoHuYGnn3N0xGBozBpgoG9FamkKURqgyf0cgm1kQD4beEw7+YBjY9FkzMJOxCLFPKbWq5q4Rr", + "o4cEl9iP06FD5Zo1NqsGAuLHDj/hmIPpMApKaQX7RLJsXllckJPvKxfX6OBR+O5y9LDa+n+dvfqZoAQM", + "/M4WK10j7eoLoBjwWMM8PU5+eA6QZNYojLlQxz0ej3uF60S8jWOFxUFKGQzQfv8dDO07202fx98Nh9CU", + "dQ04Ir/8YVs5gnsrSy348Lj3oU8qH+bcLPJp8e3XMAG2YSae1S5dsmVFym3PSGCGFenaiqNwDKSTuIAr", + "kfK2rzqJTbmgarkuejmw9G4F5cx6DVcW448xegmPe0dj7yc87vXHPSYu8TfnTDzufQivQIMbtSjOC0eC", + "gogOR6PtzfD7bn0D/gEdjHCfWN/SqoEosgbDDjrs5z+XLe7fWtfzfaEdwSuqvJvs1eoNXff9hobe72Dz", + "f5YGxPOEpQy9twbV/IdwU2DezoEVYOA6vokVsvJsr6qjAnbIxtubiogl/u29Wdt79xZEoKKIgfR6tyfn", + "vs5NYSMv8pZ8UUZx3KzyfK+14d0zxY3u6rar2e7uh36/OCNawITmDGjs0sc7hLM2ISKX06cRWxgemWc4", + "psEZE4Y8x1+H7r9eAYQIs+eJnJ8fWf0dJCAhCRcuKKgSrYAPMLuWWMmCchX17D+LlJlbVsT51z/+iYPi", + "Yv6vf/zTZbr41z/+icd9x4JNYu7+8wWjykwZNedHBEJfBzSB95ibDGbXZpdMLcn+yBn+8BMMSbCoiKyQ", + "OaDTwkve5EpUgqds9krtGnT2QpgPFznTDtQMCvKZS61lHZwDylt/lu1S3umJ7gfAYXEGlQnArehpAJE1", + "ueAY7GXVUb2w3cTOuWY5afpqr3jsbuYvhr03lnoHdoDXZDC4xKFzhx/cpMnW2dnz7SHBZ6ClCkyfho+a", + "shn3vhl+5UmbeZLlKHWGgqtseVNEMzrlCfd2h5bcT/YIOpVKGWRQZF7wTRz5kQKPAX9r5w3dx6Jj8eps", + "B+0shkUmV6zvOIFyeMtlckjpgt2wB6cVQhfRgSs7FjNGoQ1y/MwygUpKgiI6umhYIKwROrpzU8tD2R8L", + "i6vttEpw8ECSTbAS9j+nhl3RZZ9UJF+bKyqhBl7qug+Fx8LGG7s1GCBwE6kMc4j8zA5p4N35XeCmYjNQ", + "9OBEbOo92/cWKDxcmEO/DC733dmQczssIp04DvOb4xNVWr0ktvTqzO/Gdp9oSaKEIzVEVIzFHL0BPZS5", + "FLVdLaJKF1TFg0jCJVCFtoMwhITF8zYe+7TSVu8WJZlaP4Hj9GOTXL804WKxOgE4xBauc735/pkr081+", + "71r8Mxnw7QJdx4JvzTjM8hu7ul+t+R2s+eF185b9kHn9mcejvT23f9vFPXn9e9pbXXP7pbJk96FqJFse", + "6AtNo1KR06fHhMaxYlpv/3srImGmlkpL+Q/uR2DF9+F/5sYild0Lr2+pE8iXwg5eu1ET6ufVzDZevd92", + "aqnIWm+6IitZeeXd/u3R6PQ610gp9Ja09vUm2ehxz3UE8ctVahmgaJSwQnwpzmmVijZpla0vf3HlrBWX", + "bClwlHIH8u70y67rXDTvhjtgis8aDPEeGWEdb4FQUcAyfEnU/LbYRTevdernz4s0R3cnBd21KjpE5l/S", + "czFuLBtwQYt21HqBvmDGYhzd5jvd9RCYOOi53am2A13aWRfTslWtA46dEGpi1r99j22Rbk9f296f6eWL", + "y3MdicUt+VcRpcNjt1yrdQ/cY5cQ//betzU4xzv2p3EEFlhk+OB004iNBjlmqV6KaPurS80np2hc7Moj", + "Vvl5s7jQZOP38p11V3LdE+E16jn4g+viXcsFmSUAFW2NADGfoROhqUAfWka0dwejfO38tomihjnfyS8x", + "+P80T6x1I5LikqkKXGv1St35Az3XNz+VPPNae7u+ff1ywEQk48J40i6Tui+f+MFk6b8W0H/3p+4LDGrn", + "XjxoExg/Yv9tRAkp8Hv/194PDsH3f+39QJOMC/a/9p8k1DBttm+NWEZ3ddPd9QPmCyY+eL/w+qKtsKYd", + "Q+fVYPQ6ZXoM6+tQZg172iu7i2i6T0Z9n17IawJ2d5Lw7ozu4SV117BFxz43id1SqTyEXQOX/N9b6Dur", + "EfSd6+tq3XMMhbHQDl8gVpJUHnqZOux67onb8SaBoTiblBFFqY76CF/+T6WSsJO+llKiWNeveokueonq", + "cq1VTdiCt6ucsH3ck/W9ILbQauOnr5h7d2DQcRRZwdyrWbhL1L2F1AY/fR4ALP9OkTi8OArV+6yjybTk", + "FGvvNX+mjp/1cYf7sKfHz0rghTsyoPpx3LlGwPV799LYk3TK57nMdRXTIaUmWjDt8E4SVr8ZvjRdRSk3", + "tGorPmMqHd3lnXbnyoivdH9LapLmhlrmbb0gNj1GfKmujxFXHjE3EAveYcK4zG19n9VzuyU6xaPBdyXj", + "Gmj+h36XcYUeSZAJR5PyHUPwaXM0Fv8bHka/GEbTX7/zAfH5aLR3iL8zcfnrdz4mXpx4UmHCKO6SMD/5", + "+Rm6nswxugDzNJdQN81xWHwRJD2fkOnf7uVWet90f7p5Kvz6dOv0dKss1/qnmy14y2+3elK3O3+8eXoL", + "Lbj99id9vv3Jbcq1p6XOZzMecSYwNR5qJfWKE7V9Yn41J99MP0qFc+KoeGDWJJHOz8iCa22Q0It865/U", + "xbHfmiFHEmoMSzMDEGsRg0zdNnRQL3IDwE8+YQ5O0Ode5OV8Qte7b2riGwlH7tYS+6wL3A2+dIuksXf9", + "1HUdf6Ghs9IGy8f+cVmKNu2vy/sl3tt9U3a4au/+Vfklk5h9vq0uXQYvhEACSGt/SnMbZ1zULLFjh+TN", + "m5c+ptgi87l0okb6HKI+z/xYVHOIIgyeT85qC9Sx/RwGAdT3WTljRuOEC4ZBGEyHwn/rmX/v9Vh8egk4", + "nNb4rm3vXY6l3dB7lIDvjRXciaxZOBsYZykujmYl43FxWry8iafmi+JXjgEFGE9I1tuhuZEDh1Kws5AW", + "vzYM4X2a0AgRvKGYBZd1wDAWTbraFKK9KJkkTFnQ4Cw3XtwC/G43OC6MLNLiO8nsHJqf5MLw5LzvoUgj", + "lmhCxdKh+Y1FrTMn8yF4AwKT4AgVy+yIGzm+YdAcVI1QCnEWql0SmlzRpR4LB/dgq6cSo6Iji6+dJEPy", + "o0SkHZvrvsJ4baLpb/RYnPM4YRMHlHNOuCZ6IZVhgsUklZdM1/tlVCWcKZzEUworp0lKlyXOqFsfmTEL", + "V1mD45HwbypijimLoediykdjQcneaERSRoV24BqazvDCcW0QHERtQN8SSg5Gj12txr5hagW//FtwmpRi", + "lzKiU8hODVQMQ+RmGzcwdSnEFdNGKti+GVfa7leh33S5YWsby7XPiB33SS5K+BDU9Vesa7BdJlcC5+nM", + "k4yr4hp0iElTFlFYTyHr/SBgtYyiXIUuSNjqSi77f0fBsTK9M1yqMDhHgiqDCGIAJCI/LvBMSzxK29+2", + "UFVJVH+OiyZ4SKQilFToutRosChH1riFoLPnZWJmMXHlz7e/9WcHjq9jBP74i/mXdD8hEcnZrHYAN19N", + "9gCvC4pbJeE/6zl96phejcXFnM6F1IZHnhl6/Pzqo/nrg7DTg3D9ygapeSbVRbsn9A9SXXR9gXmPzS/r", + "IVad4WdoiIDhoZPO/dsjUBtuHytANHf+SGvSV3FKUejiRhe+ygQyGDBV1crfudmg+qrbskibcJkqa+wu", + "cNHgETJxP9qk/hV/czQxRK7VP62zGhIbbHbpszZd2qdHiSR3PV4Jp6oK+GDfgtr6H/S9OIR05TdsC6X3", + "1e0KMlUAGt2I1Fp07mFJA1CtY/FW20Qq59b0dE4KHgwCrc3x4PKKAGwr/IbtW1RXmmXnBZT+9hF5gQe5", + "sm228y2bEgloTcuEWTTWyzQ9P1pNa//u5AQrYZmFTWB/fkR8Kvvi/tBQqgrDCrNIqDbkZwcuu1U8xnFH", + "z0GHWZnftgNoLVMdjEUIrBWwTm2DfEbOK7it5y2ggp7fvpRz/dmYispkUHYuRhL3dETaZCLutTl58CRs", + "+NkdjUJ5EzrCx9ph3DJ67MpgXsp5kYiqRso0y7qSrxsmUvFlmq6hYbJVchCiTSxz8xdtYqYUVnbU3Ubc", + "ZItG9h+GXgChOuRRf7C3x6JlqewMw0sFTLWSr8X+6zJNe/2eGw/UuzJZGsjc8vFwvM0GP/RDO1TB3P0T", + "i+f3diW6pGXFUfHJgqzE415QAOWK5wW39CbQv/UbqoL927junA6g/Rnx2hb405s4vaLxzsR396JpiE9B", + "+ri3U4xiVkMX/jlJ8pXx1vVmQhaoyjiyx3cxMoReJlEiNavZyb4c4EanL2yI3u36Nr/GAxhenPt0l10c", + "gc5c1TNf8zNQYWxyufFjJn66d+57szqCLxmEQq/MBthfA+1vk1POZ09In25LVqbahUK+0ub1lbWdCBOe", + "W6sswlWIXSrR3MiUGh5hOsBoIaWukH0BzW+T5DodfEGZqKGyygIXiHEOpHrutPnnTsQ8cppHQqufXB9D", + "rO7CN8I1/Keyxg8V5UrB8fv+BYWZaTSkTFOczUhGc81AzsxBI7aMEubzPzKwq0c0M7limEaakZQLnuZp", + "NecC7NglRXyo8930vE+muSEJaNlU8dH7LEUyTZmIrZw0FgtGLzlTsCgJNUxEy4FmQnN0jYK0D5izEzQ1", + "WUzRYIaBlIoBBWICi5QZGlNDUdA5hxM/sbFg5yRTDInIakcEe19SA9iOc/GtzaYDzZ77gZ4ThukiuF4U", + "mcsjGjMRBdMonH3ebOzTq/TPmGlO9J4crG7ES+/T46qquvbD+Tycsb5YyI8ObH6N0KvbH9X1IBpPRv+e", + "R9rO1c/xnux0xRKvO8Wfh4GuILrPxkh3/1Y4qUic2+4qpxLJ/M+qRywYSg0RAgJ07Tbe1L5WZGctlvla", + "PG/nD//n8Q20i58JJ+y3Puzb8gCWk/4cWK5b1Rvx3HtSqzpdUlUnd38s2A3q/sQnqSpc7ktRttaQ9wq+", + "XeVORlF8fUnxlW032bbzG7kp2/a62RXPiAoj52KArrZhDu7UuK2s2qkO/k2Dehqzq7DMe2eRpeXizmE0", + "PWvM6BJUJX8GX+s19qNIKmVRRKhhXxb0d0VrWI2yQN1cmWC074Ne352cbLdxCWXW8ghlvmAOUYlsgmpp", + "vKrAfXXJlOKxhzd9evKstMmqXAzJq5QbYiS5YCwrA4MwOBNQOws8lcawm8Ap/R4TRi0zyYXZOIqy6O0M", + "pvxBOliZz1CUdLkkvprDO5vDUbP/5bEz5DKEFhNY/zI11GzMcc3FTKrUymV0KnNoHXgQLBPsp3W2mPGE", + "6aU2LLXOnQAFAccN8xK53POunt3lPro2w8mxUYcZUynXmkuhx8KF3GRMQd9QHdqv+KkFDQKGFvz11DLJ", + "z8MHEgZj3f6oaVs1RL7CnNa9o94OzbKdmBra4mfnhvcRQ/oBnRqJXqZTCKAEr8gLTbYSfmGfJ+RSkwT+", + "2F7rFTnBep86s/7NTxas9LGYyWDeUkuzBTH/qYLjHFvzhsnPgq3d0zvzo70IX7DqwfW8cCZbWazeyGMV", + "o8kA8/J7LCaSG57w3y3bhUa4NjyyUWS02Md3JwWDH47FCTOKR9beG4H/OYJU4Et3J1My2gEcv/0o4wjo", + "t89wcMh82z+n2OPT07dYLmWpVMv+WFDhGn7z5NRammc0Ys2BCmbAkkyOd15t8Fk/w2X6N/aftBNcC0gR", + "3PCv5snrw8a0niHdckRltu7ZJrM/vYOvkya/6ji+TB0H0HA5m60Cq80DrIX1GZcyyVP4h/3jeBNUnaHR", + "4h0W/Wwkbzucjd34CX4Rh9LNKWY2x/O9GGDsgn2p/rOwcH4KKMTUPBODt8AT82ek7k9vSqiu42doenUr", + "6vOnfzZn665vPjcGD5pSXY8v5ZhbSvMzMXK9JuyK8nZN2PeJjC60Q9epqjCJYhQh8+HHEuLcmStRTMBg", + "X+JQqSz2GdP9sWgoQy2IkyaUGKZSLmiyg3O2jSBYu9eo0UvJMeY+wpiZgQZVdyaTBBHZEdEQZgMlZr6B", + "inVZuwyT1TJVw6iRANskU+YB7LdDT7e/Um5+kKqORv+58MU3lfVHlEfKUfe/AYC/vcePAuQ/oe/RbTvO", + "nXHbj2jrhSx/tGqpPsG9Gff2R3rc65Nxby8d92AHnlJU51JDHpCUi9wAsuUzq2vDqOrDEdEskiLWHkff", + "axP3R7pl5o4sWwJ2D7HeXYo9jqpwKV+7TkLsAcoRqI8BRGSreuDcmYz7eOhiInNjTQ/uXLlSMTOoHtm+", + "c7tx5Yx8fdt34eR/dce3xqNwl4FdVrbecvYs1wvWrnJ7aZNm5WYK61Ak2dYL8nc51X2IOrCaeUDMW+F7", + "UPvUdnAXuSOgq+vkjXBz/5o0okPSiHKtwvib1tkTrmRPHRaEk73PpDLwu8MtcDSELwkEA8FUmK+eHo9F", + "BKzIokUqlkrkTg7i3t7CT/56Rp4/fd0nzzDhM/kxn24PySuItMFWtbMXjYWVxCzziigAq9sZxKHr2Y4d", + "qec2Hdehg2tJ8XufztKDJyNg5fF75R3W+70FozFKJH/0XkrbWQBI+vVLOECwf65mse29tcIH6PDVcvBk", + "ZphabfbExWyJAgbFXdIeVdAJbtgxdqg9mF7Zp5UNLNrJ/l4vAH7y4Wsej9tPFH43Fjvs16V2hGi+LzVH", + "MB7EgjmGWGD1ui4yYbRFLDtetvaBgV0eP/skj4pbtVKv4121dAH/rqcLZ/rFGpqy2j4BERcZdDZaen2g", + "8sIiXNtqJKIZjbhZ9glNEndHuZug8I4ZFOLvVDF6AXr7IaCQuZ59cDHYavveUEtiri9sC84WOyTgHqfz", + "aTE4ggfNWo1xzSFE10gS0STKE2oYYbMZizAuGFPy6BZbbjGU3i2enbKTICxMxcM+/+LSFoZpAnevJIsm", + "xe3Yrd5RLEooT9vx5J2ghs6P6GowhUalAN+FxFIcjZTUmrimBizhcw5pNdBTAjJpgJqLpmwssoQKwRTJ", + "tfXQh6EPMsW0zhXzDcDd5SiqT0qsyExJ41wTEimVHgtP4e9OiDYsW0Nmr23LJzjnW5JtbeOup3tSUjfG", + "0K4KcUUIbIilFLvgQEegGroHt3g7oPuWEr+Ug/9G8Tn4vmZKUstkrV+SPdZ+Oe2hr0VPt6YwPStKdUth", + "WrRaiZCsRA+uxfqblPDpce96HoiBzi94Kxyk+3S9iOafoFLHvuuRs+FBuE8fOctQNtZ/x8SnZ5WAxa4K", + "rJLCvzR1UmXktaNaC/rdDPHVOcr3NqNuO2N53RuE15eM3EVrobxtD97PjxBGd4s4cdd5875s2qohb9Xe", + "pi3wA5sTJHwWFHg7mRHuGXHlBpkRPisMAKCde8RiCR7U+4rpr9mepfqa3OBWQ/lthgOEhmsL5bdczzmv", + "rn0ovXNluj2TXIt/JgneLtB15He/7F9f/R2eDJXF2mSCBoJnaWaWrpK3VZZOZ5r/zoYthuDCb/X2TME3", + "cOn8dOTh6bTVofMe0cvu0ZBzLz6jLhsk1+DmV2bd84HdXxjeYfXM1S6WHbh1BlRFC37J2pXu9RPslihT", + "bJDJDI0rsV0wtx7+LjNUDee/E9e8w391/8J0opjIgMUk5opFBrxFhJHIEWwf32iipDTuu1TLdi8Re0R+", + "UDJ94maz4T50Z8opw0o/w3Q5iKmhg0vPbdao0D7Cu9P7UwLDI1yQF9+TLfbeKJu0hMzg5QPR5n5J2fuI", + "sVgjTW5XB7w7Cg8RGp7Mp11GuSb9zCv8gyYkyrWRqd/742dkC9PZzZmAvQBRf4aSbKbkJY9ZXBsjOO/b", + "Vd1tWdDr6l1BqChyEfrHhR3cvcgwXS6k+e88q7OFwiVmygXFwW1M8FI/UxZQAPqjXHgHHLdHfhRfrzD3", + "8tvyjx2gREyt6hbRSGnhpre/XnNf8jVXDYbyd1rttvPuOeuV193iozqGLd1GEooidu5u1dbvPp+QHq6/", + "yGgepzq/LB6kbWrzz4sER3d3P9y1uvzdFxwCCsryy8ay2QbUZZhgXqJPd8wuWSKzFFPcY9lev5erpHfU", + "WxiTHe3soO/3QmpzdPD44X7vw68f/v8BAM9LG8e+AQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8de7c33d3..ff2334317 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/netip" "os" + "path/filepath" "strconv" "strings" "time" @@ -425,15 +426,20 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) - if err != nil { - return nil, err + registryURL := cfg.Registry.URL + if !cfg.MacOSBuilds { + var err error + registryURL, err = builderRegistryURL(registryURL, networkManager) + if err != nil { + return nil, err + } } if registryURL != cfg.Registry.URL { log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ + MachineBuildOnly: cfg.MacOSOnly, MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, @@ -444,6 +450,17 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DockerSocket: cfg.Build.DockerSocket, } + if cfg.MacOSBuilds { + workDir := filepath.Join(p.BuildsDir(), "machine-work") + if err := os.MkdirAll(workDir, 0700); err != nil { + return nil, err + } + buildConfig.MachineBuild = &builds.MachineBuildBackend{ + Driver: &builds.GuestMachineBuildDriver{Instances: instanceManager, WorkDir: workDir}, + Publisher: &builds.StreamingMachinePublisher{RegistryURL: registryURL, Tokens: builds.NewRegistryTokenGenerator(cfg.JwtSecret)}, + } + } + // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider if cfg.Build.SecretsDir != "" { diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 77249b920..3c3857673 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -111,6 +111,12 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodPost && req.URL.Query().Get("mount") != "" { + if matches := blobUploadPattern.FindStringSubmatch(req.URL.Path); matches != nil && r.mountBlob(w, req, matches[1]) { + return + } + } + // Intercept manifest PUT requests to store in blob store and trigger conversion if req.Method == http.MethodPut { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) @@ -161,6 +167,27 @@ func (r *Registry) Handler() http.Handler { }) } +// blobUploadPattern matches POST /v2/{name}/blobs/uploads/. +var blobUploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/?$`) + +// mountBlob answers a cross-repository mount when the blob is already stored, so +// the client does not upload it again. The middleware has already checked that the +// token may pull from the source repository. It returns false, leaving the request +// to the upload handler, when the blob is not stored. +func (r *Registry) mountBlob(w http.ResponseWriter, req *http.Request, repo string) bool { + hash, err := v1.NewHash(req.URL.Query().Get("mount")) + if err != nil { + return false + } + if _, err := r.blobStore.Stat(req.Context(), repo, hash); err != nil { + return false + } + w.Header().Set("Location", "/v2/"+repo+"/blobs/"+hash.String()) + w.Header().Set("Docker-Content-Digest", hash.String()) + w.WriteHeader(http.StatusCreated) + return true +} + // storeManifestBlob stores a manifest in the blob store by its digest. func (r *Registry) storeManifestBlob(digest string, data []byte) error { digestHex := strings.TrimPrefix(digest, "sha256:") diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go new file mode 100644 index 000000000..a9acd4275 --- /dev/null +++ b/lib/registrypush/macos_test.go @@ -0,0 +1,48 @@ +package registrypush + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache/testutil" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { + host, _ := newTestRegistry(t) + img, err := random.Image(256, 1) + require.NoError(t, err) + config, err := img.ConfigFile() + require.NoError(t, err) + config.OS, config.Architecture = "darwin", "arm64" + config.Config.Labels = map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", + images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + } + img, err = mutate.ConfigFile(img, config) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + img = mutate.ConfigMediaType(img, types.OCIConfigJSON) + p := paths.New(t.TempDir()) + digest, err := testutil.WriteImage(p, img) + require.NoError(t, err) + target := host + "/releases/macos:canary" + result, err := PushFromCache(context.Background(), p, digest, target, nil, Options{Insecure: true}) + require.NoError(t, err) + require.Equal(t, digest, result.Digest) + ref, err := name.ParseReference(host+"/releases/macos@"+digest, name.Insecure) + require.NoError(t, err) + published, err := remote.Image(ref) + require.NoError(t, err) + actual, err := published.ConfigFile() + require.NoError(t, err) + require.Equal(t, config, actual) +} diff --git a/openapi.yaml b/openapi.yaml index 73aa90cde..177cb24f3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4683,6 +4683,24 @@ paths: dockerfile: type: string description: Dockerfile content. Required if not included in the source tarball. + machine_base_image: + type: string + description: | + Experimental installed darwin/arm64 base pinned by sha256 digest. + Requires server macos_only and macos_builds. The source must contain + build.sh at its root, which runs as root in the guest. Machine source + is limited to 64 MiB compressed; only machine_network_mode, + timeout_seconds, memory_mb, cpus and tags are accepted alongside + source and this selector. CPU/memory zero inherit the base. + Linux/cache/secrets/admin options are rejected. Output retains the + base identity and cannot run concurrently with it. + machine_network_mode: + type: string + enum: [isolated, egress] + description: | + Native machine builds only. Defaults to isolated. Explicit egress enables + unrestricted VZ NAT for checksum-verified artifact downloads; it does not + enforce domain allowlists. Linux builds do not use this selector. base_image_digest: type: string description: Optional pinned base image digest From 87a6db8416e04e4d5a3f70e48f92fa4a7c04f102 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:03:07 +0000 Subject: [PATCH 2/2] Give each macOS build its own machine identity The builder instance is created with a fresh machine identifier and MAC, the same rebind a cold clone gets, and its SSH host keys are rekeyed on first start. The exported bundle carries that identity, so export validation now requires the hardware model and resources to match the base and the identity to differ from it. Builds on one base are no longer serialized, and an output can run beside its base. --- docs/macos-builds.md | 9 +- lib/builds/macos_build.go | 9 +- lib/builds/macos_build_failure_test.go | 4 +- lib/builds/macos_build_guest.go | 5 +- lib/builds/macos_build_manager.go | 13 +- lib/builds/macos_build_manager_test.go | 48 ++++-- lib/builds/macos_build_test.go | 9 +- lib/builds/manager.go | 6 +- lib/instances/create.go | 5 + lib/instances/macos_identity.go | 23 +++ lib/instances/macos_identity_test.go | 17 ++ lib/instances/types.go | 1 + lib/oapi/oapi.go | 213 +++++++++++++------------ openapi.yaml | 5 +- 14 files changed, 219 insertions(+), 148 deletions(-) diff --git a/docs/macos-builds.md b/docs/macos-builds.md index 81f2eb217..c5e488cee 100644 --- a/docs/macos-builds.md +++ b/docs/macos-builds.md @@ -2,8 +2,8 @@ A build job can provision a digest-pinned installed macOS base in an isolated VZ VM, run the source's `build.sh` in it, and publish the result as an OCI machine -image. This is not macOS installation, Linux rootfs conversion, memory forking or -identity rekeying. It depends on the shared system GuestService. +image. This is not macOS installation, Linux rootfs conversion or memory forking. +It depends on the shared system GuestService. The normal build manager owns the queue, persisted request, deadline, source staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` @@ -50,8 +50,9 @@ Networking defaults to isolated. `machine_network_mode=egress` gives unrestricte VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, cache, build-arg, secret and admin options are rejected. -Jobs on the same base digest are serialized. Identity is preserved, so concurrent -guests with the same identity are never allowed. +Each builder gets its own machine identifier and MAC, and its SSH host keys are +rekeyed on first start, the same way a cold clone is. The output keeps that identity. +Builds on one base run concurrently, and an output can run beside its base. ## Source diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go index 6eb8e9ef9..d58a55d27 100644 --- a/lib/builds/macos_build.go +++ b/lib/builds/macos_build.go @@ -282,8 +282,13 @@ func validateMachineExport(root string, base *images.MacOSImage) error { if err != nil { return err } - if base != nil && (!bytes.Equal(platform.HardwareModel, base.HardwareModel) || !bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || !strings.EqualFold(platform.MAC, base.MAC) || platform.CPUs != base.CPUs || platform.Memory != base.Memory) { - return fmt.Errorf("machine export changed preserved identity/resources") + if base != nil { + if !bytes.Equal(platform.HardwareModel, base.HardwareModel) || platform.CPUs != base.CPUs || platform.Memory != base.Memory { + return fmt.Errorf("machine export changed hardware model or resources") + } + if bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || strings.EqualFold(platform.MAC, base.MAC) { + return fmt.Errorf("machine export kept the base identity") + } } for _, entry := range entries { if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go index c0e4e6e0f..632bd7ee7 100644 --- a/lib/builds/macos_build_failure_test.go +++ b/lib/builds/macos_build_failure_test.go @@ -164,7 +164,7 @@ func TestMachineBuildMalformedExport(t *testing.T) { if err != nil { return err } - return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:01", "02:00:00:00:00:01:02:03", 1)), 0600) + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:02", "02:00:00:00:00:02:03:04", 1)), 0600) case "identity", "unknown metadata": var raw map[string]any data, err := os.ReadFile(config) @@ -175,7 +175,7 @@ func TestMachineBuildMalformedExport(t *testing.T) { return err } if kind == "identity" { - raw["machine_identifier"] = []byte{9} + raw["machine_identifier"] = f.image.MacOS.MachineIdentifier // kept the base identity } else { raw["password"] = "must-not-publish" } diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go index ebf341318..3794b92ab 100644 --- a/lib/builds/macos_build_guest.go +++ b/lib/builds/macos_build_guest.go @@ -21,7 +21,8 @@ type machineExportManager interface { // GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system // GuestService. It never installs an agent or silently upgrades an unmanaged base. -// The backend validates the base and policy before Start. +// The backend validates the base and policy before Start. The builder gets its own +// machine identifier and MAC, so it and its output can run beside the base. type GuestMachineBuildDriver struct { Instances instances.Manager WorkDir string @@ -50,7 +51,7 @@ func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base * if err != nil { return nil, err } - inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, FreshMacOSIdentity: true, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) if inst == nil { _ = os.RemoveAll(dir) if err == nil { diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go index dcd5be191..eebd4b9f6 100644 --- a/lib/builds/macos_build_manager.go +++ b/lib/builds/macos_build_manager.go @@ -161,17 +161,10 @@ func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) return policy, nil } -// enqueueMachineBuild queues a machine build whose source is already staged. -func machineBuildSerialKey(req CreateBuildRequest) string { - ref, err := images.ParseNormalizedRef(req.MachineBaseImage) - if err != nil || !ref.IsDigest() { - return "machine:" + req.MachineBaseImage - } - return "machine:" + ref.Digest() -} - +// enqueueMachineBuild queues a machine build whose source is already staged. Each +// builder has its own identity, so builds on one base are not serialized. func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { - queuePos := m.queue.EnqueueSerial(meta.ID, req, machineBuildSerialKey(req), func() { + queuePos := m.queue.EnqueueSerial(meta.ID, req, "", func() { m.runBuild(context.Background(), meta.ID, req, policy) }) build := meta.toBuild() diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go index fdc2cab7d..c30799a76 100644 --- a/lib/builds/macos_build_manager_test.go +++ b/lib/builds/macos_build_manager_test.go @@ -104,10 +104,27 @@ func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { require.Error(t, err, "machine builds must not persist a builder config or registry token") require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) } -func TestMachineBuildSerializesByBaseDigest(t *testing.T) { - first := CreateBuildRequest{MachineBaseImage: "registry-a/base@" + machineTestDigest} - second := CreateBuildRequest{MachineBaseImage: "registry-b/another-name@" + machineTestDigest} - require.Equal(t, machineBuildSerialKey(first), machineBuildSerialKey(second)) +func TestMachineBuildsOnOneBaseRunConcurrently(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + var ids []string + for range 2 { + build, err := m.CreateBuild(context.Background(), req, []byte("source")) + require.NoError(t, err) + ids = append(ids, build.ID) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 2 }, 5*time.Second, 10*time.Millisecond, "builds on one base must not serialize") + for _, id := range ids { + // Cancel only once the attempt has recorded its builder, so it is Building. + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, id) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), id)) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) } func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { @@ -134,16 +151,19 @@ func TestQueuedMachineCancellationRemovesSource(t *testing.T) { fixture.waitForCancel = true m := newMachineTestManager(t, fixture, fixture, nil, false) req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} - first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) - require.NoError(t, err) - t.Cleanup(func() { - _ = m.CancelBuild(context.Background(), first.ID) - require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) - }) - require.Eventually(t, func() bool { - meta, err := readMetadata(m.paths, first.ID) - return err == nil && meta.BuilderInstance != nil - }, 5*time.Second, 10*time.Millisecond) + // Fill both concurrency slots so the next build stays queued. + for range m.config.MaxConcurrentBuilds { + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + } queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) require.NoError(t, err) source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go index da848ce78..1b7f9c5ba 100644 --- a/lib/builds/macos_build_test.go +++ b/lib/builds/macos_build_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "testing" "github.com/kernel/hypeman/lib/images" @@ -18,6 +19,7 @@ import ( const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" type machineFixture struct { + mu sync.Mutex image *images.Image calls []string fail string @@ -35,6 +37,8 @@ func newMachineFixture() *machineFixture { return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} } func (f *machineFixture) record(phase string) error { + f.mu.Lock() + defer f.mu.Unlock() f.calls = append(f.calls, phase) if f.fail == phase { return errors.New("sensitive-command-argument") @@ -84,7 +88,10 @@ func (f *machineFixture) Export(_ context.Context, root string) error { if err := f.record("export"); err != nil { return err } - data, _ := json.Marshal(*f.image.MacOS) + // The builder has its own identity; the export carries it, not the base's. + output := *f.image.MacOS + output.MachineIdentifier, output.MAC = []byte{3}, "02:00:00:00:00:02" + data, _ := json.Marshal(output) for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { return err diff --git a/lib/builds/manager.go b/lib/builds/manager.go index a4327e358..3584484bb 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -1546,11 +1546,7 @@ func (m *manager) RecoverPendingBuilds() { continue } - serialKey := meta.Request.BuilderID - if machineBuild(meta.Request) { - serialKey = machineBuildSerialKey(*meta.Request) - } - m.queue.EnqueueSerial(meta.ID, *meta.Request, serialKey, func() { + m.queue.EnqueueSerial(meta.ID, *meta.Request, meta.Request.BuilderID, func() { policy := DefaultBuildPolicy() if meta.Request.BuildPolicy != nil { policy = *meta.Request.BuildPolicy diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ecdde66..8af4a9147 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -385,6 +385,11 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } + if req.FreshMacOSIdentity { + if err := m.freshMacOSIdentity(stored); err != nil { + return nil, err + } + } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 797468f83..6c9ef6a7a 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -224,6 +224,29 @@ func (m *manager) clearMacOSStopReceipt(stored *StoredMetadata) error { return nil } +// freshMacOSIdentity gives a new macOS instance its own machine identifier and MAC +// instead of its image's, so it can run beside the image's other instances. The +// guest's SSH host keys are rekeyed on first start. +func (m *manager) freshMacOSIdentity(stored *StoredMetadata) error { + if stored.MacOS == nil { + return nil + } + if err := prepareMacOSClone(stored); err != nil { + return err + } + used, err := m.storedMACs(stored.Id) + if err != nil { + return fmt.Errorf("list MACs for macOS identity: %w", err) + } + mac, err := network.NewUniqueMAC(used) + if err != nil { + return fmt.Errorf("allocate macOS MAC: %w", err) + } + stored.MacOS.MAC = mac + stored.MAC = mac + return nil +} + // assignMacOSCloneMAC gives a networked clone a MAC no stored instance uses. It runs // under the fork admission lock, so concurrent forks cannot pick the same MAC. func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go index 2978e7da1..0d3f98506 100644 --- a/lib/instances/macos_identity_test.go +++ b/lib/instances/macos_identity_test.go @@ -4,10 +4,12 @@ package instances import ( "net" + "os" "testing" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) @@ -42,6 +44,21 @@ func TestPrepareMacOSCloneRebindsIdentityWithoutAliasingSource(t *testing.T) { require.False(t, source.IdentityRekeyPending) } +func TestFreshMacOSIdentityRebindsIdentifierAndMAC(t *testing.T) { + p := paths.New(t.TempDir()) + require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) + m := &manager{paths: p} + stored := testMacOSSource() + stored.Id = "builder" + require.NoError(t, m.freshMacOSIdentity(stored)) + require.True(t, stored.IdentityRekeyPending) + require.NotEqual(t, []byte("source-identifier"), stored.MacOS.MachineIdentifier) + require.NotEqual(t, "a6:fd:95:db:a2:62", stored.MacOS.MAC) + require.Equal(t, stored.MacOS.MAC, stored.MAC) + _, err := net.ParseMAC(stored.MAC) + require.NoError(t, err) +} + func TestPrepareMacOSCloneRefusesWithoutGuestAgent(t *testing.T) { for _, disable := range []func(*StoredMetadata){ func(s *StoredMetadata) { s.MacOS.GuestAgent = false }, diff --git a/lib/instances/types.go b/lib/instances/types.go index bd8989885..7c50cb359 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -267,6 +267,7 @@ type CreateInstanceRequest struct { Name string // Required Image string // Required: OCI reference Platform string // Optional: target platform as os/arch[/variant]; drives image resolution and auto-pull. Empty means host platform. + FreshMacOSIdentity bool // Internal: give a macOS instance its own machine identifier and MAC instead of the image's. Size int64 // Base memory in bytes (default: 1GB) HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) OverlaySize int64 // Overlay disk size in bytes (default: 10GB) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index eb421b5b1..b6b825e65 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -2014,8 +2014,9 @@ type CreateBuildMultipartBody struct { // is limited to 64 MiB compressed; only machine_network_mode, // timeout_seconds, memory_mb, cpus and tags are accepted alongside // source and this selector. CPU/memory zero inherit the base. - // Linux/cache/secrets/admin options are rejected. Output retains the - // base identity and cannot run concurrently with it. + // Linux/cache/secrets/admin options are rejected. The builder and its + // output get their own machine identifier and MAC, so the output can + // run beside the base. MachineBaseImage *string `json:"machine_base_image,omitempty"` // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables @@ -19643,110 +19644,110 @@ var swaggerSpec = []string{ "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", - "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1LcheYUnhSgGK4gX", - "wVjYm8k/4WAcERXAvFQuYLFLzBnUt3KzgXaqKxRgyNZ9yhX2dInOsLX0K1xLIPB4SJ6/B8mHG8IwP6eL", - "fdNjkQvFoHuYGnn3N0xGBozBpgoG9FamkKURqgyf0cgm1kQD4beEw7+YBjY9FkzMJOxCLFPKbWq5q4Rr", - "o4cEl9iP06FD5Zo1NqsGAuLHDj/hmIPpMApKaQX7RLJsXllckJPvKxfX6OBR+O5y9LDa+n+dvfqZoAQM", - "/M4WK10j7eoLoBjwWMM8PU5+eA6QZNYojLlQxz0ej3uF60S8jWOFxUFKGQzQfv8dDO07202fx98Nh9CU", - "dQ04Ir/8YVs5gnsrSy348Lj3oU8qH+bcLPJp8e3XMAG2YSae1S5dsmVFym3PSGCGFenaiqNwDKSTuIAr", - "kfK2rzqJTbmgarkuejmw9G4F5cx6DVcW448xegmPe0dj7yc87vXHPSYu8TfnTDzufQivQIMbtSjOC0eC", - "gogOR6PtzfD7bn0D/gEdjHCfWN/SqoEosgbDDjrs5z+XLe7fWtfzfaEdwSuqvJvs1eoNXff9hobe72Dz", - "f5YGxPOEpQy9twbV/IdwU2DezoEVYOA6vokVsvJsr6qjAnbIxtubiogl/u29Wdt79xZEoKKIgfR6tyfn", - "vs5NYSMv8pZ8UUZx3KzyfK+14d0zxY3u6rar2e7uh36/OCNawITmDGjs0sc7hLM2ISKX06cRWxgemWc4", - "psEZE4Y8x1+H7r9eAYQIs+eJnJ8fWf0dJCAhCRcuKKgSrYAPMLuWWMmCchX17D+LlJlbVsT51z/+iYPi", - "Yv6vf/zTZbr41z/+icd9x4JNYu7+8wWjykwZNedHBEJfBzSB95ibDGbXZpdMLcn+yBn+8BMMSbCoiKyQ", - "OaDTwkve5EpUgqds9krtGnT2QpgPFznTDtQMCvKZS61lHZwDylt/lu1S3umJ7gfAYXEGlQnArehpAJE1", - "ueAY7GXVUb2w3cTOuWY5afpqr3jsbuYvhr03lnoHdoDXZDC4xKFzhx/cpMnW2dnz7SHBZ6ClCkyfho+a", - "shn3vhl+5UmbeZLlKHWGgqtseVNEMzrlCfd2h5bcT/YIOpVKGWRQZF7wTRz5kQKPAX9r5w3dx6Jj8eps", - "B+0shkUmV6zvOIFyeMtlckjpgt2wB6cVQhfRgSs7FjNGoQ1y/MwygUpKgiI6umhYIKwROrpzU8tD2R8L", - "i6vttEpw8ECSTbAS9j+nhl3RZZ9UJF+bKyqhBl7qug+Fx8LGG7s1GCBwE6kMc4j8zA5p4N35XeCmYjNQ", - "9OBEbOo92/cWKDxcmEO/DC733dmQczssIp04DvOb4xNVWr0ktvTqzO/Gdp9oSaKEIzVEVIzFHL0BPZS5", - "FLVdLaJKF1TFg0jCJVCFtoMwhITF8zYe+7TSVu8WJZlaP4Hj9GOTXL804WKxOgE4xBauc735/pkr081+", - "71r8Mxnw7QJdx4JvzTjM8hu7ul+t+R2s+eF185b9kHn9mcejvT23f9vFPXn9e9pbXXP7pbJk96FqJFse", - "6AtNo1KR06fHhMaxYlpv/3srImGmlkpL+Q/uR2DF9+F/5sYild0Lr2+pE8iXwg5eu1ET6ufVzDZevd92", - "aqnIWm+6IitZeeXd/u3R6PQ610gp9Ja09vUm2ehxz3UE8ctVahmgaJSwQnwpzmmVijZpla0vf3HlrBWX", - "bClwlHIH8u70y67rXDTvhjtgis8aDPEeGWEdb4FQUcAyfEnU/LbYRTevdernz4s0R3cnBd21KjpE5l/S", - "czFuLBtwQYt21HqBvmDGYhzd5jvd9RCYOOi53am2A13aWRfTslWtA46dEGpi1r99j22Rbk9f296f6eWL", - "y3MdicUt+VcRpcNjt1yrdQ/cY5cQ//betzU4xzv2p3EEFlhk+OB004iNBjlmqV6KaPurS80np2hc7Moj", - "Vvl5s7jQZOP38p11V3LdE+E16jn4g+viXcsFmSUAFW2NADGfoROhqUAfWka0dwejfO38tomihjnfyS8x", - "+P80T6x1I5LikqkKXGv1St35Az3XNz+VPPNae7u+ff1ywEQk48J40i6Tui+f+MFk6b8W0H/3p+4LDGrn", - "XjxoExg/Yv9tRAkp8Hv/194PDsH3f+39QJOMC/a/9p8k1DBttm+NWEZ3ddPd9QPmCyY+eL/w+qKtsKYd", - "Q+fVYPQ6ZXoM6+tQZg172iu7i2i6T0Z9n17IawJ2d5Lw7ozu4SV117BFxz43id1SqTyEXQOX/N9b6Dur", - "EfSd6+tq3XMMhbHQDl8gVpJUHnqZOux67onb8SaBoTiblBFFqY76CF/+T6WSsJO+llKiWNeveokueonq", - "cq1VTdiCt6ucsH3ck/W9ILbQauOnr5h7d2DQcRRZwdyrWbhL1L2F1AY/fR4ALP9OkTi8OArV+6yjybTk", - "FGvvNX+mjp/1cYf7sKfHz0rghTsyoPpx3LlGwPV799LYk3TK57nMdRXTIaUmWjDt8E4SVr8ZvjRdRSk3", - "tGorPmMqHd3lnXbnyoivdH9LapLmhlrmbb0gNj1GfKmujxFXHjE3EAveYcK4zG19n9VzuyU6xaPBdyXj", - "Gmj+h36XcYUeSZAJR5PyHUPwaXM0Fv8bHka/GEbTX7/zAfH5aLR3iL8zcfnrdz4mXpx4UmHCKO6SMD/5", - "+Rm6nswxugDzNJdQN81xWHwRJD2fkOnf7uVWet90f7p5Kvz6dOv0dKss1/qnmy14y2+3elK3O3+8eXoL", - "Lbj99id9vv3Jbcq1p6XOZzMecSYwNR5qJfWKE7V9Yn41J99MP0qFc+KoeGDWJJHOz8iCa22Q0It865/U", - "xbHfmiFHEmoMSzMDEGsRg0zdNnRQL3IDwE8+YQ5O0Ode5OV8Qte7b2riGwlH7tYS+6wL3A2+dIuksXf9", - "1HUdf6Ghs9IGy8f+cVmKNu2vy/sl3tt9U3a4au/+Vfklk5h9vq0uXQYvhEACSGt/SnMbZ1zULLFjh+TN", - "m5c+ptgi87l0okb6HKI+z/xYVHOIIgyeT85qC9Sx/RwGAdT3WTljRuOEC4ZBGEyHwn/rmX/v9Vh8egk4", - "nNb4rm3vXY6l3dB7lIDvjRXciaxZOBsYZykujmYl43FxWry8iafmi+JXjgEFGE9I1tuhuZEDh1Kws5AW", - "vzYM4X2a0AgRvKGYBZd1wDAWTbraFKK9KJkkTFnQ4Cw3XtwC/G43OC6MLNLiO8nsHJqf5MLw5LzvoUgj", - "lmhCxdKh+Y1FrTMn8yF4AwKT4AgVy+yIGzm+YdAcVI1QCnEWql0SmlzRpR4LB/dgq6cSo6Iji6+dJEPy", - "o0SkHZvrvsJ4baLpb/RYnPM4YRMHlHNOuCZ6IZVhgsUklZdM1/tlVCWcKZzEUworp0lKlyXOqFsfmTEL", - "V1mD45HwbypijimLoediykdjQcneaERSRoV24BqazvDCcW0QHERtQN8SSg5Gj12txr5hagW//FtwmpRi", - "lzKiU8hODVQMQ+RmGzcwdSnEFdNGKti+GVfa7leh33S5YWsby7XPiB33SS5K+BDU9Vesa7BdJlcC5+nM", - "k4yr4hp0iElTFlFYTyHr/SBgtYyiXIUuSNjqSi77f0fBsTK9M1yqMDhHgiqDCGIAJCI/LvBMSzxK29+2", - "UFVJVH+OiyZ4SKQilFToutRosChH1riFoLPnZWJmMXHlz7e/9WcHjq9jBP74i/mXdD8hEcnZrHYAN19N", - "9gCvC4pbJeE/6zl96phejcXFnM6F1IZHnhl6/Pzqo/nrg7DTg3D9ygapeSbVRbsn9A9SXXR9gXmPzS/r", - "IVad4WdoiIDhoZPO/dsjUBtuHytANHf+SGvSV3FKUejiRhe+ygQyGDBV1crfudmg+qrbskibcJkqa+wu", - "cNHgETJxP9qk/hV/czQxRK7VP62zGhIbbHbpszZd2qdHiSR3PV4Jp6oK+GDfgtr6H/S9OIR05TdsC6X3", - "1e0KMlUAGt2I1Fp07mFJA1CtY/FW20Qq59b0dE4KHgwCrc3x4PKKAGwr/IbtW1RXmmXnBZT+9hF5gQe5", - "sm228y2bEgloTcuEWTTWyzQ9P1pNa//u5AQrYZmFTWB/fkR8Kvvi/tBQqgrDCrNIqDbkZwcuu1U8xnFH", - "z0GHWZnftgNoLVMdjEUIrBWwTm2DfEbOK7it5y2ggp7fvpRz/dmYispkUHYuRhL3dETaZCLutTl58CRs", - "+NkdjUJ5EzrCx9ph3DJ67MpgXsp5kYiqRso0y7qSrxsmUvFlmq6hYbJVchCiTSxz8xdtYqYUVnbU3Ubc", - "ZItG9h+GXgChOuRRf7C3x6JlqewMw0sFTLWSr8X+6zJNe/2eGw/UuzJZGsjc8vFwvM0GP/RDO1TB3P0T", - "i+f3diW6pGXFUfHJgqzE415QAOWK5wW39CbQv/UbqoL927junA6g/Rnx2hb405s4vaLxzsR396JpiE9B", - "+ri3U4xiVkMX/jlJ8pXx1vVmQhaoyjiyx3cxMoReJlEiNavZyb4c4EanL2yI3u36Nr/GAxhenPt0l10c", - "gc5c1TNf8zNQYWxyufFjJn66d+57szqCLxmEQq/MBthfA+1vk1POZ09In25LVqbahUK+0ub1lbWdCBOe", - "W6sswlWIXSrR3MiUGh5hOsBoIaWukH0BzW+T5DodfEGZqKGyygIXiHEOpHrutPnnTsQ8cppHQqufXB9D", - "rO7CN8I1/Keyxg8V5UrB8fv+BYWZaTSkTFOczUhGc81AzsxBI7aMEubzPzKwq0c0M7limEaakZQLnuZp", - "NecC7NglRXyo8930vE+muSEJaNlU8dH7LEUyTZmIrZw0FgtGLzlTsCgJNUxEy4FmQnN0jYK0D5izEzQ1", - "WUzRYIaBlIoBBWICi5QZGlNDUdA5hxM/sbFg5yRTDInIakcEe19SA9iOc/GtzaYDzZ77gZ4ThukiuF4U", - "mcsjGjMRBdMonH3ebOzTq/TPmGlO9J4crG7ES+/T46qquvbD+Tycsb5YyI8ObH6N0KvbH9X1IBpPRv+e", - "R9rO1c/xnux0xRKvO8Wfh4GuILrPxkh3/1Y4qUic2+4qpxLJ/M+qRywYSg0RAgJ07Tbe1L5WZGctlvla", - "PG/nD//n8Q20i58JJ+y3Puzb8gCWk/4cWK5b1Rvx3HtSqzpdUlUnd38s2A3q/sQnqSpc7ktRttaQ9wq+", - "XeVORlF8fUnxlW032bbzG7kp2/a62RXPiAoj52KArrZhDu7UuK2s2qkO/k2Dehqzq7DMe2eRpeXizmE0", - "PWvM6BJUJX8GX+s19qNIKmVRRKhhXxb0d0VrWI2yQN1cmWC074Ne352cbLdxCWXW8ghlvmAOUYlsgmpp", - "vKrAfXXJlOKxhzd9evKstMmqXAzJq5QbYiS5YCwrA4MwOBNQOws8lcawm8Ap/R4TRi0zyYXZOIqy6O0M", - "pvxBOliZz1CUdLkkvprDO5vDUbP/5bEz5DKEFhNY/zI11GzMcc3FTKrUymV0KnNoHXgQLBPsp3W2mPGE", - "6aU2LLXOnQAFAccN8xK53POunt3lPro2w8mxUYcZUynXmkuhx8KF3GRMQd9QHdqv+KkFDQKGFvz11DLJ", - "z8MHEgZj3f6oaVs1RL7CnNa9o94OzbKdmBra4mfnhvcRQ/oBnRqJXqZTCKAEr8gLTbYSfmGfJ+RSkwT+", - "2F7rFTnBep86s/7NTxas9LGYyWDeUkuzBTH/qYLjHFvzhsnPgq3d0zvzo70IX7DqwfW8cCZbWazeyGMV", - "o8kA8/J7LCaSG57w3y3bhUa4NjyyUWS02Md3JwWDH47FCTOKR9beG4H/OYJU4Et3J1My2gEcv/0o4wjo", - "t89wcMh82z+n2OPT07dYLmWpVMv+WFDhGn7z5NRammc0Ys2BCmbAkkyOd15t8Fk/w2X6N/aftBNcC0gR", - "3PCv5snrw8a0niHdckRltu7ZJrM/vYOvkya/6ji+TB0H0HA5m60Cq80DrIX1GZcyyVP4h/3jeBNUnaHR", - "4h0W/Wwkbzucjd34CX4Rh9LNKWY2x/O9GGDsgn2p/rOwcH4KKMTUPBODt8AT82ek7k9vSqiu42doenUr", - "6vOnfzZn665vPjcGD5pSXY8v5ZhbSvMzMXK9JuyK8nZN2PeJjC60Q9epqjCJYhQh8+HHEuLcmStRTMBg", - "X+JQqSz2GdP9sWgoQy2IkyaUGKZSLmiyg3O2jSBYu9eo0UvJMeY+wpiZgQZVdyaTBBHZEdEQZgMlZr6B", - "inVZuwyT1TJVw6iRANskU+YB7LdDT7e/Um5+kKqORv+58MU3lfVHlEfKUfe/AYC/vcePAuQ/oe/RbTvO", - "nXHbj2jrhSx/tGqpPsG9Gff2R3rc65Nxby8d92AHnlJU51JDHpCUi9wAsuUzq2vDqOrDEdEskiLWHkff", - "axP3R7pl5o4sWwJ2D7HeXYo9jqpwKV+7TkLsAcoRqI8BRGSreuDcmYz7eOhiInNjTQ/uXLlSMTOoHtm+", - "c7tx5Yx8fdt34eR/dce3xqNwl4FdVrbecvYs1wvWrnJ7aZNm5WYK61Ak2dYL8nc51X2IOrCaeUDMW+F7", - "UPvUdnAXuSOgq+vkjXBz/5o0okPSiHKtwvib1tkTrmRPHRaEk73PpDLwu8MtcDSELwkEA8FUmK+eHo9F", - "BKzIokUqlkrkTg7i3t7CT/56Rp4/fd0nzzDhM/kxn24PySuItMFWtbMXjYWVxCzziigAq9sZxKHr2Y4d", - "qec2Hdehg2tJ8XufztKDJyNg5fF75R3W+70FozFKJH/0XkrbWQBI+vVLOECwf65mse29tcIH6PDVcvBk", - "ZphabfbExWyJAgbFXdIeVdAJbtgxdqg9mF7Zp5UNLNrJ/l4vAH7y4Wsej9tPFH43Fjvs16V2hGi+LzVH", - "MB7EgjmGWGD1ui4yYbRFLDtetvaBgV0eP/skj4pbtVKv4121dAH/rqcLZ/rFGpqy2j4BERcZdDZaen2g", - "8sIiXNtqJKIZjbhZ9glNEndHuZug8I4ZFOLvVDF6AXr7IaCQuZ59cDHYavveUEtiri9sC84WOyTgHqfz", - "aTE4ggfNWo1xzSFE10gS0STKE2oYYbMZizAuGFPy6BZbbjGU3i2enbKTICxMxcM+/+LSFoZpAnevJIsm", - "xe3Yrd5RLEooT9vx5J2ghs6P6GowhUalAN+FxFIcjZTUmrimBizhcw5pNdBTAjJpgJqLpmwssoQKwRTJ", - "tfXQh6EPMsW0zhXzDcDd5SiqT0qsyExJ41wTEimVHgtP4e9OiDYsW0Nmr23LJzjnW5JtbeOup3tSUjfG", - "0K4KcUUIbIilFLvgQEegGroHt3g7oPuWEr+Ug/9G8Tn4vmZKUstkrV+SPdZ+Oe2hr0VPt6YwPStKdUth", - "WrRaiZCsRA+uxfqblPDpce96HoiBzi94Kxyk+3S9iOafoFLHvuuRs+FBuE8fOctQNtZ/x8SnZ5WAxa4K", - "rJLCvzR1UmXktaNaC/rdDPHVOcr3NqNuO2N53RuE15eM3EVrobxtD97PjxBGd4s4cdd5875s2qohb9Xe", - "pi3wA5sTJHwWFHg7mRHuGXHlBpkRPisMAKCde8RiCR7U+4rpr9mepfqa3OBWQ/lthgOEhmsL5bdczzmv", - "rn0ovXNluj2TXIt/JgneLtB15He/7F9f/R2eDJXF2mSCBoJnaWaWrpK3VZZOZ5r/zoYthuDCb/X2TME3", - "cOn8dOTh6bTVofMe0cvu0ZBzLz6jLhsk1+DmV2bd84HdXxjeYfXM1S6WHbh1BlRFC37J2pXu9RPslihT", - "bJDJDI0rsV0wtx7+LjNUDee/E9e8w391/8J0opjIgMUk5opFBrxFhJHIEWwf32iipDTuu1TLdi8Re0R+", - "UDJ94maz4T50Z8opw0o/w3Q5iKmhg0vPbdao0D7Cu9P7UwLDI1yQF9+TLfbeKJu0hMzg5QPR5n5J2fuI", - "sVgjTW5XB7w7Cg8RGp7Mp11GuSb9zCv8gyYkyrWRqd/742dkC9PZzZmAvQBRf4aSbKbkJY9ZXBsjOO/b", - "Vd1tWdDr6l1BqChyEfrHhR3cvcgwXS6k+e88q7OFwiVmygXFwW1M8FI/UxZQAPqjXHgHHLdHfhRfrzD3", - "8tvyjx2gREyt6hbRSGnhpre/XnNf8jVXDYbyd1rttvPuOeuV193iozqGLd1GEooidu5u1dbvPp+QHq6/", - "yGgepzq/LB6kbWrzz4sER3d3P9y1uvzdFxwCCsryy8ay2QbUZZhgXqJPd8wuWSKzFFPcY9lev5erpHfU", - "WxiTHe3soO/3QmpzdPD44X7vw68f/v8BAM9LG8e+AQIA", + "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1Lf6C89pAsfCHh8C", + "BkSHcnEl/Hyr/AUqnDx52ifauoK6ehEVY6FycKOFGVWHt47EqgsZ4NvWy8qPwpEv+szWsrRwLeEcxEPy", + "/D0ISNwQhmk8XYicHotcKAbdwwqQd3/DnGXAP2xGYQB5ZQo5H6HK8BmNbP5NtCN+Szj8i2ng5mPBxEzC", + "ZsUypdxmoLtKuDZ6SHAn/DgdiFSuWWNPa1ghfuzwE445mDWjIKhWTFCk3ubNxgU5+b5yv40OHoWvOEc2", + "q63/19mrnwkKysAWbbHSg9KuvgDCAsc2TOfjxIzngFxmbceYMnXc4/G4V3hYxNs4VlgcVOAPBmjm/w6G", + "9p3tps/j74ZDaMp6EByRX/6wrRzB9ZalFqN43PvQJ5UPc24W+bT49muYANugFc9qdzPZspLntuc3MMOK", + "EG6lVjgS0glmwLxIKRRUfcmmXFC1XBfkHFh6t4JyZp2LK4vxxxidice9o7F3Jx73+uMeE5f4m/M5Hvc+", + "hFegwbRa9OuFv0FBRIej0fZmlH63vgE3gg62uk+slmlVVBTJhWEHHUT0n8tk92+tEvq+UKII6LyETbM3", + "sLeH3fdTG3q/g83/WRqQ4hOWMnTyGlTTJMJNgek9B1bOgev4JsbKyuu+qrUKmCsbT3QqIpb4J/pmpfDd", + "GxqBiiIGQu7dnpz7OjeFKb1Ib/JF2c5xs8rzvdbUd88UN7qr265m4rsf+v3ibG0BS5uzs7FLHxYRTu6E", + "wF1O7UZsYXiLnuGYBmdMGPIcfx26/3o9EQLRnidyfn5k1XyQp4QkXLjYoUpQA8gtbi2xksXuKurZfxaZ", + "NbesiPOvf/wTB8XF/F//+KdLiPGvf/wTj/uOxaTEFP/nC0aVmTJqzo8IRMgOaALvMTcZTMLNLplakv2R", + "sw/iJxiSYFERgCFzeGDCg9/kSlRirGySS+0adGZFmA8XOdMO+wwK8pnLwGX9oAM6Xn+W7VLe6YnuBzBk", + "cQaVCcCt6GkAATi54BgTZp/PvbB5xc65ZmBpunSvOPZu5i+GvTeWegd2gNdkMLjEoXOHH9ykydbZ2fPt", + "IcFnoKUKzLKGj5qyGfe+GX7lSZt5kuUodYaCq2x5U0QzOuUJ9+aJlhRR9gg6lUoZi1AkaPBNHPmRAo8B", + "t2znNN3HomPx6mwHzTGGRSZXrO84gXKwzGUOSeli4rAH5F/coCfpwJUdixmj0AY5fmaZQCVzQRFEXTQs", + "EP0I/eG5qaWr7I+Fhd92WiU4eCDJJlgJ+59Tw67osk8qkq9NKZVQAy913YfCY2HDkt0aDBDfiVSGOUR+", + "Zoc08F7/Lr5TsRkoenAiNkOf7XsLFB4uGqJfxqD77mxkuh0WkU4ch/nN8YkqrfoSW3p15ndjG/VwUcKR", + "GlAJN0enQY94LkVtV4vg0wVV8SCScAlUEfAgWiFh8byNxz6ttNW7RUmm1k/gOP3YJNcvTbhYrE4ADrFF", + "9Vxv5X/mynQz87sW/0x2frtA1zH0W2sPs/zGru5Xo38Ho3943bwDQMgK/8zD1t5edIDt4p6CAzztra65", + "/VJZsvtQNZItjweGFlSpyOnTY0LjWDGtt/+9FZEwU0ulpfwH9yOw4vtwU3Njkcruhde31AnkS2EHr92o", + "CfXzaiYlr95vO7WMZa03XZG8rLzybv/2aHR6nWukFHpLWvt6k2x0zOc6gjDnKrUMUDRKWCG+FOe0SkWb", + "tMrW5b+4ctaKS7YU+FO5A3l3+mXXdS6ad8MdMMVnDYZ4j4ywDstAqCjQG74kan5b7KKb1zr18+dFmqO7", + "k4LuWhUdIvMv6bkYN5YNuKAFRWq9QF8wY6GQbvOd7noITBz03O5U24Eu7ayLadmq1gHHTgg1Mevfvse2", + "SLenr23vz/TyxeW5jsTilvyriNLhsVuu1boH7rHLm39779sa6uMd+9M4AgssMnxwummEUINUtFQvRbT9", + "1aXmk1M0LnblEav8vFlcaLLxe/nOuiu57onwGvUc3MZ18a7lgswSQJS2RoCYz9CJ0FQQEi0j2ruDUb52", + "7t1EUcOc7+SXiBFwmifWuhFJcclUBdW1eqXu/IEO7pufSp55rb1d375+OWAiknFhPGmXSd2XT/xgsvRf", + "i/u/+1P3Bca+cy8etAmMH7H/NvCEFDC//2vvBwf0+7/2fqBJxgX7X/tPEmqYNtu3Riyju7rp7voB8wUT", + "H7xfeH3RVljTjqHzasx6nTI91PV1KLMGUe2V3UXQ3Sejvk8v5DVxvTtJeHdG9/CSumt0o2OfwsRuqVQe", + "6a4BX/7vLfSd1Qj6zvV1te45hsJYBIgvEFJJKo/QTB3EPffE7XiTwFCcTcqIolRHfYQv/6dSSdhJX0sp", + "UazrV71EF71EdbnWqiZswdtVTtg+7sn6XhBbaLXx01dovjsw6DiKrEDz1SzcJTjfQmqDnz4PnJZ/p0gc", + "XhyF6n3W0WRacoq195o/U8fP+rjDfdjT42clPsMdGVD9OO5cI+D6vXtp7Ek65fNc5roamp1SEy2YdrAo", + "CavfDF+arqKUG1q1FZ8xlY7u8k67c2XEV7q/JTVJc0Mt87ZeEJseI75U18eIK4/QHAgZ76BjXIK3vk/+", + "ud0SneJB47uScQ1b/0O/y7hCjyRImKNJ+Y4h+LQ5Gov/DQ+jXwyj6a/f+YD4fDTaO8Tfmbj89TsfEy9O", + "PKkwYRR3uZqf/PwMXU/mGF2A6ZxLRJzmOCwMCZKez9v0b/dyK71vuj/dPBV+fbp1erpVlmv9080WvOW3", + "Wz33250/3jy9hRbcfvuTPt/+5Dbl2tNS57MZjzgTmEEPtZJ6xYnaPjG/mpNvph+lwjlxVDwwa5JI52dk", + "wbU2SOhFWvZP6uLYb02kIwk1hqWZASS2iEFCbxs6qBe5AeAnn1cHJ+hTNPJyPqHr3Tc18Y2EI3dr+X/W", + "Be4GX7pFbtm7fuq6jr/Q0Flpg+Vj/7gsRZv21+X9Eu/tvik7XLV3/6r8kknMPt9Wly6DF0IgT6S1P6W5", + "jTMuapYQs0Py5s1LH1NsAfxc1lEjfapRn45+LKqpRhEGz+dwtQXqEIAOgwDq++SdMaNxwgXDIAymQ+G/", + "9QTB93osPr0EHM5+fNe29y7H0m7oPUrA98YK7kTWLJwNjLMUF0ezkhi5OC1e3sRT80XxK8eAAownJOvt", + "0NzIgUMp2FlIC3MbRvo+TWiEQN9QzGLQOmAYCzpdbQrRXpRMEqYstnCWGy9uAcy3GxwXRhbZ851kdg7N", + "T3JheHLetz6QUDbRhIqlQ/Mbi1pnTuZD8AYEJsERKpbZETdSgcOgOagaoRTiLFS7JDS5oks9Fg7uwVZP", + "JUZFRxaGO0mG5EeJSDs2JX6F8dp81N/osTjnccImDijnnHBN9EIqwwSLSSovma73y6hKOFM4iacUVk6T", + "lC5LnFG3PjJjFq6yBscj4d9UxBwzG0PPxZSPxoKSvdGIpIwK7cA1NJ3hhePaIDiI2oC+JZQcjB67Wo19", + "wwwMfvm34DQpxS5lRKeQxBqoGIbIzTZuYOoyjSumjVSwfTOutN2vQr/pUsjWNpZrnzg77pNclPAhqOuv", + "WNdgu0yuBM7TmScZV8U16BCTpiyisJ5C1vtBXGsZRbkKXZCw1ZWU9/+OgmNleme4VGFwjgRVBhHEAEhE", + "flzgmZZ4lLa/baGqkqj+HBdN8JBIRSip0HWp0WBRjqxxC0Fnz8v8zWLiyp9vf+vPDhxfxwj88RfzL+l+", + "QiKSs1ntAG6+muwBXhcUt0rCf9Zz+tQxvRqLizmdC6kNjzwz9DD71Ufz1wdhpwfh+pUNUvNMqot2T+gf", + "pLro+gLzHptf1kOsOsPP0BABw0Mnnfu3R6A23D5WgGju/JHWpK/ilKLQxY0ufJUJJDpgqqqVv3OzQfVV", + "t2WRNuEyVdbYXeCiwSNk4n60uf8r/uZoYohcq39aZzUkNtjs0mdturRPjxJJ7nq8Ek5VFfDBvgW19T/o", + "e3EI6cpv2BZK76vbFWSqADS6Eam16NzDkgagWsfirbbJLc6t6emcFDwYBFqb48GlHwHYVvgN27eorjTL", + "zgso/e0j8gIPcmXbbOdbNnMS0JqWCbNorJdpen60mv3+3ckJVsIyC5vn/vyI+Iz3xf2hoVQVhhVmkVBt", + "yM8OXHareIzjjp6DDrMyv20H0FqmOhiLEFgrYJ3aBvmMnFdwW89bQAU9v30p5/qzMRWVOaPsXIwk7umI", + "tMlE3Gtz8uBJ2PCzOxqF8iZ0hI+1w7hl9NiVwbyU8yJfVY2UaZZ1JV83TKTiyzRdQ8Nkq+QgRJtY5uYv", + "2sRMKazsqLuNuMkWjew/DL0AQnXIo/5gb49Fy1LZGYaXCphqJV+L/ddlmvb6PTceqHdlsjSQueXj4Xib", + "DX7oh3aogrn7JxbP7+1KdLnNiqPikwVZice9oADKFc8LbulNoH/rN1QF+7dx3TkdQPsz4rUt8Kc3cXpF", + "452J7+5F0xCfgvRxb6cYxayGLvxzkuQr463rzYQsUJVxZI/vYmQIvUyiRGpWs5N9OcCNTl/YEL3b9W1+", + "jQcwvDj3WTG7OAKduapnvuZnoMLY5HLjx0z8dO/c92Z1BF8yCIVemQ2wvwba3yannM+ekD7dlqxMtQuF", + "fKXN6ytrOxEmPLdWWYSrELuMo7mRKTU8wnSA0UJKXSH7Aprf5tJ1OviCMlFDZZUFLhDjHEj13Gnzz52I", + "eeQ0j4RWP7k+hljdhW+Ea/hPZY0fKsqVguP3/QsKM9NoSJmmOJuRjOaagZyZg0ZsGSXM539kYFePaGZy", + "xTDbNCMpFzzN02rOBdixS4r4UOe76XmfTHNDEtCyqeKj91mKZJoyEVs5aSwWjF5ypmBREmqYiJYDzYTm", + "6BoFaR8wZydoarKYosEMAykVAwrEBBYpMzSmhqKgcw4nfmJjwc5JphgSkdWOCPa+pIYY85t+a7PpQLPn", + "fqDnhGG6CK4XRYLziMZMRME0CmefNxv79Cr9M2aaE70nB6sb8dL79Liqqq79cD4PZ6wvFvKjA5tfI/Tq", + "9kd1PYjGk9G/55G2c/VzvCc7XbHE607x52GgK4juszHS3b8VTioS57a7yqlEMv+z6hELhlJDhIAAXbuN", + "N7WvFdlZi2W+Fs/b+cP/eXwD7eJnwgn7rQ/7tjyA5aQ/B5brVvVGPPee1KpOl1TVyd0fC3aDuj/xSaoK", + "l/tSlK015L2Cb1e5k1EUX19SfGXbTbbt/EZuyra9bnbFM6LCyLkYoKttmIM7NW4rq3aqg3/ToJ7G7Cos", + "895ZZGm5uHMYTc8aM7oEVcmfwdd6jf0okkpZFBFq2JcF/V3RGlajLFA3VyYY7fug13cnJ9ttXEKZtTxC", + "mS+YQ1Qim6BaGq8qcF9dMqV47OFNn548K22yKhdD8irlhhhJLhjLysAgDM4E1M4CT6Ux7CZwSr/HhFHL", + "THJhNo6iLHo7gyl/kA5W5jMUJV0uia/m8M7mcNTsf3nsDLkMocUE1r9MDTUbc1xzMZMqtXIZncocWgce", + "BMsE+2mdLWY8YXqpDUutcydAQcBxw7xELve8q2d3uY+uzXBybNRhxlTKteZS6LFwITcZU9A3VIf2K35q", + "QYOAoQV/PbVM8vPwgYTBWLc/atpWDZGvMKd176i3Q7NsJ6aGtvjZueF9xJB+QKdGopfpFAIowSvyQpOt", + "hF/Y5wm51CSBP7bXekVOsN6nzqx/85MFK30sZjKYt9TSbEHMf6rgOMfWvGHys2Br9/TO/GgvwhesenA9", + "L5zJVharN/JYxWgywLz8HouJ5IYn/HfLdqERrg2PbBQZLfbx3UnB4IdjccKM4pG190bgf44gFfjS3cmU", + "jHYAx28/yjgC+u0zHBwy3/bPKfb49PQtlktZKtWyPxZUuIbfPDm1luYZjVhzoIIZsCST451XG3zWz3CZ", + "/o39J+0E1wJSBDf8q3ny+rAxrWdItxxRma17tsnsT+/g66TJrzqOL1PHATRczmarwGrzAGthfcalTPIU", + "/mH/ON4EVWdotHiHRT8bydsOZ2M3foJfxKF0c4qZzfF8LwYYu2Bfqv8sLJyfAgoxNc/E4C3wxPwZqfvT", + "mxKq6/gZml7divr86Z/N2brrm8+NwYOmVNfjSznmltL8TIxcrwm7orxdE/Z9IqML7dB1qipMohhFyHz4", + "sYQ4d+ZKFBMw2Jc4VCqLfcZ0fywaylAL4qQJJYaplAua7OCcbSMI1u41avRScoy5jzBmZqBB1Z3JJEFE", + "dkQ0hNlAiZlvoGJd1i7DZLVM1TBqJMA2yZR5APvt0NPtr5SbH6Sqo9F/LnzxTWX9EeWRctT9bwDgb+/x", + "owD5T+h7dNuOc2fc9iPaeiHLH61aqk9wb8a9/ZEe9/pk3NtLxz3YgacU1bnUkAck5SI3gGz5zOraMKr6", + "cEQ0i6SItcfR99rE/ZFumbkjy5aA3UOsd5dij6MqXMrXrpMQe4ByBOpjABHZqh44dybjPh66mMjcWNOD", + "O1euVMwMqke279xuXDkjX9/2XTj5X93xrfEo3GVgl5Wtt5w9y/WCtavcXtqkWbmZwjoUSbb1gvxdTnUf", + "og6sZh4Q81b4HtQ+tR3cRe4I6Oo6eSPc3L8mjeiQNKJcqzD+pnX2hCvZU4cF4WTvM6kM/O5wCxwN4UsC", + "wUAwFearp8djEQErsmiRiqUSuZODuLe38JO/npHnT1/3yTNM+Ex+zKfbQ/IKIm2wVe3sRWNhJTHLvCIK", + "wOp2BnHoerZjR+q5Tcd16OBaUvzep7P04MkIWHn8XnmH9X5vwWiMEskfvZfSdhYAkn79Eg4Q7J+rWWx7", + "b63wATp8tRw8mRmmVps9cTFbooBBcZe0RxV0ght2jB1qD6ZX9mllA4t2sr/XC4CffPiax+P2E4XfjcUO", + "+3WpHSGa70vNEYwHsWCOIRZYva6LTBhtEcuOl619YGCXx88+yaPiVq3U63hXLV3Av+vpwpl+sYamrLZP", + "QMRFBp2Nll4fqLywCNe2GoloRiNuln1Ck8TdUe4mKLxjBoX4O1WMXoDefggoZK5nH1wMttq+N9SSmOsL", + "24KzxQ4JuMfpfFoMjuBBs1ZjXHMI0TWSRDSJ8oQaRthsxiKMC8aUPLrFllsMpXeLZ6fsJAgLU/Gwz7+4", + "tIVhmsDdK8miSXE7dqt3FIsSytN2PHknqKHzI7oaTKFRKcB3IbEURyMltSauqQFL+JxDWg30lIBMGqDm", + "oikbiyyhQjBFcm099GHog0wxrXPFfANwdzmK6pMSKzJT0jjXhERKpcfCU/i7E6INy9aQ2Wvb8gnO+ZZk", + "W9u46+melNSNMbSrQlwRAhtiKcUuONARqIbuwS3eDui+pcQv5eC/UXwOvq+ZktQyWeuXZI+1X0576GvR", + "060pTM+KUt1SmBatViIkK9GDa7H+JiV8ety7ngdioPML3goH6T5dL6L5J6jUse965Gx4EO7TR84ylI31", + "3zHx6VklYLGrAquk8C9NnVQZee2o1oJ+N0N8dY7yvc2o285YXvcG4fUlI3fRWihv24P38yOE0d0iTtx1", + "3rwvm7ZqyFu1t2kL/MDmBAmfBQXeTmaEe0ZcuUFmhM8KAwBo5x6xWIIH9b5i+mu2Z6m+Jje41VB+m+EA", + "oeHaQvkt13POq2sfSu9cmW7PJNfin0mCtwt0HfndL/vXV3+HJ0NlsTaZoIHgWZqZpavkbZWl05nmv7Nh", + "iyG48Fu9PVPwDVw6Px15eDptdei8R/SyezTk3IvPqMsGyTW4+ZVZ93xg9xeGd1g9c7WLZQdunQFV0YJf", + "snale/0EuyXKFBtkMkPjSmwXzK2Hv8sMVcP578Q17/Bf3b8wnSgmMmAxiblikQFvEWEkcgTbxzeaKCmN", + "+y7Vst1LxB6RH5RMn7jZbLgP3ZlyyrDSzzBdDmJq6ODSc5s1KrSP8O70/pTA8AgX5MX3ZIu9N8omLSEz", + "ePlAtLlfUvY+YizWSJPb1QHvjsJDhIYn82mXUa5JP/MK/6AJiXJtZOr3/vgZ2cJ0dnMmYC9A1J+hJJsp", + "ecljFtfGCM77dlV3Wxb0unpXECqKXIT+cWEHdy8yTJcLaf47z+psoXCJmXJBcXAbE7zUz5QFFID+KBfe", + "AcftkR/F1yvMvfy2/GMHKBFTq7pFNFJauOntr9fcl3zNVYOh/J1Wu+28e8565XW3+KiOYUu3kYSiiJ27", + "W7X1u88npIfrLzKax6nOL4sHaZva/PMiwdHd3Q93rS5/9wWHgIKy/LKxbLYBdRkmmJfo0x2zS5bILMUU", + "91i21+/lKukd9RbGZEc7O+j7vZDaHB08frjf+/Drh/9/ABXpBjflAQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 177cb24f3..a51553ab5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4692,8 +4692,9 @@ paths: is limited to 64 MiB compressed; only machine_network_mode, timeout_seconds, memory_mb, cpus and tags are accepted alongside source and this selector. CPU/memory zero inherit the base. - Linux/cache/secrets/admin options are rejected. Output retains the - base identity and cannot run concurrently with it. + Linux/cache/secrets/admin options are rejected. The builder and its + output get their own machine identifier and MAC, so the output can + run beside the base. machine_network_mode: type: string enum: [isolated, egress]