From 281e2d9eb4d048504867e559b019815abc04dd32 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sat, 26 Sep 2026 20:37:14 +0530 Subject: [PATCH 01/24] feat(gcp): Managed Service for Apache Kafka control plane (managedkafka.googleapis.com v1) Operations: clusters create/update/delete (LROs registered with the shared location-operations poller), get, list (pageToken/pageSize); topics create/get/list/patch/delete (synchronous; create/patch return Topic, delete returns Empty). Deleting a cluster removes its topics. Snapshot/restore included. Validation: vcpuCount >= 3; memoryBytes 1-8 GiB per vCPU inclusive; 1-10 networkConfigs with projects/*/regions/*/subnetworks/* subnets; RFC 1035 clusterId; Kafka topic-name alphabet; partitionCount/replicationFactor > 0; updateMask required, unknown/immutable/output-only paths rejected 400; partitionCount may only increase; replicationFactor and kmsKey immutable. Duplicates 409, missing resources 404. Routing: the clusters path is identical to GKE's and AlloyDB's, so the handler registers ahead of GKE and claims only Kafka-shaped creates (capacityConfig/gcpConfig body), clusters it owns, lists where it owns a cluster, and clusters/{c}/topics. Everything else falls through. Out of scope: consumer groups, ACLs, Connect clusters/connectors, schema registry, tlsConfig/updateOptions, brokerDetails/FULL view. --- docs/coverage/README.md | 1 + docs/coverage/coverage.json | 46 ++ docs/coverage/gcp/README.md | 1 + docs/coverage/gcp/managedkafka.md | 24 + docs/sdk-server.md | 1 + providers/gcp/gcp.go | 3 + providers/gcp/managedkafka/clone.go | 37 ++ providers/gcp/managedkafka/managedkafka.go | 227 ++++++++++ .../gcp/managedkafka/managedkafka_test.go | 237 ++++++++++ providers/gcp/managedkafka/snapshot.go | 90 ++++ providers/gcp/managedkafka/topics.go | 142 ++++++ providers/gcp/managedkafka/validate.go | 300 +++++++++++++ server/gcp/from_provider.go | 1 + server/gcp/gcp.go | 27 ++ server/gcp/managedkafka/handler.go | 275 ++++++++++++ server/gcp/managedkafka/matches_test.go | 102 +++++ server/gcp/managedkafka/operations.go | 224 ++++++++++ server/gcp/managedkafka/sdk_roundtrip_test.go | 421 ++++++++++++++++++ server/gcp/managedkafka/wire.go | 260 +++++++++++ server/gcp/operations_ownership_test.go | 50 +++ services/managedkafka/driver/driver.go | 102 +++++ 21 files changed, 2571 insertions(+) create mode 100644 docs/coverage/gcp/managedkafka.md create mode 100644 providers/gcp/managedkafka/clone.go create mode 100644 providers/gcp/managedkafka/managedkafka.go create mode 100644 providers/gcp/managedkafka/managedkafka_test.go create mode 100644 providers/gcp/managedkafka/snapshot.go create mode 100644 providers/gcp/managedkafka/topics.go create mode 100644 providers/gcp/managedkafka/validate.go create mode 100644 server/gcp/managedkafka/handler.go create mode 100644 server/gcp/managedkafka/matches_test.go create mode 100644 server/gcp/managedkafka/operations.go create mode 100644 server/gcp/managedkafka/sdk_roundtrip_test.go create mode 100644 server/gcp/managedkafka/wire.go create mode 100644 services/managedkafka/driver/driver.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index e137e36a5..cb6af7a82 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -118,6 +118,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `managedcassandra` | - | [ManagedCassandra](./azure/managedcassandra.md) | - | - | 15 | | `managedgrafana` | - | [ManagedGrafana](./azure/managedgrafana.md) | - | - | 8 | | `managedidentity` | - | [ManagedIdentity](./azure/managedidentity.md) | - | - | 8 | +| `managedkafka` | - | - | [ManagedKafka](./gcp/managedkafka.md) | - | 11 | | `managedlustre` | - | [ManagedLustre](./azure/managedlustre.md) | - | - | 12 | | `memorydb` | [MemoryDB](./aws/memorydb.md) | - | - | - | 33 | | `messagequeue` | [SQS](./aws/sqs.md) | [QueueStorage](./azure/queuestorage.md) | [PubSub](./gcp/pubsub.md) | - | 14 | diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index c12670652..8050f4e39 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -10070,6 +10070,52 @@ "azure": "ManagedIdentity" } }, + { + "service": "managedkafka", + "interface": "ManagedKafka", + "operations": [ + { + "name": "CreateCluster" + }, + { + "name": "CreateTopic" + }, + { + "name": "DeleteCluster", + "doc": "DeleteCluster removes the cluster and every topic under it." + }, + { + "name": "DeleteTopic" + }, + { + "name": "GetCluster" + }, + { + "name": "GetOperation", + "doc": "GetOperation resolves a (done) long-running operation by name, for a" + }, + { + "name": "GetTopic" + }, + { + "name": "ListClusters" + }, + { + "name": "ListTopics" + }, + { + "name": "UpdateCluster", + "doc": "UpdateCluster applies the fields of c named by mask (field-mask paths" + }, + { + "name": "UpdateTopic", + "doc": "UpdateTopic applies the fields of t named by mask (\"*\" means every mutable" + } + ], + "providers": { + "gcp": "ManagedKafka" + } + }, { "service": "managedlustre", "interface": "provider-native", diff --git a/docs/coverage/gcp/README.md b/docs/coverage/gcp/README.md index cfcdbcbc9..683608c1d 100644 --- a/docs/coverage/gcp/README.md +++ b/docs/coverage/gcp/README.md @@ -44,6 +44,7 @@ Services cloudemu emulates for GCP, by native name. Back to the [cross-provider | [KMS](./kms.md) | (provider-native) | 17 | | [LB](./lb.md) | `loadbalancer` | 19 | | [LRO](./lro.md) | (provider-native) | 1 | +| [ManagedKafka](./managedkafka.md) | `managedkafka` | 11 | | [Memorystore](./memorystore.md) | `cache` | 17 | | [Metastore](./metastore.md) | `metastore` | 6 | | [NetworkConnectivity](./networkconnectivity.md) | `networkconnectivity` | 11 | diff --git a/docs/coverage/gcp/managedkafka.md b/docs/coverage/gcp/managedkafka.md new file mode 100644 index 000000000..39ede2ddd --- /dev/null +++ b/docs/coverage/gcp/managedkafka.md @@ -0,0 +1,24 @@ + +# ManagedKafka + +GCP's `managedkafka` service · portable interface `driver.ManagedKafka` · [GCP index](./README.md) + +## Operations (11) + +| Operation | Description | +| --- | --- | +| `CreateCluster` | | +| `CreateTopic` | | +| `DeleteCluster` | DeleteCluster removes the cluster and every topic under it. | +| `DeleteTopic` | | +| `GetCluster` | | +| `GetOperation` | GetOperation resolves a (done) long-running operation by name, for a | +| `GetTopic` | | +| `ListClusters` | | +| `ListTopics` | | +| `UpdateCluster` | UpdateCluster applies the fields of c named by mask (field-mask paths | +| `UpdateTopic` | UpdateTopic applies the fields of t named by mask ("*" means every mutable | + +## Not in scope + +_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 346aa102b..9232e33dc 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -228,6 +228,7 @@ All handlers speak REST + JSON. | **GKE** | Clusters (Create/Get/List/Update/Delete + `:setLogging`/`:setMonitoring`/`:setMasterAuth`/`:setLegacyAbac`/`:setNetworkPolicy`/`:setMaintenancePolicy`/`:setResourceLabels`/`:startIpRotation`/`:completeIpRotation`), NodePools (Create/Get/List/Update/Delete + `:setSize`/`:setAutoscaling`/`:setManagement`/`:rollback`), Operations (Get/List/`:cancel`). The cluster endpoint points at the shared [Kubernetes data plane](#kubernetes). | | **Cloud Asset Inventory** | `assets.list` (filter by `assetTypes[]`), `searchAllResources` (query + asset-type filter), `searchAllIamPolicies` (returns empty; not implemented), `exportAssets` (sync; inline results in the returned Operation), `batchGetAssetsHistory`, Feeds (create/list/get/patch/delete), `operations.get`. Resource names returned as GCP-shaped `//service/path` URNs. | | **IAM (iam.googleapis.com v1)** | ServiceAccounts (Create/Get/List/Delete/Patch), custom Roles (Create/Get/List/Delete/Patch), ServiceAccountKeys (Create/Get/List/Delete). Real `google.golang.org/api/iam/v1` clients round-trip end-to-end; errors surface as typed `*googleapi.Error`. Resource-level `getIamPolicy`/`setIamPolicy` bindings on individual GCP resources are out of scope. | +| **Managed Service for Apache Kafka (managedkafka.googleapis.com v1)** | Clusters (Create/Update/Delete as LROs polled through the shared location-operations route, Get, List with `pageToken`/`pageSize`; `updateMask` required, only masked fields change, unknown/immutable/output-only paths are 400) and Topics (Create/Get/List/Patch/Delete, synchronous; `partitionCount` can only increase, `replicationFactor` immutable). Validates `vcpuCount >= 3`, 1-8 GiB memory per vCPU, 1-10 `networkConfigs[].subnet`, and the RFC 1035 `clusterId`. Shares the `/clusters` path with GKE/AlloyDB: it claims only Kafka-shaped creates and clusters it owns. Real `google.golang.org/api/managedkafka/v1` clients round-trip end-to-end. Consumer groups, ACLs, Connect and Schema Registry are out of scope. | An operation cloudemu doesn't implement returns `501 Not Implemented` or the provider's native `UnknownOperation` / `NotImplemented` / `NOT_FOUND` error. diff --git a/providers/gcp/gcp.go b/providers/gcp/gcp.go index c6f264936..598b46d56 100644 --- a/providers/gcp/gcp.go +++ b/providers/gcp/gcp.go @@ -41,6 +41,7 @@ import ( gkehubprov "github.com/stackshy/cloudemu/v2/providers/gcp/gkehub" "github.com/stackshy/cloudemu/v2/providers/gcp/iam" "github.com/stackshy/cloudemu/v2/providers/gcp/loadbalancer" + managedkafkaprov "github.com/stackshy/cloudemu/v2/providers/gcp/managedkafka" "github.com/stackshy/cloudemu/v2/providers/gcp/memorystore" metastoreprov "github.com/stackshy/cloudemu/v2/providers/gcp/metastore" "github.com/stackshy/cloudemu/v2/providers/gcp/monitoring" @@ -120,6 +121,7 @@ type Provider struct { Metastore *metastoreprov.Mock VPCAccess *vpcaccessprov.Mock CloudIDS *cloudidsprov.Mock + ManagedKafka *managedkafkaprov.Mock SecureSourceManager *securesourcemanagerprov.Mock NetworkConnectivity *networkconnectivity.Mock Composer *composer.Mock @@ -190,6 +192,7 @@ func New(opts ...config.Option) *Provider { Metastore: metastoreprov.New(o), VPCAccess: vpcaccessprov.New(o), CloudIDS: cloudidsprov.New(o), + ManagedKafka: managedkafkaprov.New(o), SecureSourceManager: securesourcemanagerprov.New(o), NetworkConnectivity: networkconnectivity.New(o), Composer: composer.New(o), diff --git a/providers/gcp/managedkafka/clone.go b/providers/gcp/managedkafka/clone.go new file mode 100644 index 000000000..ba9b4901d --- /dev/null +++ b/providers/gcp/managedkafka/clone.go @@ -0,0 +1,37 @@ +package managedkafka + +import ( + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +// cloneCluster returns a deep copy of c so a stored cluster is never aliased by a +// value handed back to a caller. +func cloneCluster(c *mkdriver.Cluster) mkdriver.Cluster { + out := *c + out.Subnets = append([]string(nil), c.Subnets...) + out.Labels = cloneStringMap(c.Labels) + + return out +} + +// cloneTopic returns a deep copy of t. +func cloneTopic(t *mkdriver.Topic) mkdriver.Topic { + out := *t + out.Configs = cloneStringMap(t.Configs) + + return out +} + +// cloneStringMap deep-copies a string map; an empty map clones to nil. +func cloneStringMap(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + + return out +} diff --git a/providers/gcp/managedkafka/managedkafka.go b/providers/gcp/managedkafka/managedkafka.go new file mode 100644 index 000000000..37c952a37 --- /dev/null +++ b/providers/gcp/managedkafka/managedkafka.go @@ -0,0 +1,227 @@ +// Package managedkafka provides an in-memory mock of the Google Cloud Managed +// Service for Apache Kafka control plane (managedkafka.googleapis.com/v1). It +// models clusters, the topics nested under them, and the long-running +// operations cluster mutations return. It is control-plane only: there are no +// brokers and no produce/consume data plane. +package managedkafka + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/memstore" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +var _ mkdriver.ManagedKafka = (*Mock)(nil) + +const ( + clustersColl = "clusters" + topicsColl = "topics" + + // stateActive is the steady state a created cluster reports. Real Managed + // Kafka passes through CREATING first; CloudEmu completes synchronously. + stateActive = "ACTIVE" + + opCreate = "create" + opUpdate = "update" + opDelete = "delete" +) + +// Mock is the in-memory Managed Kafka control-plane implementation. Clusters and +// topics are keyed by their full GCP resource names. +type Mock struct { + mu sync.RWMutex + + clusters *memstore.Store[mkdriver.Cluster] + topics *memstore.Store[mkdriver.Topic] + operations *memstore.Store[mkdriver.Operation] + + opSeq atomic.Uint64 + opts *config.Options +} + +// New creates a new Managed Kafka mock. +func New(opts *config.Options) *Mock { + return &Mock{ + clusters: memstore.New[mkdriver.Cluster](), + topics: memstore.New[mkdriver.Topic](), + operations: memstore.New[mkdriver.Operation](), + opts: opts, + } +} + +// clusterName builds the full cluster resource name. +func clusterName(project, location, id string) string { + return "projects/" + project + "/locations/" + location + "/" + clustersColl + "/" + id +} + +// topicName builds the full topic resource name. +func topicName(project, location, clusterID, id string) string { + return clusterName(project, location, clusterID) + "/" + topicsColl + "/" + id +} + +// newOp records a completed operation scoped to the project+location it acted in +// and returns it. The caller holds the write lock. +func (m *Mock) newOp(project, location, opType, target string) *mkdriver.Operation { + scope := "projects/" + project + "/locations/" + location + op := mkdriver.Operation{ + Name: fmt.Sprintf("%s/operations/operation-%d-%s", scope, m.opSeq.Add(1), idgen.UUID()), + Done: true, + TargetName: target, + Type: opType, + } + m.operations.Set(op.Name, op) + + return &op +} + +// CreateCluster validates and stores a new cluster, reporting it ACTIVE, and +// returns the completed LRO. +func (m *Mock) CreateCluster(_ context.Context, c *mkdriver.Cluster) (*mkdriver.Cluster, *mkdriver.Operation, error) { + if err := validateClusterID(c.ID); err != nil { + return nil, nil, err + } + + if err := validateCluster(c); err != nil { + return nil, nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + key := clusterName(c.Project, c.Location, c.ID) + if m.clusters.Has(key) { + return nil, nil, cerrors.Newf(cerrors.AlreadyExists, "cluster %q already exists", key) + } + + now := m.opts.Clock.Now().UTC() + stored := cloneCluster(c) + stored.State = stateActive + stored.CreateTime = now + stored.UpdateTime = now + m.clusters.Set(key, stored) + + op := m.newOp(c.Project, c.Location, opCreate, key) + out := cloneCluster(&stored) + + return &out, op, nil +} + +// GetCluster returns a cluster by identity, cloned. +func (m *Mock) GetCluster(_ context.Context, project, location, id string) (*mkdriver.Cluster, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + c, ok := m.clusters.Get(clusterName(project, location, id)) + if !ok { + return nil, clusterNotFound(project, location, id) + } + + out := cloneCluster(&c) + + return &out, nil +} + +// ListClusters returns every cluster in a project+location, ordered by name. +func (m *Mock) ListClusters(_ context.Context, project, location string) ([]mkdriver.Cluster, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + prefix := "projects/" + project + "/locations/" + location + "/" + clustersColl + "/" + all := m.clusters.SortedValues() + out := make([]mkdriver.Cluster, 0, len(all)) + + for i := range all { + if strings.HasPrefix(clusterName(all[i].Project, all[i].Location, all[i].ID), prefix) { + out = append(out, cloneCluster(&all[i])) + } + } + + return out, nil +} + +// UpdateCluster applies the masked fields of c to the stored cluster, +// re-validates the result, and returns the completed LRO. Unknown, immutable and +// output-only mask paths are rejected with INVALID_ARGUMENT before anything +// changes. +func (m *Mock) UpdateCluster(_ context.Context, c *mkdriver.Cluster, mask []string) ( + *mkdriver.Cluster, *mkdriver.Operation, error, +) { + m.mu.Lock() + defer m.mu.Unlock() + + key := clusterName(c.Project, c.Location, c.ID) + + stored, ok := m.clusters.Get(key) + if !ok { + return nil, nil, clusterNotFound(c.Project, c.Location, c.ID) + } + + next := cloneCluster(&stored) + if err := applyClusterMask(&next, c, mask); err != nil { + return nil, nil, err + } + + if err := validateCluster(&next); err != nil { + return nil, nil, err + } + + next.UpdateTime = m.opts.Clock.Now().UTC() + m.clusters.Set(key, next) + + op := m.newOp(c.Project, c.Location, opUpdate, key) + out := cloneCluster(&next) + + return &out, op, nil +} + +// DeleteCluster removes a cluster together with every topic under it and returns +// the completed LRO. +func (m *Mock) DeleteCluster(_ context.Context, project, location, id string) (*mkdriver.Operation, error) { + m.mu.Lock() + defer m.mu.Unlock() + + key := clusterName(project, location, id) + if !m.clusters.Has(key) { + return nil, clusterNotFound(project, location, id) + } + + m.clusters.Delete(key) + + prefix := key + "/" + topicsColl + "/" + for _, k := range m.topics.Keys() { + if strings.HasPrefix(k, prefix) { + m.topics.Delete(k) + } + } + + return m.newOp(project, location, opDelete, key), nil +} + +// GetOperation returns a (done) long-running operation by name. An unknown name +// is reported as a done operation: the mock completes synchronously, so any op +// id a standalone poll asks for has already finished. In an assembled server +// the shared LRO poller answers polls instead, and 404s unknown names. +func (m *Mock) GetOperation(_ context.Context, name string) (*mkdriver.Operation, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + op, ok := m.operations.Get(name) + if !ok { + return &mkdriver.Operation{Name: name, Done: true}, nil + } + + return &op, nil +} + +// clusterNotFound builds the NOT_FOUND error carrying the full resource name. +func clusterNotFound(project, location, id string) error { + return cerrors.Newf(cerrors.NotFound, "cluster %q not found", clusterName(project, location, id)) +} diff --git a/providers/gcp/managedkafka/managedkafka_test.go b/providers/gcp/managedkafka/managedkafka_test.go new file mode 100644 index 000000000..75ed783d0 --- /dev/null +++ b/providers/gcp/managedkafka/managedkafka_test.go @@ -0,0 +1,237 @@ +package managedkafka + +import ( + "context" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +const ( + proj = "p" + region = "us-central1" + subnet = "projects/p/regions/us-central1/subnetworks/s" +) + +func newMock(t *testing.T) (*Mock, *config.FakeClock) { + t.Helper() + + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + return New(config.NewOptions(config.WithProjectID(proj), config.WithClock(clock))), clock +} + +func cluster(id string) *mkdriver.Cluster { + return &mkdriver.Cluster{ + Project: proj, Location: region, ID: id, + VcpuCount: 3, MemoryBytes: 3 * gib, Subnets: []string{subnet}, + KmsKey: "k1", Labels: map[string]string{"a": "b"}, + } +} + +func mustCreate(t *testing.T, m *Mock, id string) { + t.Helper() + + if _, _, err := m.CreateCluster(context.Background(), cluster(id)); err != nil { + t.Fatalf("CreateCluster(%s): %v", id, err) + } +} + +func TestClusterCRUDAndClone(t *testing.T) { + m, clock := newMock(t) + ctx := context.Background() + + c, op, err := m.CreateCluster(ctx, cluster("c1")) + if err != nil { + t.Fatalf("create: %v", err) + } + + if c.State != stateActive || !op.Done || op.Type != opCreate || c.CreateTime != clock.Now() { + t.Fatalf("created = %+v op = %+v", c, op) + } + + // Mutating a returned value must not alias the store. + c.Labels["a"] = "mutated" + c.Subnets[0] = "x" + + got, err := m.GetCluster(ctx, proj, region, "c1") + if err != nil || got.Labels["a"] != "b" || got.Subnets[0] != subnet { + t.Fatalf("store aliased: %+v %v", got, err) + } + + if _, _, err = m.CreateCluster(ctx, cluster("c1")); !cerrors.IsAlreadyExists(err) { + t.Fatalf("duplicate: %v", err) + } + + if list, _ := m.ListClusters(ctx, proj, region); len(list) != 1 { + t.Fatalf("list = %d", len(list)) + } + + if _, err = m.DeleteCluster(ctx, proj, region, "c1"); err != nil { + t.Fatalf("delete: %v", err) + } + + if _, err = m.GetCluster(ctx, proj, region, "c1"); !cerrors.IsNotFound(err) { + t.Fatalf("get after delete: %v", err) + } +} + +func TestClusterValidation(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + cases := map[string]func(c *mkdriver.Cluster){ + "vcpu 2": func(c *mkdriver.Cluster) { c.VcpuCount = 2; c.MemoryBytes = 2 * gib }, + "mem under": func(c *mkdriver.Cluster) { c.MemoryBytes = 3*gib - 1 }, + "mem over": func(c *mkdriver.Cluster) { c.MemoryBytes = 24*gib + 1 }, + "vcpu overflow": func(c *mkdriver.Cluster) { c.VcpuCount = 1 << 62 }, + "no subnets": func(c *mkdriver.Cluster) { c.Subnets = nil }, + "11 subnets": func(c *mkdriver.Cluster) { c.Subnets = make([]string, maxNetworkConfigs+1) }, + "bad subnet shape": func(c *mkdriver.Cluster) { c.Subnets = []string{"projects/p/zones/z/subnetworks/s"} }, + "bad id": func(c *mkdriver.Cluster) { c.ID = "9starts-with-digit" }, + "empty id": func(c *mkdriver.Cluster) { c.ID = "" }, + "bad mode": func(c *mkdriver.Cluster) { c.RebalanceMode = "MODE_UNSPECIFIED" }, + } + + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + c := cluster("v") + mutate(c) + + if _, _, err := m.CreateCluster(ctx, c); !cerrors.IsInvalidArgument(err) { + t.Fatalf("want INVALID_ARGUMENT, got %v", err) + } + }) + } +} + +func TestClusterMask(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + mustCreate(t, m, "c1") + + desired := &mkdriver.Cluster{ + Project: proj, Location: region, ID: "c1", + VcpuCount: 4, MemoryBytes: 32 * gib, Subnets: []string{subnet, subnet + "2"}, + RebalanceMode: rebalanceNone, Labels: map[string]string{"x": "y"}, + } + + got, op, err := m.UpdateCluster(ctx, desired, []string{"capacity_config", "gcp_config.access_config.network_configs"}) + if err != nil { + t.Fatalf("update: %v", err) + } + + if op.Type != opUpdate || got.VcpuCount != 4 || got.MemoryBytes != 32*gib || len(got.Subnets) != 2 || + got.RebalanceMode != "" || got.Labels["a"] != "b" || got.KmsKey != "k1" { + t.Fatalf("masked update = %+v", got) + } + + desired.KmsKey = "k1" + + all, _, err := m.UpdateCluster(ctx, desired, []string{"*"}) + if err != nil || all.RebalanceMode != rebalanceNone || all.Labels["x"] != "y" { + t.Fatalf("* update = %+v %v", all, err) + } + + for _, bad := range [][]string{nil, {"nope"}, {"state"}, {"gcpConfig.kmsKey"}} { + if _, _, err := m.UpdateCluster(ctx, desired, bad); !cerrors.IsInvalidArgument(err) { + t.Fatalf("mask %v: want INVALID_ARGUMENT, got %v", bad, err) + } + } + + desired.KmsKey = "other" + if _, _, err := m.UpdateCluster(ctx, desired, []string{"gcpConfig"}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("kmsKey change via gcpConfig: %v", err) + } + + if _, _, err := m.UpdateCluster(ctx, &mkdriver.Cluster{Project: proj, Location: region, ID: "ghost"}, + []string{"labels"}); !cerrors.IsNotFound(err) { + t.Fatalf("update missing: %v", err) + } +} + +func TestTopicsAndCascade(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + topic := func(id string, parts int32) *mkdriver.Topic { + return &mkdriver.Topic{ + Project: proj, Location: region, ClusterID: "c1", ID: id, + PartitionCount: parts, ReplicationFactor: 3, Configs: map[string]string{"k": "v"}, + } + } + + if _, err := m.CreateTopic(ctx, topic("t1", 1)); !cerrors.IsNotFound(err) { + t.Fatalf("topic under missing cluster: %v", err) + } + + mustCreate(t, m, "c1") + + if _, err := m.CreateTopic(ctx, topic("t1", 1)); err != nil { + t.Fatalf("create topic: %v", err) + } + + for _, id := range []string{"", "bad/id", ".."} { + if _, err := m.CreateTopic(ctx, topic(id, 1)); !cerrors.IsInvalidArgument(err) { + t.Fatalf("topic id %q: %v", id, err) + } + } + + if _, err := m.UpdateTopic(ctx, topic("t1", 4), []string{"partition_count"}); err != nil { + t.Fatalf("grow: %v", err) + } + + if _, err := m.UpdateTopic(ctx, topic("t1", 2), []string{"partitionCount"}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("shrink: %v", err) + } + + rf := topic("t1", 4) + rf.ReplicationFactor = 1 + + if _, err := m.UpdateTopic(ctx, rf, []string{"*"}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("replicationFactor change via *: %v", err) + } + + if _, err := m.DeleteCluster(ctx, proj, region, "c1"); err != nil { + t.Fatalf("delete cluster: %v", err) + } + + if m.topics.Len() != 0 { + t.Fatalf("topics survived cluster delete: %d", m.topics.Len()) + } +} + +func TestSnapshotRoundTrip(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + mustCreate(t, m, "c1") + + if _, err := m.CreateTopic(ctx, &mkdriver.Topic{ + Project: proj, Location: region, ClusterID: "c1", ID: "t", PartitionCount: 1, ReplicationFactor: 1, + }); err != nil { + t.Fatalf("topic: %v", err) + } + + snap, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + + restored, _ := newMock(t) + if err := restored.Restore(ctx, snap); err != nil { + t.Fatalf("restore: %v", err) + } + + if _, err := restored.GetTopic(ctx, proj, region, "c1", "t"); err != nil { + t.Fatalf("topic after restore: %v", err) + } + + if restored.opSeq.Load() != m.opSeq.Load() { + t.Fatalf("opSeq = %d, want %d", restored.opSeq.Load(), m.opSeq.Load()) + } +} diff --git a/providers/gcp/managedkafka/snapshot.go b/providers/gcp/managedkafka/snapshot.go new file mode 100644 index 000000000..b0fbc680b --- /dev/null +++ b/providers/gcp/managedkafka/snapshot.go @@ -0,0 +1,90 @@ +package managedkafka + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/stackshy/cloudemu/v2/internal/snapshot" +) + +var _ snapshot.Snapshottable = (*Mock)(nil) + +// managedkafkaSnapshot is the full serialized state of the Managed Kafka mock. +// Every store holds fully-exported mkdriver value types keyed by their full GCP +// resource name, so each round-trips through the generic memstore helper. opSeq +// is the operation-name counter, captured beside the stores so restored +// operation ids do not collide with fresh ones. The wired deps (m.opts) and the +// RWMutex are intentionally not serialized. +type managedkafkaSnapshot struct { + Clusters json.RawMessage `json:"clusters,omitempty"` + Topics json.RawMessage `json:"topics,omitempty"` + Operations json.RawMessage `json:"operations,omitempty"` + OpSeq uint64 `json:"opSeq,omitempty"` +} + +// Snapshot captures every cluster, topic and operation as JSON. includeAssets is +// unused: Managed Kafka is control-plane only and holds no bulk object bodies. +func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var snap managedkafkaSnapshot + + dumps := []struct { + dst *json.RawMessage + fn func() ([]byte, error) + }{ + {&snap.Clusters, m.clusters.Snapshot}, + {&snap.Topics, m.topics.Snapshot}, + {&snap.Operations, m.operations.Snapshot}, + } + + for _, d := range dumps { + b, err := d.fn() + if err != nil { + return nil, fmt.Errorf("managedkafka: snapshot store: %w", err) + } + + *d.dst = b + } + + snap.OpSeq = m.opSeq.Load() + + return json.Marshal(snap) +} + +// Restore rebuilds every cluster, topic and operation under its original +// resource name. +func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { + var snap managedkafkaSnapshot + if err := json.Unmarshal(data, &snap); err != nil { + return fmt.Errorf("managedkafka: parse snapshot: %w", err) + } + + m.mu.Lock() + defer m.mu.Unlock() + + loads := []struct { + src json.RawMessage + fn func([]byte) error + }{ + {snap.Clusters, m.clusters.LoadSnapshot}, + {snap.Topics, m.topics.LoadSnapshot}, + {snap.Operations, m.operations.LoadSnapshot}, + } + + for _, l := range loads { + if len(l.src) == 0 { + continue + } + + if err := l.fn(l.src); err != nil { + return fmt.Errorf("managedkafka: restore store: %w", err) + } + } + + m.opSeq.Store(snap.OpSeq) + + return nil +} diff --git a/providers/gcp/managedkafka/topics.go b/providers/gcp/managedkafka/topics.go new file mode 100644 index 000000000..2062f4eb7 --- /dev/null +++ b/providers/gcp/managedkafka/topics.go @@ -0,0 +1,142 @@ +package managedkafka + +import ( + "context" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +// CreateTopic validates and stores a new topic under an existing cluster. Topic +// creation is synchronous in the real API (it returns the Topic, not an LRO). +func (m *Mock) CreateTopic(_ context.Context, t *mkdriver.Topic) (*mkdriver.Topic, error) { + if err := validateTopicID(t.ID); err != nil { + return nil, err + } + + if err := validateTopic(t); err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + if !m.clusters.Has(clusterName(t.Project, t.Location, t.ClusterID)) { + return nil, clusterNotFound(t.Project, t.Location, t.ClusterID) + } + + key := topicName(t.Project, t.Location, t.ClusterID, t.ID) + if m.topics.Has(key) { + return nil, cerrors.Newf(cerrors.AlreadyExists, "topic %q already exists", key) + } + + stored := cloneTopic(t) + m.topics.Set(key, stored) + + out := cloneTopic(&stored) + + return &out, nil +} + +// GetTopic returns a topic by identity, cloned. A missing parent cluster is +// NOT_FOUND on the cluster. +func (m *Mock) GetTopic(_ context.Context, project, location, clusterID, id string) (*mkdriver.Topic, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + if !m.clusters.Has(clusterName(project, location, clusterID)) { + return nil, clusterNotFound(project, location, clusterID) + } + + t, ok := m.topics.Get(topicName(project, location, clusterID, id)) + if !ok { + return nil, topicNotFound(project, location, clusterID, id) + } + + out := cloneTopic(&t) + + return &out, nil +} + +// ListTopics returns every topic in a cluster, ordered by name. +func (m *Mock) ListTopics(_ context.Context, project, location, clusterID string) ([]mkdriver.Topic, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + parent := clusterName(project, location, clusterID) + if !m.clusters.Has(parent) { + return nil, clusterNotFound(project, location, clusterID) + } + + prefix := parent + "/" + topicsColl + "/" + all := m.topics.SortedValues() + out := make([]mkdriver.Topic, 0, len(all)) + + for i := range all { + if strings.HasPrefix(topicName(all[i].Project, all[i].Location, all[i].ClusterID, all[i].ID), prefix) { + out = append(out, cloneTopic(&all[i])) + } + } + + return out, nil +} + +// UpdateTopic applies the masked fields of t to the stored topic. partitionCount +// may only increase; replicationFactor is immutable. +func (m *Mock) UpdateTopic(_ context.Context, t *mkdriver.Topic, mask []string) (*mkdriver.Topic, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if !m.clusters.Has(clusterName(t.Project, t.Location, t.ClusterID)) { + return nil, clusterNotFound(t.Project, t.Location, t.ClusterID) + } + + key := topicName(t.Project, t.Location, t.ClusterID, t.ID) + + stored, ok := m.topics.Get(key) + if !ok { + return nil, topicNotFound(t.Project, t.Location, t.ClusterID, t.ID) + } + + next := cloneTopic(&stored) + if err := applyTopicMask(&next, t, mask); err != nil { + return nil, err + } + + if next.PartitionCount < stored.PartitionCount { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "partition_count can only be increased (current %d, requested %d)", stored.PartitionCount, next.PartitionCount) + } + + if err := validateTopic(&next); err != nil { + return nil, err + } + + m.topics.Set(key, next) + + out := cloneTopic(&next) + + return &out, nil +} + +// DeleteTopic removes a topic. The real API returns Empty synchronously. +func (m *Mock) DeleteTopic(_ context.Context, project, location, clusterID, id string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if !m.clusters.Has(clusterName(project, location, clusterID)) { + return clusterNotFound(project, location, clusterID) + } + + if !m.topics.Delete(topicName(project, location, clusterID, id)) { + return topicNotFound(project, location, clusterID, id) + } + + return nil +} + +// topicNotFound builds the NOT_FOUND error carrying the full resource name. +func topicNotFound(project, location, clusterID, id string) error { + return cerrors.Newf(cerrors.NotFound, "topic %q not found", topicName(project, location, clusterID, id)) +} diff --git a/providers/gcp/managedkafka/validate.go b/providers/gcp/managedkafka/validate.go new file mode 100644 index 000000000..ff87de6f9 --- /dev/null +++ b/providers/gcp/managedkafka/validate.go @@ -0,0 +1,300 @@ +package managedkafka + +import ( + "math" + "regexp" + "strings" + "unicode" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +// Real-service limits, as documented on the google.golang.org/api/managedkafka/v1 +// CapacityConfig / AccessConfig / ClustersCreateCall types. +const ( + minVcpuCount = 3 + + gib = int64(1) << 30 + + // minBytesPerVcpu / maxBytesPerVcpu bound the vCPU:GiB ratio to 1:1..1:8. + minBytesPerVcpu = 1 * gib + maxBytesPerVcpu = 8 * gib + + minNetworkConfigs = 1 + maxNetworkConfigs = 10 + + // maxTopicIDLen is Apache Kafka's own topic-name length limit. + maxTopicIDLen = 249 + + maskAll = "*" + + pathCapacity = "capacityConfig" + pathRebalance = "rebalanceConfig" + pathLabels = "labels" + pathName = "name" + + rebalanceNone = "NO_REBALANCE" + rebalanceOnScaleUp = "AUTO_REBALANCE_ON_SCALE_UP" + subnetPathSegments = 6 // projects/{p}/regions/{r}/subnetworks/{s} + subnetProjectsIdx = 0 + subnetRegionsIdx = 2 + subnetSubnetworksIdx = 4 +) + +var ( + // clusterIDPattern is the RFC 1035 label the real API enforces (1-63 chars). + clusterIDPattern = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + + // topicIDPattern is Apache Kafka's legal topic-name alphabet. + topicIDPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) +) + +// validateClusterID enforces the clusterId format. +func validateClusterID(id string) error { + if id == "" { + return cerrors.New(cerrors.InvalidArgument, "cluster_id is required") + } + + if !clusterIDPattern.MatchString(id) { + return cerrors.Newf(cerrors.InvalidArgument, + "cluster_id %q must be 1-63 characters and match [a-z]([-a-z0-9]*[a-z0-9])?", id) + } + + return nil +} + +// validateTopicID enforces the topicId format. +func validateTopicID(id string) error { + if id == "" { + return cerrors.New(cerrors.InvalidArgument, "topic_id is required") + } + + if len(id) > maxTopicIDLen || !topicIDPattern.MatchString(id) || id == "." || id == ".." { + return cerrors.Newf(cerrors.InvalidArgument, "topic_id %q is not a valid Kafka topic name", id) + } + + return nil +} + +// validateCluster checks the required cluster configuration: capacity, network +// access, and the rebalance mode enum. +func validateCluster(c *mkdriver.Cluster) error { + if err := validateCapacity(c.VcpuCount, c.MemoryBytes); err != nil { + return err + } + + if err := validateSubnets(c.Subnets); err != nil { + return err + } + + switch c.RebalanceMode { + case "", rebalanceNone, rebalanceOnScaleUp: + return nil + default: + return cerrors.Newf(cerrors.InvalidArgument, "rebalance_config.mode %q is not a valid mode", c.RebalanceMode) + } +} + +// validateCapacity enforces vcpuCount >= 3 and 1 GiB..8 GiB of memory per vCPU +// (inclusive). +func validateCapacity(vcpu, memory int64) error { + if vcpu < minVcpuCount { + return cerrors.Newf(cerrors.InvalidArgument, + "capacity_config.vcpu_count must be at least %d, got %d", minVcpuCount, vcpu) + } + + if vcpu > math.MaxInt64/maxBytesPerVcpu { + return cerrors.Newf(cerrors.InvalidArgument, "capacity_config.vcpu_count %d is too large", vcpu) + } + + if memory < vcpu*minBytesPerVcpu || memory > vcpu*maxBytesPerVcpu { + return cerrors.Newf(cerrors.InvalidArgument, + "capacity_config.memory_bytes must be between 1 GiB and 8 GiB per vCPU (%d..%d for %d vCPUs), got %d", + vcpu*minBytesPerVcpu, vcpu*maxBytesPerVcpu, vcpu, memory) + } + + return nil +} + +// validateSubnets requires 1..10 network configs, each naming a subnet as +// projects/{project}/regions/{region}/subnetworks/{subnet}. +func validateSubnets(subnets []string) error { + if len(subnets) < minNetworkConfigs || len(subnets) > maxNetworkConfigs { + return cerrors.Newf(cerrors.InvalidArgument, + "gcp_config.access_config.network_configs must contain %d to %d entries, got %d", + minNetworkConfigs, maxNetworkConfigs, len(subnets)) + } + + for _, s := range subnets { + if !validSubnet(s) { + return cerrors.Newf(cerrors.InvalidArgument, + "network_configs.subnet %q must be projects/{project}/regions/{region}/subnetworks/{subnet}", s) + } + } + + return nil +} + +// validSubnet reports whether s is a well-formed subnetwork resource name. +func validSubnet(s string) bool { + parts := strings.Split(s, "/") + if len(parts) != subnetPathSegments { + return false + } + + for _, p := range parts { + if p == "" { + return false + } + } + + return parts[subnetProjectsIdx] == "projects" && parts[subnetRegionsIdx] == "regions" && + parts[subnetSubnetworksIdx] == "subnetworks" +} + +// validateTopic requires positive partition and replication counts. +func validateTopic(t *mkdriver.Topic) error { + if t.PartitionCount <= 0 { + return cerrors.Newf(cerrors.InvalidArgument, "partition_count must be greater than 0, got %d", t.PartitionCount) + } + + if t.ReplicationFactor <= 0 { + return cerrors.Newf(cerrors.InvalidArgument, "replication_factor must be greater than 0, got %d", t.ReplicationFactor) + } + + return nil +} + +// clusterMaskAppliers maps each mutable cluster field-mask path (camelCase) to +// the copy it performs from the request onto the stored cluster. +// +//nolint:gochecknoglobals // immutable lookup table +var clusterMaskAppliers = map[string]func(dst, src *mkdriver.Cluster){ + pathCapacity: func(dst, src *mkdriver.Cluster) { + dst.VcpuCount, dst.MemoryBytes = src.VcpuCount, src.MemoryBytes + }, + "capacityConfig.vcpuCount": func(dst, src *mkdriver.Cluster) { dst.VcpuCount = src.VcpuCount }, + "capacityConfig.memoryBytes": func(dst, src *mkdriver.Cluster) { dst.MemoryBytes = src.MemoryBytes }, + "gcpConfig.accessConfig": copySubnets, + "gcpConfig.accessConfig.networkConfigs": copySubnets, + pathRebalance: func(dst, src *mkdriver.Cluster) { dst.RebalanceMode = src.RebalanceMode }, + "rebalanceConfig.mode": func(dst, src *mkdriver.Cluster) { dst.RebalanceMode = src.RebalanceMode }, + pathLabels: func(dst, src *mkdriver.Cluster) { dst.Labels = cloneStringMap(src.Labels) }, +} + +// clusterFixedPaths are cluster field-mask paths that exist on the resource but +// cannot be updated (immutable or output-only). +// +//nolint:gochecknoglobals // immutable lookup set +var clusterFixedPaths = map[string]bool{ + pathName: true, "state": true, "createTime": true, "updateTime": true, + "satisfiesPzi": true, "satisfiesPzs": true, "gcpConfig.kmsKey": true, + "brokerDetails": true, "kafkaVersion": true, +} + +func copySubnets(dst, src *mkdriver.Cluster) { dst.Subnets = append([]string(nil), src.Subnets...) } + +// applyClusterMask copies the masked fields of src onto dst. The mask is +// required; "*" updates every mutable field. "gcpConfig" updates the access +// config and rejects a kmsKey change (kmsKey is immutable). +func applyClusterMask(dst, src *mkdriver.Cluster, mask []string) error { + if len(mask) == 0 { + return cerrors.New(cerrors.InvalidArgument, "update_mask is required") + } + + for _, raw := range mask { + path := camelPath(raw) + + switch { + case path == maskAll || path == "gcpConfig": + if src.KmsKey != "" && src.KmsKey != dst.KmsKey { + return cerrors.New(cerrors.InvalidArgument, "gcp_config.kms_key is immutable") + } + + applyAllCluster(dst, src, path) + case clusterMaskAppliers[path] != nil: + clusterMaskAppliers[path](dst, src) + case clusterFixedPaths[path]: + return cerrors.Newf(cerrors.InvalidArgument, "field %q in update_mask is immutable or output only", raw) + default: + return cerrors.Newf(cerrors.InvalidArgument, "unknown field %q in update_mask", raw) + } + } + + return nil +} + +// applyAllCluster applies the "*" mask (every mutable field) or the "gcpConfig" +// mask (the access config only). +func applyAllCluster(dst, src *mkdriver.Cluster, path string) { + copySubnets(dst, src) + + if path != maskAll { + return + } + + for _, p := range []string{pathCapacity, pathRebalance, pathLabels} { + clusterMaskAppliers[p](dst, src) + } +} + +// applyTopicMask copies the masked fields of src onto dst. The mask is +// required; "*" updates every mutable field (partitionCount, configs), and +// rejects a replicationFactor change. +func applyTopicMask(dst, src *mkdriver.Topic, mask []string) error { + if len(mask) == 0 { + return cerrors.New(cerrors.InvalidArgument, "update_mask is required") + } + + for _, raw := range mask { + switch camelPath(raw) { + case maskAll: + if src.ReplicationFactor != 0 && src.ReplicationFactor != dst.ReplicationFactor { + return cerrors.New(cerrors.InvalidArgument, "replication_factor is immutable") + } + + dst.PartitionCount = src.PartitionCount + dst.Configs = cloneStringMap(src.Configs) + case "partitionCount": + dst.PartitionCount = src.PartitionCount + case "configs": + dst.Configs = cloneStringMap(src.Configs) + case "replicationFactor", pathName: + return cerrors.Newf(cerrors.InvalidArgument, "field %q in update_mask is immutable", raw) + default: + return cerrors.Newf(cerrors.InvalidArgument, "unknown field %q in update_mask", raw) + } + } + + return nil +} + +// camelPath converts a snake_case field-mask path (the proto form) to the +// camelCase JSON form, so both spellings are accepted. +func camelPath(p string) string { + p = strings.TrimSpace(p) + if !strings.Contains(p, "_") { + return p + } + + var b strings.Builder + + upper := false + + for _, r := range p { + if r == '_' { + upper = true + continue + } + + if upper { + r = unicode.ToUpper(r) + upper = false + } + + b.WriteRune(r) + } + + return b.String() +} diff --git a/server/gcp/from_provider.go b/server/gcp/from_provider.go index 77bcbedab..c06bc32e4 100644 --- a/server/gcp/from_provider.go +++ b/server/gcp/from_provider.go @@ -35,6 +35,7 @@ func DriversFrom(p *gcpprovider.Provider) Drivers { Metastore: p.Metastore, VPCAccess: p.VPCAccess, CloudIDS: p.CloudIDS, + ManagedKafka: p.ManagedKafka, SecureSourceManager: p.SecureSourceManager, NetworkConnectivity: p.NetworkConnectivity, Composer: p.Composer, diff --git a/server/gcp/gcp.go b/server/gcp/gcp.go index 2c6c7b6b3..d154a867e 100644 --- a/server/gcp/gcp.go +++ b/server/gcp/gcp.go @@ -51,6 +51,7 @@ import ( kmssrv "github.com/stackshy/cloudemu/v2/server/gcp/kms" lbsrv "github.com/stackshy/cloudemu/v2/server/gcp/loadbalancer" "github.com/stackshy/cloudemu/v2/server/gcp/lro" + managedkafkasrv "github.com/stackshy/cloudemu/v2/server/gcp/managedkafka" memorystoresrv "github.com/stackshy/cloudemu/v2/server/gcp/memorystore" metastoresrv "github.com/stackshy/cloudemu/v2/server/gcp/metastore" "github.com/stackshy/cloudemu/v2/server/gcp/monitoring" @@ -98,6 +99,7 @@ import ( "github.com/stackshy/cloudemu/v2/services/kubernetes" lbdriver "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" logdriver "github.com/stackshy/cloudemu/v2/services/logging/driver" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" mqdriver "github.com/stackshy/cloudemu/v2/services/messagequeue/driver" metastoredriver "github.com/stackshy/cloudemu/v2/services/metastore/driver" mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" @@ -238,6 +240,13 @@ type Drivers struct { // /v1/projects/ handler, and its location-scoped operation polls are owned by // the shared LRO poller. CloudIDS cloudidsdriver.CloudIDs + // ManagedKafka serves the managedkafka.googleapis.com v1 Managed Service for + // Apache Kafka cluster + topic control plane against the managedkafka driver. + // Its /v1/projects/{p}/locations/{l}/clusters[/…] paths are identical to + // GKE's and AlloyDB's, so the handler registers ahead of both and claims only + // genuinely-Kafka traffic (content+ownership); its location-scoped operation + // polls are owned by the shared LRO poller. + ManagedKafka mkdriver.ManagedKafka // SecureSourceManager serves the securesourcemanager.googleapis.com v1 // instance + repository control plane against the securesourcemanager driver. // Its paths live under /v1/projects/{p}/locations/{l}/{instances|repositories} @@ -381,6 +390,20 @@ func New(d Drivers) *server.Server { srv := server.New() + // Managed Kafka shares the exact /v1/projects/{p}/locations/{l}/clusters[/…] + // grammar with GKE and AlloyDB (all greedy on that collection), so it + // registers AHEAD of both and its Matches claims only genuinely-Kafka traffic: + // a create body carrying capacityConfig/gcpConfig, an item/list it owns, or + // the Kafka-only clusters/{c}/topics sub-collection. Everything else falls + // through. Its registry is wired below, once the shared LRO poller exists, + // which also makes it yield location operation polls to that poller. + var kafkaH *managedkafkasrv.Handler + + if d.ManagedKafka != nil { + kafkaH = managedkafkasrv.New(d.ManagedKafka) + srv.Register(kafkaH) + } + // GKE registers ahead of the shared LRO poller because it answers a richer // operation shape (operationType/targetLink/selfLink/zone/timestamps) for // its OWN operations. Its Matches claims a named operation poll only when @@ -409,6 +432,10 @@ func New(d Drivers) *server.Server { opsReg := lro.NewRegistry() srv.Register(lro.New(opsReg)) + if kafkaH != nil { + kafkaH.SetOperationRegistry(opsReg) + } + // Shared compute-operation registry. The compute handler's /operations route // serves every compute#operation poll (its own, plus the networks and load- // balancing handlers', which mint compute operations but have no operations diff --git a/server/gcp/managedkafka/handler.go b/server/gcp/managedkafka/handler.go new file mode 100644 index 000000000..c709f72c2 --- /dev/null +++ b/server/gcp/managedkafka/handler.go @@ -0,0 +1,275 @@ +// Package managedkafka implements the Google Cloud Managed Service for Apache +// Kafka control plane (managedkafka.googleapis.com/v1) as a server.Handler. Real +// google.golang.org/api/managedkafka/v1 clients and the Terraform google +// provider's google_managed_kafka_cluster / google_managed_kafka_topic resources +// hit this handler unchanged. +// +// Coverage: +// +// POST /v1/…/clusters?clusterId= : CreateCluster (LRO) +// GET /v1/…/clusters : ListClusters +// GET /v1/…/clusters/{c} : GetCluster +// PATCH /v1/…/clusters/{c}?updateMask= : UpdateCluster (LRO) +// DELETE /v1/…/clusters/{c} : DeleteCluster (LRO) +// POST /v1/…/clusters/{c}/topics?topicId= : CreateTopic (sync, returns Topic) +// GET /v1/…/clusters/{c}/topics[/{t}] : List/GetTopic +// PATCH /v1/…/clusters/{c}/topics/{t}?updateMask= : UpdateTopic (sync, returns Topic) +// DELETE /v1/…/clusters/{c}/topics/{t} : DeleteTopic (sync, returns Empty) +// GET /v1/…/operations/{op} : Operations.Get (shared poller) +// +// Path sharing: /v1/projects/{p}/locations/{l}/clusters[/{c}] is byte-identical +// to GKE's (container/v1) and AlloyDB's cluster paths, and a custom-endpoint +// client sends the emulator's own Host, so URL alone cannot tell them apart. In +// an assembled server (a shared LRO registry is wired) this handler registers +// AHEAD of GKE/AlloyDB and claims a cluster request only when it is genuinely +// Managed Kafka traffic, the Filestore/Spanner content+ownership pattern: +// +// - a create whose body carries capacityConfig or gcpConfig (GKE wraps its +// body in {"cluster": …}; AlloyDB bodies carry neither key); +// - an item request for a cluster this store owns; +// - a list in a project+location where this store owns at least one cluster; +// - anything under clusters/{c}/topics (no sibling service has topics). +// +// Everything else falls through to GKE/AlloyDB. Operation polls are yielded to +// the shared LRO poller. A standalone package server (no registry) claims every +// clusters/topics/operations path. +package managedkafka + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "slices" + "strings" + + "github.com/stackshy/cloudemu/v2/server/gcp/lro" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +const ( + pathPrefix = "/v1/projects/" + projectsSeg = "projects" + locationsSeg = "locations" + operationsSeg = "operations" + clustersSeg = "clusters" + topicsSeg = "topics" + + scopeParts = 4 // [projects, {p}, locations, {l}] + + // rest-segment counts after the location scope. + restCollection = 1 // [clusters] + restItem = 2 // [clusters, {c}] + restTopics = 3 // [clusters, {c}, topics] + restTopic = 4 // [clusters, {c}, topics, {t}] + + maxProbeBytes = 1 << 20 +) + +// Handler serves managedkafka.googleapis.com v1 requests against a ManagedKafka +// driver. +type Handler struct { + db mkdriver.ManagedKafka + + // ops records created operations with the shared poller. Nil in a standalone + // package server, where this handler serves its own /operations/ poll and + // claims every clusters path. + ops *lro.Registry +} + +// New returns a Managed Kafka handler backed by db. +func New(db mkdriver.ManagedKafka) *Handler { return &Handler{db: db} } + +// SetOperationRegistry wires the shared LRO poller so created operations are +// resolvable (with their response) through the full server's operations route, +// and switches Matches to the content+ownership mode an assembled server needs. +func (h *Handler) SetOperationRegistry(reg *lro.Registry) { h.ops = reg } + +// route holds the parsed components of a Managed Kafka v1 path. +type route struct { + project string + location string + resource string // clusters | operations + cluster string // cluster id, or operation id for an operations route + topics bool // path is under clusters/{c}/topics + topic string // topic id; empty for the topics collection +} + +// parseRoute extracts the components of a Managed Kafka v1 path. It recognizes +// only the clusters (with nested topics) and operations resources under a +// locations scope; a custom verb (clusters/{c}:promote, …) is not ours. +func parseRoute(urlPath string) (route, bool) { + if !strings.HasPrefix(urlPath, pathPrefix) || strings.Contains(urlPath, ":") { + return route{}, false + } + + parts := strings.Split(strings.TrimPrefix(urlPath, "/v1/"), "/") + if len(parts) <= scopeParts || parts[0] != projectsSeg || parts[2] != locationsSeg || slices.Contains(parts, "") { + return route{}, false + } + + rt := route{project: parts[1], location: parts[3]} + if !rt.setRest(parts[scopeParts:]) { + return route{}, false + } + + return rt, true +} + +// setRest fills the resource components from the segments after the location +// scope: operations[/{op}] or clusters[/{c}[/topics[/{t}]]]. +func (rt *route) setRest(rest []string) bool { + switch { + case rest[0] == operationsSeg && len(rest) <= restItem: + case rest[0] == clustersSeg && len(rest) < restTopics: + case rest[0] == clustersSeg && len(rest) <= restTopic && rest[2] == topicsSeg: + rt.topics = true + default: + return false + } + + rt.resource = rest[0] + + if len(rest) >= restItem { + rt.cluster = rest[1] + } + + if len(rest) == restTopic { + rt.topic = rest[3] + } + + return true +} + +// Matches claims Managed Kafka paths. See the package doc for how it shares the +// clusters path with GKE and AlloyDB in an assembled server. +func (h *Handler) Matches(r *http.Request) bool { + rt, ok := parseRoute(r.URL.Path) + if !ok { + return false + } + + standalone := h.ops == nil + + switch { + case rt.resource == operationsSeg: + return standalone + case standalone || rt.topics: + return true + case rt.cluster != "": + _, err := h.db.GetCluster(r.Context(), rt.project, rt.location, rt.cluster) + + return err == nil + case r.Method == http.MethodPost: + return bodyLooksLikeKafka(r) + case r.Method == http.MethodGet: + all, err := h.db.ListClusters(r.Context(), rt.project, rt.location) + + return err == nil && len(all) > 0 + default: + return false + } +} + +// bodyLooksLikeKafka reports whether a POST .../clusters body is a Managed Kafka +// Cluster (it carries capacityConfig or gcpConfig) rather than a GKE +// CreateClusterRequest or an AlloyDB Cluster. It reads and restores the body so +// a fall-through handler still sees the full request. +func bodyLooksLikeKafka(r *http.Request) bool { + if r.Body == nil { + return false + } + + raw, err := io.ReadAll(io.LimitReader(r.Body, maxProbeBytes)) + _ = r.Body.Close() + r.Body = io.NopCloser(bytes.NewReader(raw)) + + if err != nil { + return false + } + + var probe map[string]json.RawMessage + if json.Unmarshal(raw, &probe) != nil { + return false + } + + _, hasCapacity := probe["capacityConfig"] + _, hasGcp := probe["gcpConfig"] + + return hasCapacity || hasGcp +} + +// ServeHTTP routes on the parsed path and method. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rt, ok := parseRoute(r.URL.Path) + if !ok { + gcprest.WriteError(w, http.StatusNotFound, "notFound", "unrecognized Managed Kafka path") + return + } + + switch { + case rt.resource == operationsSeg: + h.serveOperation(w, r) + case rt.topics && rt.topic == "": + h.serveTopicCollection(w, r, &rt) + case rt.topics: + h.serveTopicItem(w, r, &rt) + case rt.cluster == "": + h.serveClusterCollection(w, r, &rt) + default: + h.serveClusterItem(w, r, &rt) + } +} + +func (h *Handler) serveClusterCollection(w http.ResponseWriter, r *http.Request, rt *route) { + switch r.Method { + case http.MethodPost: + h.createCluster(w, r, rt) + case http.MethodGet: + h.listClusters(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +func (h *Handler) serveClusterItem(w http.ResponseWriter, r *http.Request, rt *route) { + switch r.Method { + case http.MethodGet: + h.getCluster(w, r, rt) + case http.MethodPatch: + h.updateCluster(w, r, rt) + case http.MethodDelete: + h.deleteCluster(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +func (h *Handler) serveTopicCollection(w http.ResponseWriter, r *http.Request, rt *route) { + switch r.Method { + case http.MethodPost: + h.createTopic(w, r, rt) + case http.MethodGet: + h.listTopics(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +func (h *Handler) serveTopicItem(w http.ResponseWriter, r *http.Request, rt *route) { + switch r.Method { + case http.MethodGet: + h.getTopic(w, r, rt) + case http.MethodPatch: + h.updateTopic(w, r, rt) + case http.MethodDelete: + h.deleteTopic(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +func writeMethodNotAllowed(w http.ResponseWriter) { + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") +} diff --git a/server/gcp/managedkafka/matches_test.go b/server/gcp/managedkafka/matches_test.go new file mode 100644 index 000000000..09e6ce147 --- /dev/null +++ b/server/gcp/managedkafka/matches_test.go @@ -0,0 +1,102 @@ +package managedkafka + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + mkprovider "github.com/stackshy/cloudemu/v2/providers/gcp/managedkafka" + "github.com/stackshy/cloudemu/v2/server/gcp/lro" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +const loc = "/v1/projects/p/locations/us-central1" + +func sharedHandler(t *testing.T) *Handler { + t.Helper() + + mock := mkprovider.New(config.NewOptions(config.WithProjectID("p"))) + + _, _, err := mock.CreateCluster(context.Background(), &mkdriver.Cluster{ + Project: "p", Location: "us-central1", ID: "owned", + VcpuCount: 3, MemoryBytes: 3 << 30, + Subnets: []string{"projects/p/regions/us-central1/subnetworks/s"}, + }) + if err != nil { + t.Fatalf("seed cluster: %v", err) + } + + h := New(mock) + h.SetOperationRegistry(lro.NewRegistry()) + + return h +} + +func request(method, path, body string) *http.Request { + r, _ := http.NewRequest(method, "http://x"+path, strings.NewReader(body)) + + return r +} + +func TestMatchesSharedClustersPath(t *testing.T) { + h := sharedHandler(t) + + cases := []struct { + name, method, path, body string + want bool + }{ + {"kafka create body", http.MethodPost, loc + "/clusters?clusterId=k", `{"capacityConfig":{}}`, true}, + {"kafka create gcpConfig only", http.MethodPost, loc + "/clusters?clusterId=k", `{"gcpConfig":{}}`, true}, + {"gke create body", http.MethodPost, loc + "/clusters", `{"cluster":{"name":"g"}}`, false}, + {"alloydb create body", http.MethodPost, loc + "/clusters?clusterId=a", `{"network":"n"}`, false}, + {"owned item", http.MethodGet, loc + "/clusters/owned", "", true}, + {"foreign item", http.MethodGet, loc + "/clusters/gke", "", false}, + {"list where owned", http.MethodGet, loc + "/clusters", "", true}, + {"list elsewhere", http.MethodGet, "/v1/projects/p/locations/europe-west1/clusters", "", false}, + {"topics under any cluster", http.MethodGet, loc + "/clusters/gke/topics", "", true}, + {"topic item", http.MethodDelete, loc + "/clusters/owned/topics/t", "", true}, + {"gke nodePools", http.MethodGet, loc + "/clusters/owned/nodePools", "", false}, + {"gke custom verb", http.MethodPost, loc + "/clusters/owned:setLogging", "", false}, + {"operation poll yields to lro", http.MethodGet, loc + "/operations/op-1", "", false}, + {"other collection", http.MethodGet, loc + "/instances", "", false}, + {"dataproc regions", http.MethodGet, "/v1/projects/p/regions/us-central1/clusters", "", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := h.Matches(request(tc.method, tc.path, tc.body)); got != tc.want { + t.Fatalf("Matches(%s %s) = %v, want %v", tc.method, tc.path, got, tc.want) + } + }) + } +} + +// TestMatchesRestoresProbedBody guards the fall-through: a GKE create body that +// Kafka probed must still be fully readable by the next handler. +func TestMatchesRestoresProbedBody(t *testing.T) { + h := sharedHandler(t) + body := `{"cluster":{"name":"g"}}` + r := request(http.MethodPost, loc+"/clusters", body) + + if h.Matches(r) { + t.Fatalf("GKE body claimed by Kafka") + } + + got, err := io.ReadAll(r.Body) + if err != nil || string(got) != body { + t.Fatalf("body after probe = %q (err %v), want %q", got, err, body) + } +} + +func TestMatchesStandaloneClaimsEverything(t *testing.T) { + h := New(mkprovider.New(config.NewOptions(config.WithProjectID("p")))) + + for _, path := range []string{loc + "/clusters", loc + "/clusters/any", loc + "/operations/op-1"} { + if !h.Matches(request(http.MethodGet, path, "")) { + t.Fatalf("standalone handler should claim %s", path) + } + } +} diff --git a/server/gcp/managedkafka/operations.go b/server/gcp/managedkafka/operations.go new file mode 100644 index 000000000..c5bda89ba --- /dev/null +++ b/server/gcp/managedkafka/operations.go @@ -0,0 +1,224 @@ +package managedkafka + +import ( + "net/http" + "strconv" + "strings" + + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +const ( + clusterIDParam = "clusterId" + topicIDParam = "topicId" + + nextPageTokenKey = "nextPageToken" + + defaultPageSize = 500 + maxPageSize = 500 +) + +// createCluster handles POST .../clusters?clusterId=. Validation (clusterId +// format, capacity, network configs) lives in the driver; a failure is 400. +func (h *Handler) createCluster(w http.ResponseWriter, r *http.Request, rt *route) { + var body clusterJSON + if !decodeBody(w, r, &body) { + return + } + + c, op, err := h.db.CreateCluster(r.Context(), toDriverCluster(&body, rt, r.URL.Query().Get(clusterIDParam))) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeOperation(w, op, fromDriverCluster(c), clusterTypeURL) +} + +func (h *Handler) getCluster(w http.ResponseWriter, r *http.Request, rt *route) { + c, err := h.db.GetCluster(r.Context(), rt.project, rt.location, rt.cluster) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, fromDriverCluster(c)) +} + +// listClusters handles GET .../clusters with pageToken/pageSize. +func (h *Handler) listClusters(w http.ResponseWriter, r *http.Request, rt *route) { + all, err := h.db.ListClusters(r.Context(), rt.project, rt.location) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + page, err := pagination.PaginateSorted(all, + func(a, b mkdriver.Cluster) bool { return a.ID < b.ID }, + r.URL.Query().Get("pageToken"), pageSize(r)) + if err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "invalid pageToken") + return + } + + items := make([]clusterJSON, 0, len(page.Items)) + for i := range page.Items { + items = append(items, fromDriverCluster(&page.Items[i])) + } + + gcprest.WriteJSON(w, http.StatusOK, map[string]any{ + "clusters": items, + nextPageTokenKey: page.NextPageToken, + }) +} + +// updateCluster handles PATCH .../clusters/{c}?updateMask=. Only masked fields +// change; unknown, immutable and output-only paths are 400. +func (h *Handler) updateCluster(w http.ResponseWriter, r *http.Request, rt *route) { + var body clusterJSON + if !decodeBody(w, r, &body) { + return + } + + c, op, err := h.db.UpdateCluster(r.Context(), toDriverCluster(&body, rt, rt.cluster), + parseMask(r.URL.Query().Get("updateMask"))) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeOperation(w, op, fromDriverCluster(c), clusterTypeURL) +} + +// deleteCluster handles DELETE .../clusters/{c}; the LRO response is Empty. +func (h *Handler) deleteCluster(w http.ResponseWriter, r *http.Request, rt *route) { + op, err := h.db.DeleteCluster(r.Context(), rt.project, rt.location, rt.cluster) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeOperation(w, op, struct{}{}, emptyTypeURL) +} + +// createTopic handles POST .../topics?topicId=; synchronous, returns the Topic. +func (h *Handler) createTopic(w http.ResponseWriter, r *http.Request, rt *route) { + var body topicJSON + if !decodeBody(w, r, &body) { + return + } + + t, err := h.db.CreateTopic(r.Context(), toDriverTopic(&body, rt, r.URL.Query().Get(topicIDParam))) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, fromDriverTopic(t)) +} + +func (h *Handler) getTopic(w http.ResponseWriter, r *http.Request, rt *route) { + t, err := h.db.GetTopic(r.Context(), rt.project, rt.location, rt.cluster, rt.topic) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, fromDriverTopic(t)) +} + +// listTopics handles GET .../topics with pageToken/pageSize. +func (h *Handler) listTopics(w http.ResponseWriter, r *http.Request, rt *route) { + all, err := h.db.ListTopics(r.Context(), rt.project, rt.location, rt.cluster) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + page, err := pagination.PaginateSorted(all, + func(a, b mkdriver.Topic) bool { return a.ID < b.ID }, + r.URL.Query().Get("pageToken"), pageSize(r)) + if err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "invalid pageToken") + return + } + + items := make([]topicJSON, 0, len(page.Items)) + for i := range page.Items { + items = append(items, fromDriverTopic(&page.Items[i])) + } + + gcprest.WriteJSON(w, http.StatusOK, map[string]any{ + "topics": items, + nextPageTokenKey: page.NextPageToken, + }) +} + +// updateTopic handles PATCH .../topics/{t}?updateMask=; synchronous. +func (h *Handler) updateTopic(w http.ResponseWriter, r *http.Request, rt *route) { + var body topicJSON + if !decodeBody(w, r, &body) { + return + } + + t, err := h.db.UpdateTopic(r.Context(), toDriverTopic(&body, rt, rt.topic), + parseMask(r.URL.Query().Get("updateMask"))) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, fromDriverTopic(t)) +} + +// deleteTopic handles DELETE .../topics/{t}; synchronous, returns Empty. +func (h *Handler) deleteTopic(w http.ResponseWriter, r *http.Request, rt *route) { + if err := h.db.DeleteTopic(r.Context(), rt.project, rt.location, rt.cluster, rt.topic); err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, map[string]any{}) +} + +// serveOperation resolves a (done) operation poll for a standalone package +// server (no shared registry). +func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMethodNotAllowed(w) + return + } + + op, err := h.db.GetOperation(r.Context(), strings.TrimPrefix(r.URL.Path, "/v1/")) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true}) +} + +// parseMask splits a comma-separated updateMask query param into field paths. +func parseMask(raw string) []string { + var out []string + + for _, p := range strings.Split(raw, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + + return out +} + +// pageSize reads ?pageSize, clamping to a sane default and ceiling. +func pageSize(r *http.Request) int { + n, err := strconv.Atoi(r.URL.Query().Get("pageSize")) + if err != nil || n <= 0 { + return defaultPageSize + } + + return min(n, maxPageSize) +} diff --git a/server/gcp/managedkafka/sdk_roundtrip_test.go b/server/gcp/managedkafka/sdk_roundtrip_test.go new file mode 100644 index 000000000..9f746b042 --- /dev/null +++ b/server/gcp/managedkafka/sdk_roundtrip_test.go @@ -0,0 +1,421 @@ +package managedkafka_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "google.golang.org/api/googleapi" + mk "google.golang.org/api/managedkafka/v1" + "google.golang.org/api/option" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +const ( + project = "mock-project" + location = "us-central1" + parent = "projects/" + project + "/locations/" + location + subnet = "projects/" + project + "/regions/" + location + "/subnetworks/default" + gib = int64(1) << 30 + + maxPolls = 5 +) + +//nolint:gochecknoglobals // fixed test clock origin +var epoch = time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC) + +func newSDKClient(t *testing.T) (*mk.Service, *config.FakeClock) { + t.Helper() + + clock := config.NewFakeClock(epoch) + srv := gcpserver.NewFromProvider(cloudemu.NewGCP(config.WithClock(clock), config.WithProjectID(project))) + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + svc, err := mk.NewService(context.Background(), + option.WithEndpoint(ts.URL+"/"), + option.WithoutAuthentication(), + option.WithHTTPClient(ts.Client()), + ) + if err != nil { + t.Fatalf("managedkafka.NewService: %v", err) + } + + return svc, clock +} + +func validCluster() *mk.Cluster { + return &mk.Cluster{ + CapacityConfig: &mk.CapacityConfig{VcpuCount: 3, MemoryBytes: 3 * gib}, + GcpConfig: &mk.GcpConfig{ + AccessConfig: &mk.AccessConfig{NetworkConfigs: []*mk.NetworkConfig{{Subnet: subnet}}}, + KmsKey: "projects/" + project + "/locations/" + location + "/keyRings/kr/cryptoKeys/k", + }, + RebalanceConfig: &mk.RebalanceConfig{Mode: "AUTO_REBALANCE_ON_SCALE_UP"}, + Labels: map[string]string{"env": "test"}, + } +} + +// waitOp polls an operation through the shared LRO route until done, as a real +// client does, and fails if it never completes. +func waitOp(t *testing.T, svc *mk.Service, op *mk.Operation) *mk.Operation { + t.Helper() + + for range maxPolls { + got, err := svc.Projects.Locations.Operations.Get(op.Name).Do() + if err != nil { + t.Fatalf("Operations.Get(%s): %v", op.Name, err) + } + + if got.Done { + if got.Error != nil { + t.Fatalf("operation %s failed: %+v", op.Name, got.Error) + } + + return got + } + } + + t.Fatalf("operation %s never completed", op.Name) + + return nil +} + +func wantCode(t *testing.T, err error, code int, what string) { + t.Helper() + + var gerr *googleapi.Error + if !errors.As(err, &gerr) { + t.Fatalf("%s: want HTTP %d, got %v", what, code, err) + } + + if gerr.Code != code { + t.Fatalf("%s: HTTP %d (%s), want %d", what, gerr.Code, gerr.Message, code) + } +} + +func createCluster(t *testing.T, svc *mk.Service, id string) { + t.Helper() + + op, err := svc.Projects.Locations.Clusters.Create(parent, validCluster()).ClusterId(id).Do() + if err != nil { + t.Fatalf("Clusters.Create(%s): %v", id, err) + } + + waitOp(t, svc, op) +} + +func TestSDKClusterLifecycle(t *testing.T) { + svc, clock := newSDKClient(t) + name := parent + "/clusters/kc" + + op, err := svc.Projects.Locations.Clusters.Create(parent, validCluster()).ClusterId("kc").Do() + if err != nil { + t.Fatalf("Clusters.Create: %v", err) + } + + done := waitOp(t, svc, op) + if len(done.Response) == 0 { + t.Fatalf("create operation carries no response") + } + + got, err := svc.Projects.Locations.Clusters.Get(name).Do() + if err != nil { + t.Fatalf("Clusters.Get: %v", err) + } + + if got.Name != name || got.State != "ACTIVE" { + t.Fatalf("name/state = %q/%q", got.Name, got.State) + } + + if got.CapacityConfig.VcpuCount != 3 || got.CapacityConfig.MemoryBytes != 3*gib { + t.Fatalf("capacity = %+v", got.CapacityConfig) + } + + if got.GcpConfig.AccessConfig.NetworkConfigs[0].Subnet != subnet || got.GcpConfig.KmsKey == "" { + t.Fatalf("gcpConfig = %+v", got.GcpConfig) + } + + if got.RebalanceConfig.Mode != "AUTO_REBALANCE_ON_SCALE_UP" || got.Labels["env"] != "test" { + t.Fatalf("rebalance/labels = %+v %+v", got.RebalanceConfig, got.Labels) + } + + wantTime := epoch.Format(time.RFC3339Nano) + if got.CreateTime != wantTime || got.UpdateTime != wantTime { + t.Fatalf("times = %s/%s, want %s", got.CreateTime, got.UpdateTime, wantTime) + } + + // Masked patch: only memoryBytes changes; vcpuCount, labels, subnet untouched. + clock.Advance(time.Hour) + + patch := &mk.Cluster{ + CapacityConfig: &mk.CapacityConfig{VcpuCount: 99, MemoryBytes: 12 * gib}, + Labels: map[string]string{"ignored": "yes"}, + } + + pop, err := svc.Projects.Locations.Clusters.Patch(name, patch).UpdateMask("capacityConfig.memoryBytes").Do() + if err != nil { + t.Fatalf("Clusters.Patch: %v", err) + } + + waitOp(t, svc, pop) + + updated, err := svc.Projects.Locations.Clusters.Get(name).Do() + if err != nil { + t.Fatalf("Get after patch: %v", err) + } + + if updated.CapacityConfig.VcpuCount != 3 || updated.CapacityConfig.MemoryBytes != 12*gib { + t.Fatalf("capacity after masked patch = %+v", updated.CapacityConfig) + } + + if updated.Labels["env"] != "test" || updated.Labels["ignored"] != "" { + t.Fatalf("unmasked labels changed: %+v", updated.Labels) + } + + if updated.CreateTime != wantTime || updated.UpdateTime != epoch.Add(time.Hour).Format(time.RFC3339Nano) { + t.Fatalf("times after patch = %s/%s", updated.CreateTime, updated.UpdateTime) + } + + // Labels patch. + lop, err := svc.Projects.Locations.Clusters.Patch(name, &mk.Cluster{Labels: map[string]string{"team": "data"}}). + UpdateMask("labels").Do() + if err != nil { + t.Fatalf("labels patch: %v", err) + } + + waitOp(t, svc, lop) + + relabeled, err := svc.Projects.Locations.Clusters.Get(name).Do() + if err != nil { + t.Fatalf("Get after labels patch: %v", err) + } + + if len(relabeled.Labels) != 1 || relabeled.Labels["team"] != "data" { + t.Fatalf("labels after patch = %+v", relabeled.Labels) + } + + dop, err := svc.Projects.Locations.Clusters.Delete(name).Do() + if err != nil { + t.Fatalf("Clusters.Delete: %v", err) + } + + waitOp(t, svc, dop) + + _, err = svc.Projects.Locations.Clusters.Get(name).Do() + wantCode(t, err, http.StatusNotFound, "Get after delete") +} + +func TestSDKClusterListPagination(t *testing.T) { + svc, _ := newSDKClient(t) + + for _, id := range []string{"c-a", "c-b", "c-c"} { + createCluster(t, svc, id) + } + + first, err := svc.Projects.Locations.Clusters.List(parent).PageSize(2).Do() + if err != nil { + t.Fatalf("List page 1: %v", err) + } + + if len(first.Clusters) != 2 || first.NextPageToken == "" { + t.Fatalf("page 1 = %d clusters, token %q", len(first.Clusters), first.NextPageToken) + } + + second, err := svc.Projects.Locations.Clusters.List(parent).PageSize(2).PageToken(first.NextPageToken).Do() + if err != nil { + t.Fatalf("List page 2: %v", err) + } + + if len(second.Clusters) != 1 || second.NextPageToken != "" || second.Clusters[0].Name != parent+"/clusters/c-c" { + t.Fatalf("page 2 = %+v token %q", second.Clusters, second.NextPageToken) + } +} + +func TestSDKClusterValidation(t *testing.T) { + svc, _ := newSDKClient(t) + + mutate := func(fn func(c *mk.Cluster)) *mk.Cluster { + c := validCluster() + fn(c) + + return c + } + + cases := []struct { + name string + id string + body *mk.Cluster + }{ + {"vcpu below 3", "v1", mutate(func(c *mk.Cluster) { c.CapacityConfig = &mk.CapacityConfig{VcpuCount: 2, MemoryBytes: 2 * gib} })}, + {"memory below 1GiB per vcpu", "v2", mutate(func(c *mk.Cluster) { c.CapacityConfig.MemoryBytes = 3*gib - 1 })}, + {"memory above 8GiB per vcpu", "v3", mutate(func(c *mk.Cluster) { c.CapacityConfig.MemoryBytes = 24*gib + 1 })}, + {"no network configs", "v4", mutate(func(c *mk.Cluster) { c.GcpConfig.AccessConfig.NetworkConfigs = nil })}, + {"empty subnet", "v5", mutate(func(c *mk.Cluster) { c.GcpConfig.AccessConfig.NetworkConfigs[0].Subnet = "" })}, + {"missing clusterId", "", validCluster()}, + {"bad clusterId", "Bad_ID", validCluster()}, + {"bad rebalance mode", "v6", mutate(func(c *mk.Cluster) { c.RebalanceConfig.Mode = "SOMETIMES" })}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + call := svc.Projects.Locations.Clusters.Create(parent, tc.body) + if tc.id != "" { + call = call.ClusterId(tc.id) + } + + _, err := call.Do() + wantCode(t, err, http.StatusBadRequest, tc.name) + }) + } + + // Inclusive bounds are accepted: 1 GiB and 8 GiB per vCPU. + for id, mem := range map[string]int64{"lo": 3 * gib, "hi": 24 * gib} { + c := validCluster() + c.CapacityConfig.MemoryBytes = mem + + op, err := svc.Projects.Locations.Clusters.Create(parent, c).ClusterId(id).Do() + if err != nil { + t.Fatalf("boundary memory %d rejected: %v", mem, err) + } + + waitOp(t, svc, op) + } +} + +func TestSDKClusterConflictsAndMasks(t *testing.T) { + svc, _ := newSDKClient(t) + name := parent + "/clusters/dup" + + createCluster(t, svc, "dup") + + _, err := svc.Projects.Locations.Clusters.Create(parent, validCluster()).ClusterId("dup").Do() + wantCode(t, err, http.StatusConflict, "duplicate create") + + _, err = svc.Projects.Locations.Clusters.Get(parent + "/clusters/ghost").Do() + wantCode(t, err, http.StatusNotFound, "get missing") + + _, err = svc.Projects.Locations.Clusters.Delete(parent + "/clusters/ghost").Do() + wantCode(t, err, http.StatusNotFound, "delete missing") + + badMasks := []string{"", "bogusField", "state", "createTime", "gcpConfig.kmsKey", "name"} + for _, mask := range badMasks { + call := svc.Projects.Locations.Clusters.Patch(name, validCluster()) + if mask != "" { + call = call.UpdateMask(mask) + } + + _, err := call.Do() + wantCode(t, err, http.StatusBadRequest, "patch mask "+mask) + } + + // A masked patch that breaks the capacity rule is rejected and leaves the + // cluster unchanged. + _, err = svc.Projects.Locations.Clusters.Patch(name, &mk.Cluster{ + CapacityConfig: &mk.CapacityConfig{MemoryBytes: 100 * gib}, + }).UpdateMask("capacityConfig.memoryBytes").Do() + wantCode(t, err, http.StatusBadRequest, "patch memory out of range") + + got, err := svc.Projects.Locations.Clusters.Get(name).Do() + if err != nil || got.CapacityConfig.MemoryBytes != 3*gib { + t.Fatalf("cluster changed by rejected patch: %+v %v", got, err) + } +} + +func TestSDKTopicLifecycle(t *testing.T) { + svc, _ := newSDKClient(t) + clusterName := parent + "/clusters/tc" + topics := svc.Projects.Locations.Clusters.Topics + + createCluster(t, svc, "tc") + + created, err := topics.Create(clusterName, &mk.Topic{ + PartitionCount: 3, ReplicationFactor: 3, Configs: map[string]string{"cleanup.policy": "compact"}, + }).TopicId("orders").Do() + if err != nil { + t.Fatalf("Topics.Create: %v", err) + } + + topicName := clusterName + "/topics/orders" + if created.Name != topicName || created.PartitionCount != 3 || created.ReplicationFactor != 3 { + t.Fatalf("created topic = %+v", created) + } + + _, err = topics.Create(clusterName, &mk.Topic{PartitionCount: 1, ReplicationFactor: 1}).TopicId("orders").Do() + wantCode(t, err, http.StatusConflict, "duplicate topic") + + for _, bad := range []*mk.Topic{{PartitionCount: 0, ReplicationFactor: 3}, {PartitionCount: 3, ReplicationFactor: 0}} { + _, err = topics.Create(clusterName, bad).TopicId("bad").Do() + wantCode(t, err, http.StatusBadRequest, "invalid topic counts") + } + + if _, err = topics.Create(clusterName, &mk.Topic{PartitionCount: 1, ReplicationFactor: 1}).TopicId("audit").Do(); err != nil { + t.Fatalf("second topic: %v", err) + } + + list, err := topics.List(clusterName).PageSize(1).Do() + if err != nil || len(list.Topics) != 1 || list.NextPageToken == "" || list.Topics[0].Name != clusterName+"/topics/audit" { + t.Fatalf("topic list page 1 = %+v %v", list, err) + } + + // partitionCount may only increase; replicationFactor is immutable. + grown, err := topics.Patch(topicName, &mk.Topic{PartitionCount: 6, ReplicationFactor: 9}).UpdateMask("partitionCount").Do() + if err != nil { + t.Fatalf("grow partitions: %v", err) + } + + if grown.PartitionCount != 6 || grown.ReplicationFactor != 3 || grown.Configs["cleanup.policy"] != "compact" { + t.Fatalf("topic after masked patch = %+v", grown) + } + + _, err = topics.Patch(topicName, &mk.Topic{PartitionCount: 2}).UpdateMask("partitionCount").Do() + wantCode(t, err, http.StatusBadRequest, "shrink partitions") + + _, err = topics.Patch(topicName, &mk.Topic{ReplicationFactor: 1}).UpdateMask("replicationFactor").Do() + wantCode(t, err, http.StatusBadRequest, "patch immutable replicationFactor") + + if _, err = topics.Delete(topicName).Do(); err != nil { + t.Fatalf("Topics.Delete: %v", err) + } + + _, err = topics.Get(topicName).Do() + wantCode(t, err, http.StatusNotFound, "topic get after delete") +} + +func TestSDKTopicsFollowCluster(t *testing.T) { + svc, _ := newSDKClient(t) + clusterName := parent + "/clusters/gone" + topics := svc.Projects.Locations.Clusters.Topics + + _, err := topics.Create(clusterName, &mk.Topic{PartitionCount: 1, ReplicationFactor: 1}).TopicId("t").Do() + wantCode(t, err, http.StatusNotFound, "topic create under missing cluster") + + _, err = topics.List(clusterName).Do() + wantCode(t, err, http.StatusNotFound, "topic list under missing cluster") + + createCluster(t, svc, "gone") + + if _, err = topics.Create(clusterName, &mk.Topic{PartitionCount: 1, ReplicationFactor: 1}).TopicId("t").Do(); err != nil { + t.Fatalf("topic create: %v", err) + } + + dop, err := svc.Projects.Locations.Clusters.Delete(clusterName).Do() + if err != nil { + t.Fatalf("cluster delete: %v", err) + } + + waitOp(t, svc, dop) + + // Recreate the cluster: the old topic must not resurface. + createCluster(t, svc, "gone") + + _, err = topics.Get(clusterName + "/topics/t").Do() + wantCode(t, err, http.StatusNotFound, "topic after cluster delete") +} diff --git a/server/gcp/managedkafka/wire.go b/server/gcp/managedkafka/wire.go new file mode 100644 index 000000000..2b1f2515e --- /dev/null +++ b/server/gcp/managedkafka/wire.go @@ -0,0 +1,260 @@ +package managedkafka + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +const ( + clusterTypeURL = "type.googleapis.com/google.cloud.managedkafka.v1.Cluster" + emptyTypeURL = "type.googleapis.com/google.protobuf.Empty" + + int64Base = 10 + int64Bits = 64 +) + +// int64String is a proto3-JSON int64: marshaled as a decimal string, and +// accepted on input as either a string or a bare number (both are legal proto3 +// JSON). The Go discovery client tags these fields `json:",string"`. +type int64String int64 + +// MarshalJSON renders the value as a JSON string. +func (v int64String) MarshalJSON() ([]byte, error) { + return json.Marshal(strconv.FormatInt(int64(v), int64Base)) +} + +// UnmarshalJSON accepts "123" or 123. +func (v *int64String) UnmarshalJSON(b []byte) error { + s := strings.Trim(string(b), `"`) + + n, err := strconv.ParseInt(s, int64Base, int64Bits) + if err != nil { + return err + } + + *v = int64String(n) + + return nil +} + +type capacityJSON struct { + VcpuCount int64String `json:"vcpuCount,omitempty"` + MemoryBytes int64String `json:"memoryBytes,omitempty"` +} + +type networkConfigJSON struct { + Subnet string `json:"subnet,omitempty"` +} + +type accessConfigJSON struct { + NetworkConfigs []networkConfigJSON `json:"networkConfigs,omitempty"` +} + +type gcpConfigJSON struct { + AccessConfig *accessConfigJSON `json:"accessConfig,omitempty"` + KmsKey string `json:"kmsKey,omitempty"` +} + +type rebalanceJSON struct { + Mode string `json:"mode,omitempty"` +} + +// clusterJSON mirrors the managedkafka v1 Cluster message. Output-only fields +// (name, state, createTime, updateTime, satisfiesPzi/Pzs) are ignored on input. +type clusterJSON struct { + Name string `json:"name,omitempty"` + CapacityConfig *capacityJSON `json:"capacityConfig,omitempty"` + GcpConfig *gcpConfigJSON `json:"gcpConfig,omitempty"` + RebalanceConfig *rebalanceJSON `json:"rebalanceConfig,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + State string `json:"state,omitempty"` + CreateTime string `json:"createTime,omitempty"` + UpdateTime string `json:"updateTime,omitempty"` + SatisfiesPzi bool `json:"satisfiesPzi,omitempty"` + SatisfiesPzs bool `json:"satisfiesPzs,omitempty"` +} + +// topicJSON mirrors the managedkafka v1 Topic message (int32 counts are plain +// JSON numbers). +type topicJSON struct { + Name string `json:"name,omitempty"` + PartitionCount int32 `json:"partitionCount,omitempty"` + ReplicationFactor int32 `json:"replicationFactor,omitempty"` + Configs map[string]string `json:"configs,omitempty"` +} + +// operationJSON mirrors google.longrunning.Operation. Mutating ops complete +// inline, so `done` is always true. +type operationJSON struct { + Name string `json:"name"` + Done bool `json:"done"` + Response json.RawMessage `json:"response,omitempty"` +} + +// toDriverCluster converts a request body into a driver cluster scoped to rt. +func toDriverCluster(in *clusterJSON, rt *route, id string) *mkdriver.Cluster { + c := &mkdriver.Cluster{ + Project: rt.project, + Location: rt.location, + ID: id, + Labels: in.Labels, + } + + if in.CapacityConfig != nil { + c.VcpuCount = int64(in.CapacityConfig.VcpuCount) + c.MemoryBytes = int64(in.CapacityConfig.MemoryBytes) + } + + if in.GcpConfig != nil { + c.KmsKey = in.GcpConfig.KmsKey + + if in.GcpConfig.AccessConfig != nil { + for _, nc := range in.GcpConfig.AccessConfig.NetworkConfigs { + c.Subnets = append(c.Subnets, nc.Subnet) + } + } + } + + if in.RebalanceConfig != nil { + c.RebalanceMode = in.RebalanceConfig.Mode + } + + return c +} + +// fromDriverCluster renders a driver cluster as managedkafka v1 wire JSON. +func fromDriverCluster(c *mkdriver.Cluster) clusterJSON { + out := clusterJSON{ + Name: clusterName(c.Project, c.Location, c.ID), + CapacityConfig: &capacityJSON{ + VcpuCount: int64String(c.VcpuCount), + MemoryBytes: int64String(c.MemoryBytes), + }, + GcpConfig: &gcpConfigJSON{KmsKey: c.KmsKey, AccessConfig: &accessConfigJSON{}}, + Labels: c.Labels, + State: c.State, + CreateTime: formatTime(c.CreateTime), + UpdateTime: formatTime(c.UpdateTime), + SatisfiesPzi: c.SatisfiesPzi, + SatisfiesPzs: c.SatisfiesPzs, + } + + for _, s := range c.Subnets { + out.GcpConfig.AccessConfig.NetworkConfigs = append(out.GcpConfig.AccessConfig.NetworkConfigs, + networkConfigJSON{Subnet: s}) + } + + if c.RebalanceMode != "" { + out.RebalanceConfig = &rebalanceJSON{Mode: c.RebalanceMode} + } + + return out +} + +// toDriverTopic converts a request body into a driver topic scoped to rt. +func toDriverTopic(in *topicJSON, rt *route, id string) *mkdriver.Topic { + return &mkdriver.Topic{ + Project: rt.project, + Location: rt.location, + ClusterID: rt.cluster, + ID: id, + PartitionCount: in.PartitionCount, + ReplicationFactor: in.ReplicationFactor, + Configs: in.Configs, + } +} + +// fromDriverTopic renders a driver topic as managedkafka v1 wire JSON. +func fromDriverTopic(t *mkdriver.Topic) topicJSON { + return topicJSON{ + Name: topicName(t.Project, t.Location, t.ClusterID, t.ID), + PartitionCount: t.PartitionCount, + ReplicationFactor: t.ReplicationFactor, + Configs: t.Configs, + } +} + +// decodeBody decodes a JSON request body into v; an empty body leaves v zero. +// A malformed body is 400 INVALID_ARGUMENT. +func decodeBody(w http.ResponseWriter, r *http.Request, v any) bool { + r.Body = http.MaxBytesReader(w, r.Body, gcprest.MaxBodyBytes) + + if err := json.NewDecoder(r.Body).Decode(v); err != nil && !errors.Is(err, io.EOF) { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "malformed JSON body: "+err.Error()) + return false + } + + return true +} + +// anyWithType marshals v as a google.protobuf.Any by adding the "@type" +// discriminator to its JSON object. +func anyWithType(v any, typeURL string) (json.RawMessage, error) { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + + var fields map[string]json.RawMessage + if uErr := json.Unmarshal(raw, &fields); uErr != nil { + return nil, uErr + } + + if fields == nil { + fields = map[string]json.RawMessage{} + } + + typ, err := json.Marshal(typeURL) + if err != nil { + return nil, err + } + + fields["@type"] = typ + + return json.Marshal(fields) +} + +// writeOperation writes a completed operation whose response is v (typed as +// typeURL) and records it with the shared LRO poller (a no-op on a nil +// registry), so a client polling the returned name resolves the same done +// operation with its response. +func (h *Handler) writeOperation(w http.ResponseWriter, op *mkdriver.Operation, v any, typeURL string) { + resp, err := anyWithType(v, typeURL) + if err != nil { + gcprest.WriteError(w, http.StatusInternalServerError, "internalError", err.Error()) + return + } + + if h.ops != nil { + h.ops.Register(op.Name, resp) + } + + gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true, Response: resp}) +} + +// clusterName builds the full cluster resource name. +func clusterName(project, location, id string) string { + return "projects/" + project + "/locations/" + location + "/" + clustersSeg + "/" + id +} + +// topicName builds the full topic resource name. +func topicName(project, location, clusterID, id string) string { + return clusterName(project, location, clusterID) + "/" + topicsSeg + "/" + id +} + +// formatTime renders t as RFC3339Nano UTC; a zero time renders as "". +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + + return t.UTC().Format(time.RFC3339Nano) +} diff --git a/server/gcp/operations_ownership_test.go b/server/gcp/operations_ownership_test.go index 2c873a536..d0710e50a 100644 --- a/server/gcp/operations_ownership_test.go +++ b/server/gcp/operations_ownership_test.go @@ -146,3 +146,53 @@ func TestAlloyDBServerOperationOwnership(t *testing.T) { t.Fatalf("AlloyDB op GET: code=%d body=%s (want 200 done:true)", code, body) } } + +// TestFullServerManagedKafkaSharesClustersWithGKE covers the one location +// collection two services claim on the same assembled server: Managed Kafka and +// GKE both serve /v1/projects/{p}/locations/{l}/clusters. Kafka registers first +// and claims only its own traffic, so GKE creates/lists still reach GKE, Kafka +// clusters are served by Kafka, and a Kafka cluster operation is resolved by the +// shared LRO poller (with its typed response), not by GKE's operations route. +func TestFullServerManagedKafkaSharesClustersWithGKE(t *testing.T) { + ts := fullServer(t) + + const base = "/v1/projects/demo/locations/us-central1/clusters" + + // GKE create (a CreateClusterRequest wrapping {"cluster": …}) reaches GKE. + if code, body := do(t, ts, http.MethodPost, base, `{"cluster":{"name":"gke1","initialNodeCount":1}}`); code != http.StatusOK { + t.Fatalf("GKE create: code=%d body=%s", code, body) + } + + // With no Kafka clusters in the location, the list is GKE's. + if code, body := do(t, ts, http.MethodGet, base, ""); code != http.StatusOK || !strings.Contains(body, "gke1") { + t.Fatalf("GKE list before Kafka: code=%d body=%s", code, body) + } + + kafkaBody := `{"capacityConfig":{"vcpuCount":"3","memoryBytes":"3221225472"},` + + `"gcpConfig":{"accessConfig":{"networkConfigs":[{"subnet":"projects/demo/regions/us-central1/subnetworks/s"}]}}}` + + code, createBody := do(t, ts, http.MethodPost, base+"?clusterId=kafka1", kafkaBody) + if code != http.StatusOK || !strings.Contains(createBody, "google.cloud.managedkafka.v1.Cluster") { + t.Fatalf("Kafka create: code=%d body=%s", code, createBody) + } + + op := "/v1/" + opName(t, createBody) + if code, body := do(t, ts, http.MethodGet, op, ""); code != http.StatusOK || + !strings.Contains(body, `"done":true`) || !strings.Contains(body, "managedkafka.v1.Cluster") { + t.Fatalf("Kafka op GET via shared poller: code=%d body=%s", code, body) + } + + // Each item is served by its owner. + if code, body := do(t, ts, http.MethodGet, base+"/kafka1", ""); code != http.StatusOK || !strings.Contains(body, "capacityConfig") { + t.Fatalf("Kafka get: code=%d body=%s", code, body) + } + + if code, body := do(t, ts, http.MethodGet, base+"/gke1", ""); code != http.StatusOK || strings.Contains(body, "capacityConfig") { + t.Fatalf("GKE get: code=%d body=%s", code, body) + } + + // A bogus operation in the same location still 404s. + if code, _ := do(t, ts, http.MethodGet, "/v1/projects/demo/locations/us-central1/operations/nope", ""); code != http.StatusNotFound { + t.Fatalf("bogus op GET: code=%d, want 404", code) + } +} diff --git a/services/managedkafka/driver/driver.go b/services/managedkafka/driver/driver.go new file mode 100644 index 000000000..d9df09cad --- /dev/null +++ b/services/managedkafka/driver/driver.go @@ -0,0 +1,102 @@ +// Package driver defines the portable interface for the Google Cloud Managed +// Service for Apache Kafka control plane (managedkafka.googleapis.com/v1). It is +// control-plane only: the two resource collections a Terraform google provider +// or a real google.golang.org/api/managedkafka client CRUDs are modeled: +// +// projects/{p}/locations/{region}/clusters/{cluster} +// projects/{p}/locations/{region}/clusters/{cluster}/topics/{topic} +// +// Cluster mutations (create, update, delete) return location-scoped long-running +// operations, which share the operations space the shared GCP LRO poller owns: +// +// projects/{p}/locations/{region}/operations/{op} +// +// Topic mutations are synchronous, as in the real API: create/update return the +// Topic and delete returns Empty. +// +// There is no data plane: no brokers, no bootstrap address, no produce/consume. +// Consumer groups, ACLs, Kafka Connect clusters/connectors and schema registries +// are out of scope. +package driver + +import ( + "context" + "time" +) + +// Cluster is one Managed Kafka cluster. Name components are stored separately so +// the full resource name can be rebuilt without re-parsing. State, CreateTime and +// UpdateTime are output-only and owned by the provider. +type Cluster struct { + Project string + Location string + ID string + + // VcpuCount and MemoryBytes are the capacityConfig block. The real API + // requires vcpuCount >= 3 and a vCPU:GiB ratio between 1:1 and 1:8. + VcpuCount int64 + MemoryBytes int64 + + // Subnets are gcpConfig.accessConfig.networkConfigs[].subnet, in order. + Subnets []string + + // KmsKey is gcpConfig.kmsKey (immutable after create). + KmsKey string + + // RebalanceMode is rebalanceConfig.mode; empty when no rebalanceConfig was + // supplied. + RebalanceMode string + + Labels map[string]string + + State string + SatisfiesPzi bool + SatisfiesPzs bool + CreateTime time.Time + UpdateTime time.Time +} + +// Topic is one Kafka topic nested under a cluster. +type Topic struct { + Project string + Location string + ClusterID string + ID string + + PartitionCount int32 + ReplicationFactor int32 + Configs map[string]string +} + +// Operation is a completed long-running operation. Every CloudEmu mutation +// finishes synchronously, so Done is always true. +type Operation struct { + Name string // projects/{p}/locations/{region}/operations/{op} + Done bool + TargetName string // the cluster the operation acted on + Type string // create | update | delete +} + +// ManagedKafka is the control-plane interface a provider implements. +type ManagedKafka interface { + CreateCluster(ctx context.Context, c *Cluster) (*Cluster, *Operation, error) + GetCluster(ctx context.Context, project, location, id string) (*Cluster, error) + ListClusters(ctx context.Context, project, location string) ([]Cluster, error) + // UpdateCluster applies the fields of c named by mask (field-mask paths + // relative to the Cluster resource; "*" means every mutable field). + UpdateCluster(ctx context.Context, c *Cluster, mask []string) (*Cluster, *Operation, error) + // DeleteCluster removes the cluster and every topic under it. + DeleteCluster(ctx context.Context, project, location, id string) (*Operation, error) + + CreateTopic(ctx context.Context, t *Topic) (*Topic, error) + GetTopic(ctx context.Context, project, location, clusterID, id string) (*Topic, error) + ListTopics(ctx context.Context, project, location, clusterID string) ([]Topic, error) + // UpdateTopic applies the fields of t named by mask ("*" means every mutable + // field). partitionCount may only increase. + UpdateTopic(ctx context.Context, t *Topic, mask []string) (*Topic, error) + DeleteTopic(ctx context.Context, project, location, clusterID, id string) error + + // GetOperation resolves a (done) long-running operation by name, for a + // standalone package server's own operations poll. + GetOperation(ctx context.Context, name string) (*Operation, error) +} From 07c3cda39175e0fc3e366412e6030c91c4d3c4f6 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sat, 26 Sep 2026 20:22:58 +0530 Subject: [PATCH 02/24] feat(gcp): Backup and DR backup vaults control plane (backupdr.googleapis.com v1) Add projects.locations.backupVaults: create, get, list, patch, delete. Mutations return done google.longrunning.Operations registered with the shared location-scoped LRO poller, so Operations.Get replays the typed BackupVault response and unknown operation names 404. Validation: backupMinimumEnforcedRetentionDuration is required and must be a non-negative Duration string (400 otherwise); vault id 3-63 chars; accessRestriction/backupRetentionInheritance enums and RFC 3339 effectiveTime checked. Patch requires updateMask, rejects output-only or unknown paths (400) and a stale body etag (409 ABORTED). Delete honors force (a vault holding backups is 400 FAILED_PRECONDITION without it), allowMissing, etag (409 on mismatch) and validateOnly; create/patch honor validateOnly. Duplicate create 409, missing vault 404. Output-only fields: state ACTIVE, deletable, etag rotating per update, uid, backupCount/totalStoredBytes "0", deterministic serviceAccount service-{12-digit number derived from project id}@gcp-sa-backupdr-pr.iam.gserviceaccount.com, accessRestriction defaulting to WITHIN_ORGANIZATION. List supports pageSize/pageToken and the locations/- wildcard. Out of scope: data sources, backups, backup plans, management servers, restores; requestId, filter, orderBy, view and ignoreBackupPlanReferences are accepted and ignored. The provider mock's SetUsage hook seeds usage to exercise the non-empty delete guard. --- docs/coverage/README.md | 1 + docs/coverage/coverage.json | 33 ++ docs/coverage/gcp/README.md | 1 + docs/coverage/gcp/backupdr.md | 19 ++ docs/sdk-server.md | 1 + providers/gcp/backupdr/backupdr.go | 283 ++++++++++++++++++ providers/gcp/backupdr/backupdr_test.go | 279 +++++++++++++++++ providers/gcp/backupdr/clone.go | 39 +++ providers/gcp/backupdr/snapshot.go | 86 ++++++ providers/gcp/backupdr/validate.go | 295 ++++++++++++++++++ providers/gcp/gcp.go | 3 + server/gcp/backupdr/handler.go | 177 +++++++++++ server/gcp/backupdr/matches_test.go | 68 +++++ server/gcp/backupdr/operations.go | 245 +++++++++++++++ server/gcp/backupdr/sdk_roundtrip_test.go | 347 ++++++++++++++++++++++ server/gcp/backupdr/wire.go | 214 +++++++++++++ server/gcp/dispatch_ordering_test.go | 1 + server/gcp/from_provider.go | 1 + server/gcp/gcp.go | 21 ++ server/gcp/operations_ownership_test.go | 31 ++ services/backupdr/driver/driver.go | 133 +++++++++ 21 files changed, 2278 insertions(+) create mode 100644 docs/coverage/gcp/backupdr.md create mode 100644 providers/gcp/backupdr/backupdr.go create mode 100644 providers/gcp/backupdr/backupdr_test.go create mode 100644 providers/gcp/backupdr/clone.go create mode 100644 providers/gcp/backupdr/snapshot.go create mode 100644 providers/gcp/backupdr/validate.go create mode 100644 server/gcp/backupdr/handler.go create mode 100644 server/gcp/backupdr/matches_test.go create mode 100644 server/gcp/backupdr/operations.go create mode 100644 server/gcp/backupdr/sdk_roundtrip_test.go create mode 100644 server/gcp/backupdr/wire.go create mode 100644 services/backupdr/driver/driver.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index cb6af7a82..ea4d4ba4a 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -30,6 +30,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `azurefirewall` | - | [Firewall](./azure/firewall.md) | - | - | 8 | | `azuresearch` | - | [Search](./azure/search.md) | - | - | 53 | | `backup` | [Backup](./aws/backup.md) | - | - | - | 25 | +| `backupdr` | - | - | [BackupDR](./gcp/backupdr.md) | - | 6 | | `bastion` | - | [Bastion](./azure/bastion.md) | - | - | 4 | | `batch` | [Batch](./aws/batch.md) | - | - | - | 14 | | `batch-azure` | - | [Batch](./azure/batch.md) | - | - | 17 | diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index 8050f4e39..bc507f626 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -2059,6 +2059,39 @@ "aws": "Backup" } }, + { + "service": "backupdr", + "interface": "BackupDR", + "operations": [ + { + "name": "CreateBackupVault", + "doc": "CreateBackupVault creates a vault; the retention duration is required." + }, + { + "name": "DeleteBackupVault", + "doc": "DeleteBackupVault deletes a vault, honoring force, allowMissing and etag." + }, + { + "name": "GetBackupVault", + "doc": "GetBackupVault returns one vault by identity." + }, + { + "name": "GetOperation", + "doc": "GetOperation resolves a (done) long-running operation by name, for a" + }, + { + "name": "ListBackupVaults", + "doc": "ListBackupVaults lists a project's vaults in a location (\"-\" for every location)." + }, + { + "name": "UpdateBackupVault", + "doc": "UpdateBackupVault applies a field-masked update and rotates the etag." + } + ], + "providers": { + "gcp": "BackupDR" + } + }, { "service": "bastion", "interface": "BastionHosts", diff --git a/docs/coverage/gcp/README.md b/docs/coverage/gcp/README.md index 683608c1d..9cb0c5db6 100644 --- a/docs/coverage/gcp/README.md +++ b/docs/coverage/gcp/README.md @@ -9,6 +9,7 @@ Services cloudemu emulates for GCP, by native name. Back to the [cross-provider | [AccessContextManager](./accesscontextmanager.md) | `accesscontextmanager` | 17 | | [AlloyDB](./alloydb.md) | `relationaldb` | 21 | | [ArtifactRegistry](./artifactregistry.md) | `containerregistry` | 15 | +| [BackupDR](./backupdr.md) | `backupdr` | 6 | | [BigQuery](./bigquery.md) | `bigquery` | 12 | | [Bigtable](./bigtable.md) | `bigtable` | 38 | | [BinaryAuthorization](./binaryauthorization.md) | `binaryauthorization` | 10 | diff --git a/docs/coverage/gcp/backupdr.md b/docs/coverage/gcp/backupdr.md new file mode 100644 index 000000000..3f49b6840 --- /dev/null +++ b/docs/coverage/gcp/backupdr.md @@ -0,0 +1,19 @@ + +# BackupDR + +GCP's `backupdr` service · portable interface `driver.BackupDR` · [GCP index](./README.md) + +## Operations (6) + +| Operation | Description | +| --- | --- | +| `CreateBackupVault` | CreateBackupVault creates a vault; the retention duration is required. | +| `DeleteBackupVault` | DeleteBackupVault deletes a vault, honoring force, allowMissing and etag. | +| `GetBackupVault` | GetBackupVault returns one vault by identity. | +| `GetOperation` | GetOperation resolves a (done) long-running operation by name, for a | +| `ListBackupVaults` | ListBackupVaults lists a project's vaults in a location ("-" for every location). | +| `UpdateBackupVault` | UpdateBackupVault applies a field-masked update and rotates the etag. | + +## Not in scope + +_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 9232e33dc..84257550b 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -229,6 +229,7 @@ All handlers speak REST + JSON. | **Cloud Asset Inventory** | `assets.list` (filter by `assetTypes[]`), `searchAllResources` (query + asset-type filter), `searchAllIamPolicies` (returns empty; not implemented), `exportAssets` (sync; inline results in the returned Operation), `batchGetAssetsHistory`, Feeds (create/list/get/patch/delete), `operations.get`. Resource names returned as GCP-shaped `//service/path` URNs. | | **IAM (iam.googleapis.com v1)** | ServiceAccounts (Create/Get/List/Delete/Patch), custom Roles (Create/Get/List/Delete/Patch), ServiceAccountKeys (Create/Get/List/Delete). Real `google.golang.org/api/iam/v1` clients round-trip end-to-end; errors surface as typed `*googleapi.Error`. Resource-level `getIamPolicy`/`setIamPolicy` bindings on individual GCP resources are out of scope. | | **Managed Service for Apache Kafka (managedkafka.googleapis.com v1)** | Clusters (Create/Update/Delete as LROs polled through the shared location-operations route, Get, List with `pageToken`/`pageSize`; `updateMask` required, only masked fields change, unknown/immutable/output-only paths are 400) and Topics (Create/Get/List/Patch/Delete, synchronous; `partitionCount` can only increase, `replicationFactor` immutable). Validates `vcpuCount >= 3`, 1-8 GiB memory per vCPU, 1-10 `networkConfigs[].subnet`, and the RFC 1035 `clusterId`. Shares the `/clusters` path with GKE/AlloyDB: it claims only Kafka-shaped creates and clusters it owns. Real `google.golang.org/api/managedkafka/v1` clients round-trip end-to-end. Consumer groups, ACLs, Connect and Schema Registry are out of scope. | +| **Backup and DR (backupdr.googleapis.com v1)** | Backup vaults: create (LRO; `backupVaultId`, required `backupMinimumEnforcedRetentionDuration`, `validateOnly`), get, list (`pageSize`/`pageToken`, `locations/-` wildcard; `filter`/`orderBy` accepted and ignored), patch (LRO; required `updateMask`, output-only/unknown paths 400, stale body `etag` 409 ABORTED), delete (LRO; `force`, `allowMissing`, `etag`, `validateOnly`; a vault holding backups without `force` is 400 FAILED_PRECONDITION). Output-only `state` ACTIVE, `deletable`, `etag` (rotates per update), `uid`, deterministic `serviceAccount` (`service-{12-digit number derived from the project id}@gcp-sa-backupdr-pr.iam.gserviceaccount.com`), `backupCount`/`totalStoredBytes` "0". Operations poll through the shared location-scoped LRO poller. Data sources, backups, backup plans, management servers and restores are out of scope. Real `google.golang.org/api/backupdr/v1` clients round-trip end-to-end. | An operation cloudemu doesn't implement returns `501 Not Implemented` or the provider's native `UnknownOperation` / `NotImplemented` / `NOT_FOUND` error. diff --git a/providers/gcp/backupdr/backupdr.go b/providers/gcp/backupdr/backupdr.go new file mode 100644 index 000000000..0cdab9c10 --- /dev/null +++ b/providers/gcp/backupdr/backupdr.go @@ -0,0 +1,283 @@ +// Package backupdr provides an in-memory mock of the Google Cloud Backup and DR +// control plane (backupdr.googleapis.com/v1). It models backup vaults and the +// long-running operations their mutating RPCs return. It is control-plane only: +// there are no data sources, backups, backup plans or restores, so every vault +// is empty unless a test seeds usage through SetUsage. +package backupdr + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/memstore" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +var _ bdrdriver.BackupDR = (*Mock)(nil) + +const ( + vaultsColl = "backupVaults" + + // anyLocation is the list wildcard real Backup and DR accepts in place of a + // location ("projects/p/locations/-") to list across every location. + anyLocation = "-" + + opCreate = "create" + opUpdate = "update" + opDelete = "delete" +) + +// Mock is the in-memory Backup and DR control-plane implementation. Each vault +// is keyed by its full GCP resource name. +type Mock struct { + mu sync.RWMutex + + vaults *memstore.Store[bdrdriver.BackupVault] + operations *memstore.Store[bdrdriver.Operation] + + opSeq atomic.Uint64 + opts *config.Options +} + +// New creates a new Backup and DR mock. +func New(opts *config.Options) *Mock { + return &Mock{ + vaults: memstore.New[bdrdriver.BackupVault](), + operations: memstore.New[bdrdriver.Operation](), + opts: opts, + } +} + +// resourceName builds the full backup vault resource name. +func resourceName(project, location, id string) string { + return "projects/" + project + "/locations/" + location + "/" + vaultsColl + "/" + id +} + +// newOp records a completed operation scoped to the project+location it acted in +// and returns it. The caller holds the write lock. +func (m *Mock) newOp(project, location, opType, target string) *bdrdriver.Operation { + scope := "projects/" + project + "/locations/" + location + op := bdrdriver.Operation{ + Name: fmt.Sprintf("%s/operations/operation-%d-%s", scope, m.opSeq.Add(1), idgen.UUID()), + Done: true, + TargetName: target, + Type: opType, + } + m.operations.Set(op.Name, op) + + return &op +} + +// CreateBackupVault validates the request, mints the output-only fields (state +// ACTIVE, deterministic serviceAccount, uid, etag, zero usage, timestamps) and +// returns the completed LRO. With ValidateOnly nothing is stored. +func (m *Mock) CreateBackupVault(_ context.Context, cfg *bdrdriver.BackupVaultConfig) ( + *bdrdriver.BackupVault, *bdrdriver.Operation, error, +) { + if err := validateCreate(cfg); err != nil { + return nil, nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + key := resourceName(cfg.Project, cfg.Location, cfg.ID) + if m.vaults.Has(key) { + return nil, nil, cerrors.Newf(cerrors.AlreadyExists, "backup vault %q already exists", key) + } + + now := m.opts.Clock.Now().UTC() + v := bdrdriver.BackupVault{ + Project: cfg.Project, + Location: cfg.Location, + ID: cfg.ID, + Description: cfg.Description, + Labels: cloneStrMap(cfg.Labels), + Annotations: cloneStrMap(cfg.Annotations), + BackupMinimumEnforcedRetentionDuration: cfg.BackupMinimumEnforcedRetentionDuration, + BackupRetentionInheritance: cfg.BackupRetentionInheritance, + EffectiveTime: cfg.EffectiveTime, + AccessRestriction: defaultAccessRestriction(cfg.AccessRestriction), + EncryptionConfig: cloneEncryption(cfg.EncryptionConfig), + State: stateActive, + ServiceAccount: serviceAccount(cfg.Project), + UID: idgen.UUID(), + Revision: 1, + CreateTime: now, + UpdateTime: now, + } + v.Etag = etagFor(&v) + + if !cfg.ValidateOnly { + m.vaults.Set(key, v) + } + + op := m.newOp(cfg.Project, cfg.Location, opCreate, key) + out := cloneVault(&v) + + return &out, op, nil +} + +// GetBackupVault returns a vault by identity, cloned. +func (m *Mock) GetBackupVault(_ context.Context, project, location, id string) (*bdrdriver.BackupVault, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + v, ok := m.vaults.Get(resourceName(project, location, id)) + if !ok { + return nil, notFoundErr(project, location, id) + } + + out := cloneVault(&v) + + return &out, nil +} + +// ListBackupVaults returns every vault in a project+location (or every location +// for "-"), ordered by resource name. +func (m *Mock) ListBackupVaults(_ context.Context, project, location string) ([]bdrdriver.BackupVault, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + all := m.vaults.SortedValues() + out := make([]bdrdriver.BackupVault, 0, len(all)) + + for i := range all { + if all[i].Project != project || (location != anyLocation && all[i].Location != location) { + continue + } + + out = append(out, cloneVault(&all[i])) + } + + return out, nil +} + +// UpdateBackupVault applies a field-masked update. The mask is required and may +// only name mutable fields; a supplied etag must match. Every successful update +// bumps the revision, rotates the etag and advances updateTime. With +// ValidateOnly nothing is stored. +func (m *Mock) UpdateBackupVault(_ context.Context, cfg *bdrdriver.BackupVaultConfig, mask []string) ( + *bdrdriver.BackupVault, *bdrdriver.Operation, error, +) { + fields, err := normalizeMask(mask) + if err != nil { + return nil, nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + key := resourceName(cfg.Project, cfg.Location, cfg.ID) + + v, ok := m.vaults.Get(key) + if !ok { + return nil, nil, notFoundErr(cfg.Project, cfg.Location, cfg.ID) + } + + if cfg.Etag != "" && cfg.Etag != v.Etag { + return nil, nil, fmt.Errorf("backup vault %q: %w", key, bdrdriver.ErrEtagMismatch) + } + + if err := applyMask(&v, cfg, fields); err != nil { + return nil, nil, err + } + + v.Revision++ + v.UpdateTime = m.opts.Clock.Now().UTC() + v.Etag = etagFor(&v) + + if !cfg.ValidateOnly { + m.vaults.Set(key, v) + } + + op := m.newOp(cfg.Project, cfg.Location, opUpdate, key) + out := cloneVault(&v) + + return &out, op, nil +} + +// DeleteBackupVault removes a vault and returns the completed LRO. An absent +// vault is NOT_FOUND unless AllowMissing; a stale etag is rejected; a vault that +// still holds backups is FAILED_PRECONDITION unless Force. +func (m *Mock) DeleteBackupVault(_ context.Context, req *bdrdriver.DeleteBackupVaultRequest) ( + *bdrdriver.Operation, error, +) { + m.mu.Lock() + defer m.mu.Unlock() + + key := resourceName(req.Project, req.Location, req.ID) + + v, ok := m.vaults.Get(key) + if !ok { + if req.AllowMissing { + return m.newOp(req.Project, req.Location, opDelete, key), nil + } + + return nil, notFoundErr(req.Project, req.Location, req.ID) + } + + if req.Etag != "" && req.Etag != v.Etag { + return nil, fmt.Errorf("backup vault %q: %w", key, bdrdriver.ErrEtagMismatch) + } + + if !v.Deletable() && !req.Force { + return nil, cerrors.Newf(cerrors.FailedPrecondition, + "backup vault %q contains %d backups; set force=true to delete it with its data sources", key, v.BackupCount) + } + + if !req.ValidateOnly { + m.vaults.Delete(key) + } + + return m.newOp(req.Project, req.Location, opDelete, key), nil +} + +// GetOperation returns a (done) long-running operation by name. An unknown name +// is reported as a done operation: the mock completes synchronously, so any op +// id an SDK or Terraform poll asks for has already finished. +func (m *Mock) GetOperation(_ context.Context, name string) (*bdrdriver.Operation, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + op, ok := m.operations.Get(name) + if !ok { + return &bdrdriver.Operation{Name: name, Done: true}, nil + } + + out := op + + return &out, nil +} + +// SetUsage seeds a vault's output-only backupCount and totalStoredBytes. The +// emulator has no data plane, so this is the only way a vault becomes +// non-empty (deletable=false); tests use it to exercise the force-delete guard. +// It is deliberately not part of the driver interface. +func (m *Mock) SetUsage(project, location, id string, backupCount, totalStoredBytes int64) error { + m.mu.Lock() + defer m.mu.Unlock() + + key := resourceName(project, location, id) + + v, ok := m.vaults.Get(key) + if !ok { + return notFoundErr(project, location, id) + } + + v.BackupCount = backupCount + v.TotalStoredBytes = totalStoredBytes + m.vaults.Set(key, v) + + return nil +} + +// notFoundErr builds the NOT_FOUND error carrying the full resource name. +func notFoundErr(project, location, id string) error { + return cerrors.Newf(cerrors.NotFound, "backup vault %q not found", resourceName(project, location, id)) +} diff --git a/providers/gcp/backupdr/backupdr_test.go b/providers/gcp/backupdr/backupdr_test.go new file mode 100644 index 000000000..f8cab270a --- /dev/null +++ b/providers/gcp/backupdr/backupdr_test.go @@ -0,0 +1,279 @@ +package backupdr + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +const ( + testProject = "p" + testLocation = "us-central1" + testVault = "vault-1" +) + +func newMock(t *testing.T) (*Mock, *config.FakeClock) { + t.Helper() + + clk := config.NewFakeClock(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) + + return New(config.NewOptions(config.WithProjectID(testProject), config.WithClock(clk))), clk +} + +func vaultCfg(id string) *bdrdriver.BackupVaultConfig { + return &bdrdriver.BackupVaultConfig{ + Project: testProject, Location: testLocation, ID: id, + Description: "d", + Labels: map[string]string{"env": "dev"}, + BackupMinimumEnforcedRetentionDuration: "86400s", + } +} + +func TestCreateMintsOutputFields(t *testing.T) { + m, clk := newMock(t) + ctx := context.Background() + + v, op, err := m.CreateBackupVault(ctx, vaultCfg(testVault)) + if err != nil { + t.Fatalf("CreateBackupVault: %v", err) + } + + if !op.Done || !strings.HasPrefix(op.Name, "projects/p/locations/us-central1/operations/") { + t.Fatalf("operation = %+v", op) + } + + if v.State != stateActive || v.AccessRestriction != accessWithinOrganization || !v.Deletable() { + t.Fatalf("defaults not applied: %+v", v) + } + + if v.ServiceAccount != serviceAccount(testProject) || !strings.HasSuffix(v.ServiceAccount, serviceAccountDomain) { + t.Fatalf("serviceAccount = %q", v.ServiceAccount) + } + + if v.UID == "" || v.Etag == "" || !v.CreateTime.Equal(clk.Now()) { + t.Fatalf("uid/etag/createTime not minted: %+v", v) + } + + // Returned values never alias the store. + v.Labels["env"] = "mutated" + + got, err := m.GetBackupVault(ctx, testProject, testLocation, testVault) + if err != nil { + t.Fatalf("GetBackupVault: %v", err) + } + + if got.Labels["env"] != "dev" { + t.Fatalf("store aliased by returned value: %v", got.Labels) + } + + if _, _, err := m.CreateBackupVault(ctx, vaultCfg(testVault)); !cerrors.IsAlreadyExists(err) { + t.Fatalf("duplicate create err = %v, want AlreadyExists", err) + } +} + +func TestCreateValidation(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + cases := map[string]func(*bdrdriver.BackupVaultConfig){ + "missing retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "" }, + "malformed retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "1d" }, + "negative retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "-5s" }, + "short id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "ab" }, + "bad access": func(c *bdrdriver.BackupVaultConfig) { c.AccessRestriction = "NOPE" }, + "bad inheritance": func(c *bdrdriver.BackupVaultConfig) { c.BackupRetentionInheritance = "NOPE" }, + "bad effectiveTime": func(c *bdrdriver.BackupVaultConfig) { c.EffectiveTime = "yesterday" }, + "wildcard location": func(c *bdrdriver.BackupVaultConfig) { c.Location = anyLocation }, + } + + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + cfg := vaultCfg(testVault) + mutate(cfg) + + if _, _, err := m.CreateBackupVault(ctx, cfg); !cerrors.IsInvalidArgument(err) { + t.Fatalf("err = %v, want InvalidArgument", err) + } + }) + } + + for _, ok := range []string{"0s", "1.5s", "3600s"} { + cfg := vaultCfg("ok-" + strings.ReplaceAll(strings.TrimSuffix(ok, "s"), ".", "-")) + cfg.BackupMinimumEnforcedRetentionDuration = ok + + if _, _, err := m.CreateBackupVault(ctx, cfg); err != nil { + t.Fatalf("retention %q rejected: %v", ok, err) + } + } +} + +func TestValidateOnlyDoesNotPersist(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + cfg := vaultCfg(testVault) + cfg.ValidateOnly = true + + if _, _, err := m.CreateBackupVault(ctx, cfg); err != nil { + t.Fatalf("validateOnly create: %v", err) + } + + if _, err := m.GetBackupVault(ctx, testProject, testLocation, testVault); !cerrors.IsNotFound(err) { + t.Fatalf("validateOnly create persisted the vault: %v", err) + } +} + +func TestUpdateMaskAndEtag(t *testing.T) { + m, clk := newMock(t) + ctx := context.Background() + + created, _, err := m.CreateBackupVault(ctx, vaultCfg(testVault)) + if err != nil { + t.Fatalf("create: %v", err) + } + + clk.Advance(time.Minute) + + patch := &bdrdriver.BackupVaultConfig{ + Project: testProject, Location: testLocation, ID: testVault, + Description: "new", Labels: map[string]string{"x": "y"}, Etag: created.Etag, + } + + updated, _, err := m.UpdateBackupVault(ctx, patch, []string{"description"}) + if err != nil { + t.Fatalf("update: %v", err) + } + + if updated.Description != "new" || updated.Labels["env"] != "dev" { + t.Fatalf("mask not honored: %+v", updated) + } + + if updated.Etag == created.Etag || !updated.UpdateTime.After(created.UpdateTime) { + t.Fatalf("etag/updateTime not rotated: %q -> %q", created.Etag, updated.Etag) + } + + // Stale etag (the create-time one) is rejected. + if _, _, err := m.UpdateBackupVault(ctx, patch, []string{"labels"}); !errors.Is(err, bdrdriver.ErrEtagMismatch) { + t.Fatalf("stale etag err = %v, want ErrEtagMismatch", err) + } + + // snake_case mask paths are accepted. + patch.Etag = "" + patch.BackupMinimumEnforcedRetentionDuration = "172800s" + + got, _, err := m.UpdateBackupVault(ctx, patch, []string{"backup_minimum_enforced_retention_duration"}) + if err != nil || got.BackupMinimumEnforcedRetentionDuration != "172800s" { + t.Fatalf("snake_case mask: %v %+v", err, got) + } + + for _, bad := range [][]string{nil, {"state"}, {"etag"}, {"bogus"}} { + if _, _, err := m.UpdateBackupVault(ctx, patch, bad); !cerrors.IsInvalidArgument(err) { + t.Fatalf("mask %v err = %v, want InvalidArgument", bad, err) + } + } + + patch.BackupMinimumEnforcedRetentionDuration = "-1s" + if _, _, err := m.UpdateBackupVault(ctx, patch, []string{fieldRetention}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("negative retention patch err = %v", err) + } +} + +func TestDeleteSemantics(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + v, _, err := m.CreateBackupVault(ctx, vaultCfg(testVault)) + if err != nil { + t.Fatalf("create: %v", err) + } + + req := &bdrdriver.DeleteBackupVaultRequest{Project: testProject, Location: testLocation, ID: testVault} + + if err := m.SetUsage(testProject, testLocation, testVault, 3, 1024); err != nil { + t.Fatalf("SetUsage: %v", err) + } + + if _, err := m.DeleteBackupVault(ctx, req); !cerrors.IsFailedPrecondition(err) || errors.Is(err, bdrdriver.ErrEtagMismatch) { + t.Fatalf("non-empty delete err = %v, want FailedPrecondition", err) + } + + req.Etag = "stale" + if _, err := m.DeleteBackupVault(ctx, req); !errors.Is(err, bdrdriver.ErrEtagMismatch) { + t.Fatalf("stale etag delete err = %v", err) + } + + req.Etag = v.Etag + req.Force = true + + if _, err := m.DeleteBackupVault(ctx, req); err != nil { + t.Fatalf("force delete: %v", err) + } + + if _, err := m.DeleteBackupVault(ctx, req); !cerrors.IsNotFound(err) { + t.Fatalf("second delete err = %v, want NotFound", err) + } + + req.AllowMissing = true + + if op, err := m.DeleteBackupVault(ctx, req); err != nil || !op.Done { + t.Fatalf("allowMissing delete: %v %+v", err, op) + } +} + +func TestListScopesAndWildcard(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + for _, loc := range []string{"us-central1", "europe-west1"} { + cfg := vaultCfg("vault-" + loc) + cfg.Location = loc + + if _, _, err := m.CreateBackupVault(ctx, cfg); err != nil { + t.Fatalf("create: %v", err) + } + } + + one, _ := m.ListBackupVaults(ctx, testProject, "us-central1") + all, _ := m.ListBackupVaults(ctx, testProject, anyLocation) + other, _ := m.ListBackupVaults(ctx, "other", anyLocation) + + if len(one) != 1 || len(all) != 2 || len(other) != 0 { + t.Fatalf("list sizes = %d/%d/%d, want 1/2/0", len(one), len(all), len(other)) + } +} + +func TestSnapshotRoundTrip(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + v, _, err := m.CreateBackupVault(ctx, vaultCfg(testVault)) + if err != nil { + t.Fatalf("create: %v", err) + } + + data, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + restored, _ := newMock(t) + if err := restored.Restore(ctx, data); err != nil { + t.Fatalf("Restore: %v", err) + } + + got, err := restored.GetBackupVault(ctx, testProject, testLocation, testVault) + if err != nil || got.UID != v.UID || got.Etag != v.Etag { + t.Fatalf("restored vault = %+v (err %v), want uid/etag of %+v", got, err, v) + } + + _, op, err := restored.CreateBackupVault(ctx, vaultCfg("vault-2")) + if err != nil || !strings.Contains(op.Name, "/operation-2-") { + t.Fatalf("opSeq not restored: %v %+v", err, op) + } +} diff --git a/providers/gcp/backupdr/clone.go b/providers/gcp/backupdr/clone.go new file mode 100644 index 000000000..c788f7d9a --- /dev/null +++ b/providers/gcp/backupdr/clone.go @@ -0,0 +1,39 @@ +package backupdr + +import ( + "maps" + + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +// cloneVault returns a deep copy of v so a stored vault is never aliased by a +// value handed back to a caller (which the wire layer would otherwise be free to +// mutate). The label/annotation maps and the encryption config are copied. +func cloneVault(v *bdrdriver.BackupVault) bdrdriver.BackupVault { + out := *v + out.Labels = cloneStrMap(v.Labels) + out.Annotations = cloneStrMap(v.Annotations) + out.EncryptionConfig = cloneEncryption(v.EncryptionConfig) + + return out +} + +// cloneStrMap copies a string map; an empty map clones to nil. +func cloneStrMap(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + + return maps.Clone(in) +} + +// cloneEncryption copies an optional encryption config. +func cloneEncryption(in *bdrdriver.EncryptionConfig) *bdrdriver.EncryptionConfig { + if in == nil { + return nil + } + + out := *in + + return &out +} diff --git a/providers/gcp/backupdr/snapshot.go b/providers/gcp/backupdr/snapshot.go new file mode 100644 index 000000000..c8d7c649b --- /dev/null +++ b/providers/gcp/backupdr/snapshot.go @@ -0,0 +1,86 @@ +package backupdr + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/stackshy/cloudemu/v2/internal/snapshot" +) + +var _ snapshot.Snapshottable = (*Mock)(nil) + +// backupdrSnapshot is the full serialized state of the Backup and DR mock. Every +// store holds fully-exported bdrdriver value types keyed by their full GCP +// resource name, so each round-trips through the generic memstore helper: no +// field promotion is needed. opSeq is the operation-name counter, captured beside +// the stores so restored operation ids do not collide with fresh ones. The wired +// deps (m.opts) and the RWMutex are intentionally not serialized. +type backupdrSnapshot struct { + Vaults json.RawMessage `json:"backupVaults,omitempty"` + Operations json.RawMessage `json:"operations,omitempty"` + OpSeq uint64 `json:"opSeq,omitempty"` +} + +// Snapshot captures every backup vault and operation as JSON. includeAssets is +// unused: Backup and DR admin is control-plane only and holds no bulk object bodies. +func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var snap backupdrSnapshot + + dumps := []struct { + dst *json.RawMessage + fn func() ([]byte, error) + }{ + {&snap.Vaults, m.vaults.Snapshot}, + {&snap.Operations, m.operations.Snapshot}, + } + + for _, d := range dumps { + b, err := d.fn() + if err != nil { + return nil, fmt.Errorf("backupdr: snapshot store: %w", err) + } + + *d.dst = b + } + + snap.OpSeq = m.opSeq.Load() + + return json.Marshal(snap) +} + +// Restore rebuilds every backup vault and operation under its original resource name. +func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { + var snap backupdrSnapshot + if err := json.Unmarshal(data, &snap); err != nil { + return fmt.Errorf("backupdr: parse snapshot: %w", err) + } + + m.mu.Lock() + defer m.mu.Unlock() + + loads := []struct { + src json.RawMessage + fn func([]byte) error + }{ + {snap.Vaults, m.vaults.LoadSnapshot}, + {snap.Operations, m.operations.LoadSnapshot}, + } + + for _, l := range loads { + if len(l.src) == 0 { + continue + } + + if err := l.fn(l.src); err != nil { + return fmt.Errorf("backupdr: restore store: %w", err) + } + } + + m.opSeq.Store(snap.OpSeq) + + return nil +} diff --git a/providers/gcp/backupdr/validate.go b/providers/gcp/backupdr/validate.go new file mode 100644 index 000000000..85cedff45 --- /dev/null +++ b/providers/gcp/backupdr/validate.go @@ -0,0 +1,295 @@ +package backupdr + +import ( + "hash/fnv" + "regexp" + "strconv" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +const ( + stateActive = "ACTIVE" + + accessWithinOrganization = "WITHIN_ORGANIZATION" + accessUnspecified = "ACCESS_RESTRICTION_UNSPECIFIED" + + // minVaultIDLen / maxVaultIDLen bound a vault id, per the BackupVault.name + // contract ("must be between 3-63 characters long"). + minVaultIDLen = 3 + maxVaultIDLen = 63 + + // serviceAccountDomain is the Backup and DR service-agent domain. CloudEmu + // synthesizes the vault serviceAccount as + // service-{projectNumber}@gcp-sa-backupdr-pr.iam.gserviceaccount.com, where + // projectNumber is a stable 12-digit number derived from the project id (see + // projectNumber), so it is identical on every read and across restarts. + serviceAccountDomain = "@gcp-sa-backupdr-pr.iam.gserviceaccount.com" + + // projectNumberBase / projectNumberSpan keep a derived project number at + // exactly 12 digits (100000000000..999999999999), like a real GCP project + // number. + projectNumberBase = 100000000000 + projectNumberSpan = 900000000000 + + fieldDescription = "description" + fieldLabels = "labels" + fieldAnnotations = "annotations" + fieldRetention = "backupMinimumEnforcedRetentionDuration" + fieldInheritance = "backupRetentionInheritance" + fieldEffectiveTime = "effectiveTime" + fieldAccess = "accessRestriction" + fieldEncryptionCfg = "encryptionConfig" + etagRevisionSeparator = "#" + + // hexBase / decimalBase are the strconv bases for the etag and numbers. + hexBase = 16 + decimalBase = 10 +) + +// durationPattern matches the google.protobuf.Duration JSON form: an optionally +// signed decimal number of seconds with up to nine fractional digits and an "s" +// suffix (e.g. "86400s", "1.5s", "-3s"). +var durationPattern = regexp.MustCompile(`^(-)?\d+(\.\d{1,9})?s$`) + +// validAccessRestrictions is the AccessRestriction enum from the discovery doc. +// +//nolint:gochecknoglobals // immutable lookup set +var validAccessRestrictions = map[string]bool{ + accessUnspecified: true, + "WITHIN_PROJECT": true, + accessWithinOrganization: true, + "UNRESTRICTED": true, + "WITHIN_ORG_BUT_UNRESTRICTED_FOR_BA": true, +} + +// validInheritance is the BackupRetentionInheritance enum from the discovery doc. +// +//nolint:gochecknoglobals // immutable lookup set +var validInheritance = map[string]bool{ + "BACKUP_RETENTION_INHERITANCE_UNSPECIFIED": true, + "INHERIT_VAULT_RETENTION": true, + "MATCH_BACKUP_EXPIRE_TIME": true, +} + +// outputOnlyFields are BackupVault fields a caller may not name in an +// updateMask: they are output-only (or, for name, immutable identity). +// +//nolint:gochecknoglobals // immutable lookup set +var outputOnlyFields = map[string]bool{ + "name": true, "createTime": true, "updateTime": true, "state": true, + "deletable": true, "etag": true, "serviceAccount": true, "uid": true, + "totalStoredBytes": true, "backupCount": true, +} + +// mutableFields are the BackupVault fields an updateMask may name. +// +//nolint:gochecknoglobals // immutable lookup set +var mutableFields = map[string]bool{ + fieldDescription: true, fieldLabels: true, fieldAnnotations: true, fieldRetention: true, + fieldInheritance: true, fieldEffectiveTime: true, fieldAccess: true, fieldEncryptionCfg: true, +} + +// validateCreate checks a create request: the vault id and the required +// retention duration, plus every optional enum/timestamp the caller supplied. +func validateCreate(cfg *bdrdriver.BackupVaultConfig) error { + if cfg.Location == "" || cfg.Location == anyLocation { + return cerrors.New(cerrors.InvalidArgument, "a concrete location is required") + } + + if n := len(cfg.ID); n < minVaultIDLen || n > maxVaultIDLen { + return cerrors.Newf(cerrors.InvalidArgument, + "backupVaultId %q must be between %d and %d characters", cfg.ID, minVaultIDLen, maxVaultIDLen) + } + + checks := []func(*bdrdriver.BackupVaultConfig) error{ + checkRetention, checkInheritance, checkEffectiveTime, checkAccessRestriction, + } + + for _, check := range checks { + if err := check(cfg); err != nil { + return err + } + } + + return nil +} + +// checkRetention requires backupMinimumEnforcedRetentionDuration to be a +// well-formed, non-negative google.protobuf.Duration string. +func checkRetention(cfg *bdrdriver.BackupVaultConfig) error { + d := cfg.BackupMinimumEnforcedRetentionDuration + if d == "" { + return cerrors.New(cerrors.InvalidArgument, "backupMinimumEnforcedRetentionDuration is required") + } + + m := durationPattern.FindStringSubmatch(d) + if m == nil { + return cerrors.Newf(cerrors.InvalidArgument, + "backupMinimumEnforcedRetentionDuration %q is not a valid duration (want e.g. \"86400s\")", d) + } + + if m[1] != "" { + if secs, err := strconv.ParseFloat(strings.TrimSuffix(d, "s"), 64); err != nil || secs != 0 { + return cerrors.Newf(cerrors.InvalidArgument, "backupMinimumEnforcedRetentionDuration %q must not be negative", d) + } + } + + return nil +} + +// checkInheritance validates an optional backupRetentionInheritance enum. +func checkInheritance(cfg *bdrdriver.BackupVaultConfig) error { + if v := cfg.BackupRetentionInheritance; v != "" && !validInheritance[v] { + return cerrors.Newf(cerrors.InvalidArgument, "invalid backupRetentionInheritance %q", v) + } + + return nil +} + +// checkEffectiveTime validates an optional RFC 3339 effectiveTime. +func checkEffectiveTime(cfg *bdrdriver.BackupVaultConfig) error { + if v := cfg.EffectiveTime; v != "" { + if _, err := time.Parse(time.RFC3339Nano, v); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "effectiveTime %q is not an RFC 3339 timestamp", v) + } + } + + return nil +} + +// checkAccessRestriction validates an optional accessRestriction enum. +func checkAccessRestriction(cfg *bdrdriver.BackupVaultConfig) error { + if v := cfg.AccessRestriction; v != "" && !validAccessRestrictions[v] { + return cerrors.Newf(cerrors.InvalidArgument, "invalid accessRestriction %q", v) + } + + return nil +} + +// defaultAccessRestriction applies the documented default: an absent or +// UNSPECIFIED access restriction becomes WITHIN_ORGANIZATION. +func defaultAccessRestriction(v string) string { + if v == "" || v == accessUnspecified { + return accessWithinOrganization + } + + return v +} + +// normalizeMask validates a required updateMask and returns the set of +// top-level camelCase fields it names. Paths may be snake_case (the proto +// spelling some clients send) or camelCase; a nested path +// (encryptionConfig.kmsKeyName) names its top-level field. An empty mask, an +// output-only field, or an unknown field is INVALID_ARGUMENT. +func normalizeMask(mask []string) (map[string]bool, error) { + if len(mask) == 0 { + return nil, cerrors.New(cerrors.InvalidArgument, "updateMask is required") + } + + out := make(map[string]bool, len(mask)) + + for _, path := range mask { + top := snakeToCamel(strings.SplitN(path, ".", 2)[0]) //nolint:mnd // split into head and rest + + switch { + case outputOnlyFields[top]: + return nil, cerrors.Newf(cerrors.InvalidArgument, "updateMask path %q names an output-only field", path) + case !mutableFields[top]: + return nil, cerrors.Newf(cerrors.InvalidArgument, "updateMask path %q is not a BackupVault field", path) + } + + out[top] = true + } + + return out, nil +} + +// applyMask copies each masked field from cfg onto v, validating the new value. +// Fields outside the mask are left untouched. +func applyMask(v *bdrdriver.BackupVault, cfg *bdrdriver.BackupVaultConfig, fields map[string]bool) error { + checks := map[string]func(*bdrdriver.BackupVaultConfig) error{ + fieldRetention: checkRetention, + fieldInheritance: checkInheritance, + fieldEffectiveTime: checkEffectiveTime, + fieldAccess: checkAccessRestriction, + } + + for f := range fields { + if check, ok := checks[f]; ok { + if err := check(cfg); err != nil { + return err + } + } + } + + setters := map[string]func(){ + fieldDescription: func() { v.Description = cfg.Description }, + fieldLabels: func() { v.Labels = cloneStrMap(cfg.Labels) }, + fieldAnnotations: func() { v.Annotations = cloneStrMap(cfg.Annotations) }, + fieldRetention: func() { v.BackupMinimumEnforcedRetentionDuration = cfg.BackupMinimumEnforcedRetentionDuration }, + fieldInheritance: func() { v.BackupRetentionInheritance = cfg.BackupRetentionInheritance }, + fieldEffectiveTime: func() { v.EffectiveTime = cfg.EffectiveTime }, + fieldAccess: func() { v.AccessRestriction = defaultAccessRestriction(cfg.AccessRestriction) }, + fieldEncryptionCfg: func() { v.EncryptionConfig = cloneEncryption(cfg.EncryptionConfig) }, + } + + for f := range fields { + setters[f]() + } + + return nil +} + +// snakeToCamel converts a snake_case proto field name to its JSON camelCase +// spelling; a name without underscores is returned unchanged. +func snakeToCamel(s string) string { + if !strings.Contains(s, "_") { + return s + } + + parts := strings.Split(s, "_") + + var b strings.Builder + + b.WriteString(parts[0]) + + for _, p := range parts[1:] { + if p == "" { + continue + } + + b.WriteString(strings.ToUpper(p[:1]) + p[1:]) + } + + return b.String() +} + +// projectNumber derives a stable 12-digit project number from a project id, so +// the synthesized service account is deterministic. +func projectNumber(project string) uint64 { + h := fnv.New64a() + _, _ = h.Write([]byte(project)) + + return projectNumberBase + h.Sum64()%projectNumberSpan +} + +// serviceAccount returns the deterministic Backup and DR service agent for a +// project (see serviceAccountDomain). +func serviceAccount(project string) string { + return "service-" + strconv.FormatUint(projectNumber(project), decimalBase) + serviceAccountDomain +} + +// etagFor derives the vault etag from its identity, revision and update time, +// so it changes on every successful update and is deterministic under a fake +// clock. +func etagFor(v *bdrdriver.BackupVault) string { + h := fnv.New64a() + _, _ = h.Write([]byte(resourceName(v.Project, v.Location, v.ID) + etagRevisionSeparator + + strconv.FormatInt(v.Revision, decimalBase) + etagRevisionSeparator + v.UpdateTime.Format(time.RFC3339Nano))) + + return strconv.FormatUint(h.Sum64(), hexBase) +} diff --git a/providers/gcp/gcp.go b/providers/gcp/gcp.go index 598b46d56..ac0a254ee 100644 --- a/providers/gcp/gcp.go +++ b/providers/gcp/gcp.go @@ -12,6 +12,7 @@ import ( "github.com/stackshy/cloudemu/v2/providers/gcp/alloydb" apigatewayprov "github.com/stackshy/cloudemu/v2/providers/gcp/apigateway" "github.com/stackshy/cloudemu/v2/providers/gcp/artifactregistry" + backupdrprov "github.com/stackshy/cloudemu/v2/providers/gcp/backupdr" "github.com/stackshy/cloudemu/v2/providers/gcp/bigquery" "github.com/stackshy/cloudemu/v2/providers/gcp/bigtable" "github.com/stackshy/cloudemu/v2/providers/gcp/binaryauthorization" @@ -122,6 +123,7 @@ type Provider struct { VPCAccess *vpcaccessprov.Mock CloudIDS *cloudidsprov.Mock ManagedKafka *managedkafkaprov.Mock + BackupDR *backupdrprov.Mock SecureSourceManager *securesourcemanagerprov.Mock NetworkConnectivity *networkconnectivity.Mock Composer *composer.Mock @@ -193,6 +195,7 @@ func New(opts ...config.Option) *Provider { VPCAccess: vpcaccessprov.New(o), CloudIDS: cloudidsprov.New(o), ManagedKafka: managedkafkaprov.New(o), + BackupDR: backupdrprov.New(o), SecureSourceManager: securesourcemanagerprov.New(o), NetworkConnectivity: networkconnectivity.New(o), Composer: composer.New(o), diff --git a/server/gcp/backupdr/handler.go b/server/gcp/backupdr/handler.go new file mode 100644 index 000000000..993fb09b4 --- /dev/null +++ b/server/gcp/backupdr/handler.go @@ -0,0 +1,177 @@ +// Package backupdr implements the Google Cloud Backup and DR backup vault control +// plane (backupdr.googleapis.com/v1) as a server.Handler. Real +// google.golang.org/api/backupdr/v1 clients, gcloud, and the Terraform google +// provider's google_backup_dr_backup_vault resource hit this handler unchanged. +// +// Coverage (backup vault control plane): +// +// POST /v1/…/backupVaults?backupVaultId= : CreateBackupVault (LRO) +// GET /v1/…/backupVaults : ListBackupVaults +// GET /v1/…/backupVaults/{id} : GetBackupVault +// PATCH /v1/…/backupVaults/{id}?updateMask= : UpdateBackupVault (LRO) +// DELETE /v1/…/backupVaults/{id} : DeleteBackupVault (LRO) +// GET /v1/…/operations/{op} : Operations.Get (shared poller) +// +// Every mutating RPC returns a google.longrunning.Operation with done=true and, +// for create/patch, the resulting vault embedded in `response` as an Any typed +// type.googleapis.com/google.cloud.backupdr.v1.BackupVault, so an SDK or +// Terraform LRO wait terminates on the first poll. +// +// Location-scoped operations: a vault's operations live under +// /v1/projects/{p}/locations/{l}/operations, the same space the shared GCP LRO +// poller owns. Matches returns false for operation paths when a shared registry +// is wired, letting that poller win; a standalone package server (no registry) +// serves its own polls. The backupVaults resource-type guard keeps this handler +// disjoint from every other /v1/projects/ handler. +package backupdr + +import ( + "net/http" + "strings" + + "github.com/stackshy/cloudemu/v2/server/gcp/lro" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +const ( + pathPrefix = "/v1/projects/" + projectsSeg = "projects" + locationsSeg = "locations" + operationsSeg = "operations" + vaultsColl = "backupVaults" + vaultIDParam = "backupVaultId" + minResourceParts = 4 // [projects, {p}, locations, {l}] + itemParts = 2 // [resource, {name}] + + vaultTypeURL = "type.googleapis.com/google.cloud.backupdr.v1.BackupVault" +) + +// Handler serves backupdr.googleapis.com v1 requests against a BackupDR driver. +type Handler struct { + db bdrdriver.BackupDR + + // ops records created operations with the shared poller so a client that + // polls the returned operation name gets the typed response (and unknown + // names 404). Nil in a standalone package server, where this handler serves + // its own /operations/ poll. + ops *lro.Registry +} + +// New returns a Backup and DR handler backed by db. +func New(db bdrdriver.BackupDR) *Handler { return &Handler{db: db} } + +// SetOperationRegistry wires the shared LRO poller so created operations are +// resolvable (with their response) through the full server's operations host. +func (h *Handler) SetOperationRegistry(reg *lro.Registry) { h.ops = reg } + +// route holds the parsed components of a Backup and DR v1 path. +type route struct { + project string + location string + resource string // "backupVaults" | "operations" + name string // vault id or operation id; empty for the collection +} + +// parseRoute extracts the components of a Backup and DR v1 path. It recognizes +// only the backupVaults and operations resources under a locations scope. +func parseRoute(urlPath string) (route, bool) { + if !strings.HasPrefix(urlPath, pathPrefix) { + return route{}, false + } + + parts := strings.Split(strings.TrimPrefix(urlPath, "/v1/"), "/") + if len(parts) < minResourceParts || parts[0] != projectsSeg || parts[2] != locationsSeg { + return route{}, false + } + + rest := parts[minResourceParts:] + if len(rest) == 0 || len(rest) > itemParts || !knownResource(rest[0]) { + return route{}, false + } + + rt := route{project: parts[1], location: parts[3], resource: rest[0]} + if len(rest) == itemParts { + rt.name = rest[1] + } + + return rt, true +} + +// knownResource reports whether seg is a resource collection this handler serves. +func knownResource(seg string) bool { + return seg == vaultsColl || seg == operationsSeg +} + +// Matches claims /v1/projects/{p}/locations/{l}/{backupVaults|operations}[/…] +// paths. An operations path is claimed only when this handler has no shared LRO +// registry (a standalone package server); in an assembled server the shared +// poller owns it. +func (h *Handler) Matches(r *http.Request) bool { + rt, ok := parseRoute(r.URL.Path) + if !ok { + return false + } + + if rt.resource == operationsSeg && h.ops != nil { + return false + } + + return true +} + +// ServeHTTP routes on the parsed path and method. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rt, ok := parseRoute(r.URL.Path) + if !ok { + gcprest.WriteError(w, http.StatusNotFound, "notFound", "unrecognized Backup and DR path") + return + } + + if rt.resource == operationsSeg { + h.serveOperation(w, r) + return + } + + if rt.name == "" { + h.serveCollection(w, r, rt) + return + } + + h.serveItem(w, r, rt) +} + +// serveCollection dispatches collection-level requests (create, list). +func (h *Handler) serveCollection(w http.ResponseWriter, r *http.Request, rt route) { + switch r.Method { + case http.MethodPost: + h.createVault(w, r, rt) + case http.MethodGet: + h.listVaults(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +// serveItem dispatches item-level requests (get, patch, delete). +func (h *Handler) serveItem(w http.ResponseWriter, r *http.Request, rt route) { + switch r.Method { + case http.MethodGet: + h.getVault(w, r, rt) + case http.MethodPatch: + h.patchVault(w, r, rt) + case http.MethodDelete: + h.deleteVault(w, r, rt) + default: + writeMethodNotAllowed(w) + } +} + +// resourceName builds the full backup vault resource name. +func resourceName(project, location, id string) string { + return "projects/" + project + "/locations/" + location + "/" + vaultsColl + "/" + id +} + +func writeMethodNotAllowed(w http.ResponseWriter) { + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") +} diff --git a/server/gcp/backupdr/matches_test.go b/server/gcp/backupdr/matches_test.go new file mode 100644 index 000000000..61ad246b6 --- /dev/null +++ b/server/gcp/backupdr/matches_test.go @@ -0,0 +1,68 @@ +package backupdr + +import ( + "net/http" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + backupdrprovider "github.com/stackshy/cloudemu/v2/providers/gcp/backupdr" + "github.com/stackshy/cloudemu/v2/server/gcp/lro" +) + +func newHandler() *Handler { + return New(backupdrprovider.New(config.NewOptions(config.WithProjectID("p")))) +} + +func request(method, path string) *http.Request { + r, _ := http.NewRequest(method, "http://x"+path, nil) + + return r +} + +func TestMatchesNarrowing(t *testing.T) { + h := newHandler() + + cases := []struct { + name string + path string + want bool + }{ + {"vaults collection", "/v1/projects/p/locations/us-central1/backupVaults", true}, + {"vault item", "/v1/projects/p/locations/us-central1/backupVaults/v", true}, + {"wildcard location list", "/v1/projects/p/locations/-/backupVaults", true}, + {"nested dataSources (out of scope)", "/v1/projects/p/locations/us-central1/backupVaults/v/dataSources", false}, + {"custom verb (out of scope)", "/v1/projects/p/locations/us-central1/backupVaults:fetchUsable", false}, + {"backupPlans (out of scope)", "/v1/projects/p/locations/us-central1/backupPlans", false}, + {"endpoints space (cloudids)", "/v1/projects/p/locations/us-central1/endpoints/e", false}, + {"instances space (memorystore/filestore)", "/v1/projects/p/locations/us-central1/instances/i", false}, + {"bare location", "/v1/projects/p/locations/us-central1", false}, + {"non-v1 path", "/v2/projects/p/locations/us-central1/backupVaults", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := h.Matches(request(http.MethodGet, tc.path)); got != tc.want { + t.Fatalf("Matches(%s) = %v, want %v", tc.path, got, tc.want) + } + }) + } +} + +// TestMatchesOperationsYieldToPoller verifies the handler claims operation polls +// only when it has no shared LRO registry: a standalone package server answers +// its own polls, while in an assembled server the shared poller wins. +func TestMatchesOperationsYieldToPoller(t *testing.T) { + standalone := newHandler() + + opPath := "/v1/projects/p/locations/us-central1/operations/op-1" + if !standalone.Matches(request(http.MethodGet, opPath)) { + t.Fatalf("standalone handler should claim its own operation polls") + } + + shared := newHandler() + shared.SetOperationRegistry(lro.NewRegistry()) + + if shared.Matches(request(http.MethodGet, opPath)) { + t.Fatalf("handler with shared registry must yield operation polls to the poller") + } +} diff --git a/server/gcp/backupdr/operations.go b/server/gcp/backupdr/operations.go new file mode 100644 index 000000000..767542e46 --- /dev/null +++ b/server/gcp/backupdr/operations.go @@ -0,0 +1,245 @@ +package backupdr + +import ( + "net/http" + "strconv" + "strings" + + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +const ( + defaultPageSize = 500 + maxPageSize = 500 + + paramValidateOnly = "validateOnly" + paramForce = "force" + paramAllowMissing = "allowMissing" + paramEtag = "etag" + paramUpdateMask = "updateMask" +) + +// createVault handles POST .../backupVaults?backupVaultId=. The id is the query +// param, falling back to the trailing segment of the body name. requestId is +// accepted and ignored (every CloudEmu create completes inline, so there is no +// in-flight request to deduplicate); validateOnly runs every check without +// storing the vault. +func (h *Handler) createVault(w http.ResponseWriter, r *http.Request, rt route) { + in, ok := decodeVault(w, r) + if !ok { + return + } + + validateOnly, ok := boolParam(w, r, paramValidateOnly) + if !ok { + return + } + + id := r.URL.Query().Get(vaultIDParam) + if id == "" { + id = lastSegment(in.Name) + } + + if id == "" { + gcprest.WriteError(w, http.StatusBadRequest, "invalidArgument", vaultIDParam+" is required") + return + } + + cfg := in.toConfig(rt.project, rt.location, id) + cfg.ValidateOnly = validateOnly + + v, op, err := h.db.CreateBackupVault(r.Context(), cfg) + if err != nil { + writeErr(w, err) + return + } + + h.writeVaultOperation(w, op, v) +} + +// getVault handles GET .../backupVaults/{id}. The view param is accepted and +// ignored: BASIC and FULL render identically for a vault with no data sources. +func (h *Handler) getVault(w http.ResponseWriter, r *http.Request, rt route) { + v, err := h.db.GetBackupVault(r.Context(), rt.project, rt.location, rt.name) + if err != nil { + writeErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, toVaultJSON(v)) +} + +// listVaults handles GET .../backupVaults, scoped to the request's project and +// location ("-" for every location) and ordered by resource name. filter and +// orderBy are accepted and ignored. +func (h *Handler) listVaults(w http.ResponseWriter, r *http.Request, rt route) { + all, err := h.db.ListBackupVaults(r.Context(), rt.project, rt.location) + if err != nil { + writeErr(w, err) + return + } + + page, err := pagination.PaginateSorted(all, + func(a, b bdrdriver.BackupVault) bool { + return resourceName(a.Project, a.Location, a.ID) < resourceName(b.Project, b.Location, b.ID) + }, + r.URL.Query().Get("pageToken"), pageSize(r)) + if err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "invalid pageToken") + return + } + + items := make([]vaultJSON, 0, len(page.Items)) + for i := range page.Items { + items = append(items, toVaultJSON(&page.Items[i])) + } + + gcprest.WriteJSON(w, http.StatusOK, listJSON{BackupVaults: items, NextPageToken: page.NextPageToken}) +} + +// patchVault handles PATCH .../backupVaults/{id}?updateMask=. The mask is +// required; only masked fields change. A body etag, when present, must match +// the stored vault (409 ABORTED otherwise). force, forceUpdateAccessRestriction +// and requestId are accepted and ignored (there are no backup plans or data +// sources to check against). +func (h *Handler) patchVault(w http.ResponseWriter, r *http.Request, rt route) { + in, ok := decodeVault(w, r) + if !ok { + return + } + + validateOnly, ok := boolParam(w, r, paramValidateOnly) + if !ok { + return + } + + cfg := in.toConfig(rt.project, rt.location, rt.name) + cfg.Etag = in.Etag + cfg.ValidateOnly = validateOnly + + v, op, err := h.db.UpdateBackupVault(r.Context(), cfg, parseMask(r.URL.Query().Get(paramUpdateMask))) + if err != nil { + writeErr(w, err) + return + } + + h.writeVaultOperation(w, op, v) +} + +// deleteVault handles DELETE .../backupVaults/{id}. force, allowMissing, etag +// and validateOnly are honored; ignoreBackupPlanReferences and requestId are +// accepted and ignored (there are no backup plans). The operation completes +// inline with an empty response. +func (h *Handler) deleteVault(w http.ResponseWriter, r *http.Request, rt route) { + req := &bdrdriver.DeleteBackupVaultRequest{ + Project: rt.project, Location: rt.location, ID: rt.name, + Etag: r.URL.Query().Get(paramEtag), + } + + flags := []struct { + param string + dst *bool + }{ + {paramForce, &req.Force}, + {paramAllowMissing, &req.AllowMissing}, + {paramValidateOnly, &req.ValidateOnly}, + } + + for _, f := range flags { + v, ok := boolParam(w, r, f.param) + if !ok { + return + } + + *f.dst = v + } + + op, err := h.db.DeleteBackupVault(r.Context(), req) + if err != nil { + writeErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, h.doneOperation(op.Name, nil)) +} + +// serveOperation resolves a (done) long-running operation poll for a +// standalone package server (no shared registry). The operation resource name +// is the request path without the /v1/ version prefix. +func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMethodNotAllowed(w) + return + } + + name := strings.TrimPrefix(r.URL.Path, "/v1/") + + op, err := h.db.GetOperation(r.Context(), name) + if err != nil { + writeErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true}) +} + +// boolParam reads an optional boolean query parameter; a malformed value is a +// 400. +func boolParam(w http.ResponseWriter, r *http.Request, name string) (value, ok bool) { + raw := r.URL.Query().Get(name) + if raw == "" { + return false, true + } + + v, err := strconv.ParseBool(raw) + if err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalidArgument", "invalid boolean for "+name+": "+raw) + return false, false + } + + return v, true +} + +// parseMask splits a comma-separated updateMask query param into field paths. +func parseMask(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + + for _, p := range parts { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + + return out +} + +// lastSegment returns the trailing path segment of a resource name. +func lastSegment(name string) string { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[i+1:] + } + + return name +} + +// pageSize reads ?pageSize, clamping to a sane default and ceiling. +func pageSize(r *http.Request) int { + n, err := strconv.Atoi(r.URL.Query().Get("pageSize")) + if err != nil || n <= 0 { + return defaultPageSize + } + + if n > maxPageSize { + return maxPageSize + } + + return n +} diff --git a/server/gcp/backupdr/sdk_roundtrip_test.go b/server/gcp/backupdr/sdk_roundtrip_test.go new file mode 100644 index 000000000..a2113e55c --- /dev/null +++ b/server/gcp/backupdr/sdk_roundtrip_test.go @@ -0,0 +1,347 @@ +package backupdr_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + backupdr "google.golang.org/api/backupdr/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + gcpprovider "github.com/stackshy/cloudemu/v2/providers/gcp" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +const ( + sdkProject = "mock-project" + sdkLocation = "us-central1" + retention = "86400s" + maxPolls = 10 +) + +type sdkEnv struct { + svc *backupdr.Service + cloud *gcpprovider.Provider + clock *config.FakeClock + parent string +} + +func newSDKEnv(t *testing.T) *sdkEnv { + t.Helper() + + clk := config.NewFakeClock(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + cloud := cloudemu.NewGCP(config.WithClock(clk)) + + ts := httptest.NewServer(gcpserver.NewFromProvider(cloud)) + t.Cleanup(ts.Close) + + svc, err := backupdr.NewService(context.Background(), + option.WithEndpoint(ts.URL+"/"), + option.WithoutAuthentication(), + ) + if err != nil { + t.Fatalf("backupdr.NewService: %v", err) + } + + return &sdkEnv{ + svc: svc, cloud: cloud, clock: clk, + parent: "projects/" + sdkProject + "/locations/" + sdkLocation, + } +} + +// wait polls an operation through Projects.Locations.Operations.Get (the shared +// LRO poller in the assembled server) until it reports done. +func (e *sdkEnv) wait(t *testing.T, op *backupdr.Operation) *backupdr.Operation { + t.Helper() + + for range maxPolls { + polled, err := e.svc.Projects.Locations.Operations.Get(op.Name).Do() + if err != nil { + t.Fatalf("Operations.Get(%s): %v", op.Name, err) + } + + if polled.Done { + if polled.Error != nil { + t.Fatalf("operation %s failed: %+v", op.Name, polled.Error) + } + + return polled + } + } + + t.Fatalf("operation %s never completed", op.Name) + + return nil +} + +func (e *sdkEnv) create(t *testing.T, id string, v *backupdr.BackupVault) *backupdr.Operation { + t.Helper() + + op, err := e.svc.Projects.Locations.BackupVaults.Create(e.parent, v).BackupVaultId(id).Do() + if err != nil { + t.Fatalf("BackupVaults.Create(%s): %v", id, err) + } + + return e.wait(t, op) +} + +func (e *sdkEnv) get(t *testing.T, name string) *backupdr.BackupVault { + t.Helper() + + v, err := e.svc.Projects.Locations.BackupVaults.Get(name).Do() + if err != nil { + t.Fatalf("BackupVaults.Get(%s): %v", name, err) + } + + return v +} + +// wantCode asserts err is a *googleapi.Error carrying the HTTP status code. +func wantCode(t *testing.T, what string, err error, code int) { + t.Helper() + + var gerr *googleapi.Error + if !errors.As(err, &gerr) || gerr.Code != code { + t.Fatalf("%s: err = %v, want HTTP %d", what, err, code) + } +} + +// TestSDKBackupVaultLifecycle drives create (LRO + poll), get, list, patch and +// delete through the real google.golang.org/api/backupdr/v1 client. +func TestSDKBackupVaultLifecycle(t *testing.T) { + e := newSDKEnv(t) + name := e.parent + "/backupVaults/vault-a" + + done := e.create(t, "vault-a", &backupdr.BackupVault{ + Description: "primary vault", + Labels: map[string]string{"env": "dev"}, + BackupMinimumEnforcedRetentionDuration: retention, + }) + + var fromOp backupdr.BackupVault + if err := json.Unmarshal(done.Response, &fromOp); err != nil || fromOp.Name != name { + t.Fatalf("operation response = %s (err %v), want vault %s", done.Response, err, name) + } + + got := e.get(t, name) + assertCreated(t, got, name) + + // Patch only the description; labels stay, etag and updateTime rotate. + e.clock.Advance(time.Minute) + + patchOp, err := e.svc.Projects.Locations.BackupVaults.Patch(name, &backupdr.BackupVault{ + Description: "renamed", + Labels: map[string]string{"ignored": "yes"}, + Etag: got.Etag, + }).UpdateMask("description").Do() + if err != nil { + t.Fatalf("BackupVaults.Patch: %v", err) + } + + e.wait(t, patchOp) + + patched := e.get(t, name) + if patched.Description != "renamed" || patched.Labels["env"] != "dev" || patched.Labels["ignored"] != "" { + t.Fatalf("masked patch changed the wrong fields: %+v", patched) + } + + if patched.Etag == got.Etag || patched.UpdateTime == got.UpdateTime || patched.CreateTime != got.CreateTime { + t.Fatalf("etag/updateTime not rotated or createTime moved: before=%+v after=%+v", got, patched) + } + + // A stale etag is rejected with 409 ABORTED. + _, err = e.svc.Projects.Locations.BackupVaults.Patch(name, &backupdr.BackupVault{ + Description: "lost update", Etag: got.Etag, + }).UpdateMask("description").Do() + wantCode(t, "stale-etag patch", err, http.StatusConflict) + + delOp, err := e.svc.Projects.Locations.BackupVaults.Delete(name).Etag(patched.Etag).Do() + if err != nil { + t.Fatalf("BackupVaults.Delete: %v", err) + } + + e.wait(t, delOp) + + _, err = e.svc.Projects.Locations.BackupVaults.Get(name).Do() + wantCode(t, "get after delete", err, http.StatusNotFound) +} + +func assertCreated(t *testing.T, got *backupdr.BackupVault, name string) { + t.Helper() + + if got.Name != name || got.Description != "primary vault" || got.Labels["env"] != "dev" { + t.Fatalf("body not round-tripped: %+v", got) + } + + if got.BackupMinimumEnforcedRetentionDuration != retention { + t.Fatalf("retention = %q, want %q", got.BackupMinimumEnforcedRetentionDuration, retention) + } + + if got.State != "ACTIVE" || !got.Deletable || got.AccessRestriction != "WITHIN_ORGANIZATION" { + t.Fatalf("state/deletable/accessRestriction = %q/%v/%q", got.State, got.Deletable, got.AccessRestriction) + } + + if got.BackupCount != 0 || got.TotalStoredBytes != 0 { + t.Fatalf("usage = %d/%d, want 0/0", got.BackupCount, got.TotalStoredBytes) + } + + if !strings.HasPrefix(got.ServiceAccount, "service-") || + !strings.HasSuffix(got.ServiceAccount, "@gcp-sa-backupdr-pr.iam.gserviceaccount.com") { + t.Fatalf("serviceAccount = %q", got.ServiceAccount) + } + + if got.Uid == "" || got.Etag == "" || got.CreateTime != "2026-03-04T05:06:07Z" || got.UpdateTime != got.CreateTime { + t.Fatalf("uid/etag/timestamps = %q/%q/%q/%q", got.Uid, got.Etag, got.CreateTime, got.UpdateTime) + } +} + +// TestSDKBackupVaultListPaging lists across two pages with pageSize=1 and +// confirms filter/orderBy are accepted and the "-" location wildcard spans +// locations. +func TestSDKBackupVaultListPaging(t *testing.T) { + e := newSDKEnv(t) + + for _, id := range []string{"vault-b", "vault-a"} { + e.create(t, id, &backupdr.BackupVault{BackupMinimumEnforcedRetentionDuration: retention}) + } + + var names []string + + err := e.svc.Projects.Locations.BackupVaults.List(e.parent).PageSize(1). + Filter(`state="ACTIVE"`).OrderBy("name"). + Pages(context.Background(), func(resp *backupdr.ListBackupVaultsResponse) error { + if len(resp.BackupVaults) != 1 { + t.Fatalf("page size = %d, want 1", len(resp.BackupVaults)) + } + + names = append(names, resp.BackupVaults[0].Name) + + return nil + }) + if err != nil { + t.Fatalf("List.Pages: %v", err) + } + + if len(names) != 2 || !strings.HasSuffix(names[0], "/vault-a") || !strings.HasSuffix(names[1], "/vault-b") { + t.Fatalf("paged names = %v, want [vault-a vault-b]", names) + } + + all, err := e.svc.Projects.Locations.BackupVaults.List("projects/" + sdkProject + "/locations/-").Do() + if err != nil || len(all.BackupVaults) != 2 { + t.Fatalf("wildcard list = %+v (err %v)", all, err) + } +} + +// TestSDKBackupVaultValidation covers the 400/404/409 error paths. +func TestSDKBackupVaultValidation(t *testing.T) { + e := newSDKEnv(t) + vaults := e.svc.Projects.Locations.BackupVaults + + for _, bad := range []string{"", "one day", "-10s"} { + _, err := vaults.Create(e.parent, &backupdr.BackupVault{ + BackupMinimumEnforcedRetentionDuration: bad, + }).BackupVaultId("bad-retention").Do() + wantCode(t, "create retention "+bad, err, http.StatusBadRequest) + } + + e.create(t, "vault-v", &backupdr.BackupVault{BackupMinimumEnforcedRetentionDuration: retention}) + + _, err := vaults.Create(e.parent, &backupdr.BackupVault{ + BackupMinimumEnforcedRetentionDuration: retention, + }).BackupVaultId("vault-v").Do() + wantCode(t, "duplicate create", err, http.StatusConflict) + + _, err = vaults.Get(e.parent + "/backupVaults/ghost").Do() + wantCode(t, "get missing", err, http.StatusNotFound) + + name := e.parent + "/backupVaults/vault-v" + + for _, mask := range []string{"", "state", "bogusField"} { + _, err = vaults.Patch(name, &backupdr.BackupVault{Description: "x"}).UpdateMask(mask).Do() + wantCode(t, "patch mask "+mask, err, http.StatusBadRequest) + } + + _, err = vaults.Patch(name, &backupdr.BackupVault{ + BackupMinimumEnforcedRetentionDuration: "-1s", + }).UpdateMask("backupMinimumEnforcedRetentionDuration").Do() + wantCode(t, "patch negative retention", err, http.StatusBadRequest) + + _, err = vaults.Patch(e.parent+"/backupVaults/ghost", &backupdr.BackupVault{Description: "x"}). + UpdateMask("description").Do() + wantCode(t, "patch missing", err, http.StatusNotFound) + + // An operation name nobody created is 404 from the shared poller. + _, err = e.svc.Projects.Locations.Operations.Get(e.parent + "/operations/never-created").Do() + wantCode(t, "poll unknown operation", err, http.StatusNotFound) +} + +// TestSDKBackupVaultValidateOnly confirms validateOnly checks without storing. +func TestSDKBackupVaultValidateOnly(t *testing.T) { + e := newSDKEnv(t) + vaults := e.svc.Projects.Locations.BackupVaults + + op, err := vaults.Create(e.parent, &backupdr.BackupVault{ + BackupMinimumEnforcedRetentionDuration: retention, + }).BackupVaultId("dry-run").ValidateOnly(true).RequestId("5f1c7c1e-1b1a-4d6e-9a53-6f0b6f2b8d11").Do() + if err != nil { + t.Fatalf("validateOnly create: %v", err) + } + + e.wait(t, op) + + _, err = vaults.Get(e.parent + "/backupVaults/dry-run").Do() + wantCode(t, "get after validateOnly create", err, http.StatusNotFound) + + _, err = vaults.Create(e.parent, &backupdr.BackupVault{}).BackupVaultId("dry-run").ValidateOnly(true).Do() + wantCode(t, "validateOnly create without retention", err, http.StatusBadRequest) +} + +// TestSDKBackupVaultDeleteGuards covers force (non-empty vault), allowMissing +// and a stale delete etag. +func TestSDKBackupVaultDeleteGuards(t *testing.T) { + e := newSDKEnv(t) + vaults := e.svc.Projects.Locations.BackupVaults + name := e.parent + "/backupVaults/vault-full" + + e.create(t, "vault-full", &backupdr.BackupVault{BackupMinimumEnforcedRetentionDuration: retention}) + + if err := e.cloud.BackupDR.SetUsage(sdkProject, sdkLocation, "vault-full", 2, 4096); err != nil { + t.Fatalf("SetUsage: %v", err) + } + + full := e.get(t, name) + if full.Deletable || full.BackupCount != 2 || full.TotalStoredBytes != 4096 { + t.Fatalf("seeded usage not reported: %+v", full) + } + + _, err := vaults.Delete(name).Do() + wantCode(t, "delete non-empty without force", err, http.StatusBadRequest) + + _, err = vaults.Delete(name).Force(true).Etag("stale").Do() + wantCode(t, "delete stale etag", err, http.StatusConflict) + + op, err := vaults.Delete(name).Force(true).IgnoreBackupPlanReferences(true).Do() + if err != nil { + t.Fatalf("force delete: %v", err) + } + + e.wait(t, op) + + _, err = vaults.Delete(name).Do() + wantCode(t, "delete missing", err, http.StatusNotFound) + + op, err = vaults.Delete(name).AllowMissing(true).Do() + if err != nil { + t.Fatalf("allowMissing delete: %v", err) + } + + e.wait(t, op) +} diff --git a/server/gcp/backupdr/wire.go b/server/gcp/backupdr/wire.go new file mode 100644 index 000000000..dbea55e32 --- /dev/null +++ b/server/gcp/backupdr/wire.go @@ -0,0 +1,214 @@ +package backupdr + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +// maxBodyBytes caps a decoded request body. +const maxBodyBytes = 8 << 20 + +// decimalBase formats the int64-as-string output fields. +const decimalBase = 10 + +// encryptionJSON mirrors backupdr/v1 EncryptionConfig. +type encryptionJSON struct { + KmsKeyName string `json:"kmsKeyName,omitempty"` +} + +// vaultInput is the caller-settable subset of a backupdr/v1 BackupVault body. +// Output-only keys (state, serviceAccount, uid, …) are simply not decoded, so a +// caller cannot pin them. Etag is read for the patch concurrency check. +type vaultInput struct { + Name string `json:"name"` + Description string `json:"description"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + BackupMinimumEnforcedRetentionDuration string `json:"backupMinimumEnforcedRetentionDuration"` + BackupRetentionInheritance string `json:"backupRetentionInheritance"` + EffectiveTime string `json:"effectiveTime"` + AccessRestriction string `json:"accessRestriction"` + EncryptionConfig *encryptionJSON `json:"encryptionConfig"` + Etag string `json:"etag"` +} + +// toConfig maps the decoded body onto a driver config for the given identity. +func (in *vaultInput) toConfig(project, location, id string) *bdrdriver.BackupVaultConfig { + cfg := &bdrdriver.BackupVaultConfig{ + Project: project, + Location: location, + ID: id, + Description: in.Description, + Labels: in.Labels, + Annotations: in.Annotations, + BackupMinimumEnforcedRetentionDuration: in.BackupMinimumEnforcedRetentionDuration, + BackupRetentionInheritance: in.BackupRetentionInheritance, + EffectiveTime: in.EffectiveTime, + AccessRestriction: in.AccessRestriction, + } + + if in.EncryptionConfig != nil { + cfg.EncryptionConfig = &bdrdriver.EncryptionConfig{KmsKeyName: in.EncryptionConfig.KmsKeyName} + } + + return cfg +} + +// vaultJSON is the backupdr/v1 BackupVault wire shape. backupCount and +// totalStoredBytes are int64 fields, which the JSON mapping renders as strings +// (the SDK decodes them with `json:",string"`); they and deletable are always +// emitted so a client sees "0"/true on an empty vault. +type vaultJSON struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + BackupMinimumEnforcedRetentionDuration string `json:"backupMinimumEnforcedRetentionDuration,omitempty"` + BackupRetentionInheritance string `json:"backupRetentionInheritance,omitempty"` + EffectiveTime string `json:"effectiveTime,omitempty"` + AccessRestriction string `json:"accessRestriction,omitempty"` + EncryptionConfig *encryptionJSON `json:"encryptionConfig,omitempty"` + State string `json:"state"` + ServiceAccount string `json:"serviceAccount"` + UID string `json:"uid"` + Etag string `json:"etag"` + Deletable bool `json:"deletable"` + BackupCount string `json:"backupCount"` + TotalStoredBytes string `json:"totalStoredBytes"` + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` +} + +// vaultAny is a vault wrapped as a google.protobuf.Any, the shape a completed +// operation's `response` carries. +type vaultAny struct { + Type string `json:"@type"` + vaultJSON +} + +// listJSON is ListBackupVaultsResponse. +type listJSON struct { + BackupVaults []vaultJSON `json:"backupVaults"` + NextPageToken string `json:"nextPageToken,omitempty"` +} + +// operationJSON mirrors google.longrunning.Operation. Mutating ops complete +// inline, so `done` is always true; `response` carries the resulting vault (an +// Any for create/patch, absent for delete). +type operationJSON struct { + Name string `json:"name"` + Done bool `json:"done"` + Response json.RawMessage `json:"response,omitempty"` +} + +// decodeVault reads the request body once into the caller-settable fields. +func decodeVault(w http.ResponseWriter, r *http.Request) (*vaultInput, bool) { + raw, err := io.ReadAll(io.LimitReader(r.Body, maxBodyBytes)) + if err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "reading request body: "+err.Error()) + return nil, false + } + + in := &vaultInput{} + + if len(raw) > 0 { + if err := json.Unmarshal(raw, in); err != nil { + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "malformed JSON body: "+err.Error()) + return nil, false + } + } + + return in, true +} + +// toVaultJSON renders a driver vault as backupdr/v1 wire JSON. +func toVaultJSON(v *bdrdriver.BackupVault) vaultJSON { + out := vaultJSON{ + Name: resourceName(v.Project, v.Location, v.ID), + Description: v.Description, + Labels: v.Labels, + Annotations: v.Annotations, + BackupMinimumEnforcedRetentionDuration: v.BackupMinimumEnforcedRetentionDuration, + BackupRetentionInheritance: v.BackupRetentionInheritance, + EffectiveTime: v.EffectiveTime, + AccessRestriction: v.AccessRestriction, + State: v.State, + ServiceAccount: v.ServiceAccount, + UID: v.UID, + Etag: v.Etag, + Deletable: v.Deletable(), + BackupCount: strconv.FormatInt(v.BackupCount, decimalBase), + TotalStoredBytes: strconv.FormatInt(v.TotalStoredBytes, decimalBase), + CreateTime: formatTime(v.CreateTime), + UpdateTime: formatTime(v.UpdateTime), + } + + if v.EncryptionConfig != nil { + out.EncryptionConfig = &encryptionJSON{KmsKeyName: v.EncryptionConfig.KmsKeyName} + } + + return out +} + +// writeVaultOperation writes a completed operation carrying the vault as its +// Any-typed response (create/patch). +func (h *Handler) writeVaultOperation(w http.ResponseWriter, op *bdrdriver.Operation, v *bdrdriver.BackupVault) { + raw, err := json.Marshal(vaultAny{Type: vaultTypeURL, vaultJSON: toVaultJSON(v)}) + if err != nil { + gcprest.WriteError(w, http.StatusInternalServerError, "internalError", err.Error()) + return + } + + gcprest.WriteJSON(w, http.StatusOK, h.doneOperation(op.Name, raw)) +} + +// doneOperation builds a completed google.longrunning.Operation and records it +// with the shared LRO poller (a no-op on a nil registry) so a client polling +// the returned name resolves the same done operation (with its response). +func (h *Handler) doneOperation(name string, resp json.RawMessage) operationJSON { + if h.ops != nil { + // A nil RawMessage stored in the registry's `any` would be a non-nil + // interface and replay as "response": null; register a true nil instead. + if resp == nil { + h.ops.Register(name, nil) + } else { + h.ops.Register(name, resp) + } + } + + return operationJSON{Name: name, Done: true, Response: resp} +} + +// writeErr maps a driver error onto the Google JSON error envelope. A stale +// etag is 409 ABORTED; a vault that still holds backups is FAILED_PRECONDITION, +// which the Google REST mapping renders as HTTP 400; everything else follows +// the shared cerrors mapping. +func writeErr(w http.ResponseWriter, err error) { + msg := cerrors.Message(err) + + switch { + case errors.Is(err, bdrdriver.ErrEtagMismatch): + gcprest.WriteError(w, http.StatusConflict, "ABORTED", msg) + case cerrors.IsFailedPrecondition(err): + gcprest.WriteError(w, http.StatusBadRequest, "FAILED_PRECONDITION", msg) + default: + gcprest.WriteCErr(w, err) + } +} + +// formatTime renders t as RFC3339Nano; a zero time renders as the empty string. +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + + return t.UTC().Format(time.RFC3339Nano) +} diff --git a/server/gcp/dispatch_ordering_test.go b/server/gcp/dispatch_ordering_test.go index 32f23f2bb..29e0c8b0f 100644 --- a/server/gcp/dispatch_ordering_test.go +++ b/server/gcp/dispatch_ordering_test.go @@ -49,6 +49,7 @@ func TestSpecificHandlersWinBeforeFirestore(t *testing.T) { }{ {"pubsub_topics_before_firestore", "/v1/projects/demo/topics", "topics"}, {"cloudfunctions_before_firestore", "/v1/projects/demo/locations/us-central1/functions", "functions"}, + {"backupdr_vaults_before_firestore", "/v1/projects/demo/locations/us-central1/backupVaults", "backupVaults"}, {"iam_serviceaccounts_before_firestore", "/v1/projects/demo/serviceAccounts", "accounts"}, {"secretmanager_before_firestore", "/v1/projects/demo/secrets", "secrets"}, } diff --git a/server/gcp/from_provider.go b/server/gcp/from_provider.go index c06bc32e4..f52adbc99 100644 --- a/server/gcp/from_provider.go +++ b/server/gcp/from_provider.go @@ -36,6 +36,7 @@ func DriversFrom(p *gcpprovider.Provider) Drivers { VPCAccess: p.VPCAccess, CloudIDS: p.CloudIDS, ManagedKafka: p.ManagedKafka, + BackupDR: p.BackupDR, SecureSourceManager: p.SecureSourceManager, NetworkConnectivity: p.NetworkConnectivity, Composer: p.Composer, diff --git a/server/gcp/gcp.go b/server/gcp/gcp.go index d154a867e..ebc285110 100644 --- a/server/gcp/gcp.go +++ b/server/gcp/gcp.go @@ -17,6 +17,7 @@ import ( alloydbsrv "github.com/stackshy/cloudemu/v2/server/gcp/alloydb" apigatewaysrv "github.com/stackshy/cloudemu/v2/server/gcp/apigateway" "github.com/stackshy/cloudemu/v2/server/gcp/artifactregistry" + backupdrsrv "github.com/stackshy/cloudemu/v2/server/gcp/backupdr" bigqueryserver "github.com/stackshy/cloudemu/v2/server/gcp/bigquery" bigtableserver "github.com/stackshy/cloudemu/v2/server/gcp/bigtable" binauthzsrv "github.com/stackshy/cloudemu/v2/server/gcp/binaryauthorization" @@ -72,6 +73,7 @@ import ( "github.com/stackshy/cloudemu/v2/server/wire/gcprest" acmdriver "github.com/stackshy/cloudemu/v2/services/accesscontextmanager/driver" agdriver "github.com/stackshy/cloudemu/v2/services/apigatewaygcp/driver" + backupdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" bqdriver "github.com/stackshy/cloudemu/v2/services/bigquery/driver" btdriver "github.com/stackshy/cloudemu/v2/services/bigtable/driver" badriver "github.com/stackshy/cloudemu/v2/services/binaryauthorization/driver" @@ -247,6 +249,13 @@ type Drivers struct { // genuinely-Kafka traffic (content+ownership); its location-scoped operation // polls are owned by the shared LRO poller. ManagedKafka mkdriver.ManagedKafka + // BackupDR serves the backupdr.googleapis.com v1 Backup and DR backup vault + // control plane against the backupdr driver. Its paths live under + // /v1/projects/{p}/locations/{l}/backupVaults[/…]; the handler's Matches + // narrows on the backupVaults resource segment, so it is disjoint from every + // other /v1/projects/ handler, and its location-scoped operation polls are + // owned by the shared LRO poller. + BackupDR backupdrdriver.BackupDR // SecureSourceManager serves the securesourcemanager.googleapis.com v1 // instance + repository control plane against the securesourcemanager driver. // Its paths live under /v1/projects/{p}/locations/{l}/{instances|repositories} @@ -735,6 +744,18 @@ func New(d Drivers) *server.Server { srv.Register(cloudidsH) } + // BackupDR matches /v1/projects/{p}/locations/{l}/backupVaults[/…]. Its + // backupVaults resource-segment guard is disjoint from every other + // /v1/projects/ handler, so registration order among them is unconstrained; + // registered after the shared LRO poller (which owns its operation polls, and + // which the handler's Matches yields to) and before Firestore's permissive + // prefix. + if d.BackupDR != nil { + backupdrH := backupdrsrv.New(d.BackupDR) + backupdrH.SetOperationRegistry(opsReg) + srv.Register(backupdrH) + } + // Data Fusion (datafusion.googleapis.com) shares the EXACT same instances path // grammar as Secure Source Manager (below), Memorystore, and Filestore. It // registers BEFORE all of them so its narrow Matches wins: it claims only diff --git a/server/gcp/operations_ownership_test.go b/server/gcp/operations_ownership_test.go index d0710e50a..07440c2fb 100644 --- a/server/gcp/operations_ownership_test.go +++ b/server/gcp/operations_ownership_test.go @@ -196,3 +196,34 @@ func TestFullServerManagedKafkaSharesClustersWithGKE(t *testing.T) { t.Fatalf("bogus op GET: code=%d, want 404", code) } } + +// TestFullServerBackupDROperationsResolveThroughSharedPoller proves a Backup and +// DR vault operation is recorded with the shared lro registry (not answered by +// a greedy sibling handler): the poll returns done with the typed BackupVault +// response, cancel/delete act on the real record, and a later poll 404s. +func TestFullServerBackupDROperationsResolveThroughSharedPoller(t *testing.T) { + ts := fullServer(t) + + code, body := do(t, ts, http.MethodPost, + "/v1/projects/demo/locations/us-central1/backupVaults?backupVaultId=vault-ops", + `{"backupMinimumEnforcedRetentionDuration":"86400s"}`) + if code != http.StatusOK { + t.Fatalf("BackupDR create: code=%d body=%s", code, body) + } + + op := "/v1/" + opName(t, body) + + code, body = do(t, ts, http.MethodGet, op, "") + if code != http.StatusOK || !strings.Contains(body, `"done":true`) || + !strings.Contains(body, "google.cloud.backupdr.v1.BackupVault") { + t.Fatalf("BackupDR op GET: code=%d body=%s (want 200 done with BackupVault response)", code, body) + } + + if code, body := do(t, ts, http.MethodDelete, op, ""); code != http.StatusOK { + t.Fatalf("BackupDR op delete: code=%d body=%s (want 200)", code, body) + } + + if code, _ := do(t, ts, http.MethodGet, op, ""); code != http.StatusNotFound { + t.Fatalf("BackupDR op GET after delete: code=%d (want 404)", code) + } +} diff --git a/services/backupdr/driver/driver.go b/services/backupdr/driver/driver.go new file mode 100644 index 000000000..8af19f52b --- /dev/null +++ b/services/backupdr/driver/driver.go @@ -0,0 +1,133 @@ +// Package driver defines the portable interface for the Google Cloud Backup and +// DR control plane (backupdr.googleapis.com/v1). It is control-plane only: the +// region-scoped backup vault collection a google.golang.org/api/backupdr/v1 +// client or the Terraform google provider's google_backup_dr_backup_vault +// resource CRUDs is modeled: +// +// projects/{p}/locations/{region}/backupVaults/{id} +// +// and the long-running operations its mutating RPCs return, which are +// location-scoped and share the operations space the shared GCP LRO poller owns: +// +// projects/{p}/locations/{region}/operations/{op} +// +// There is no data plane: no data sources, backups, backup plans, management +// servers or restores. A vault therefore always reports backupCount and +// totalStoredBytes of 0 and deletable=true, unless a test seeds usage through +// the provider mock's SetUsage hook to exercise the non-empty delete guard. +package driver + +import ( + "context" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +// ErrEtagMismatch is the error (wrapped) a patch or delete returns when the +// caller supplied an etag that no longer matches the stored vault. Real Backup +// and DR rejects that write so a concurrent update is never silently +// overwritten; wire layers map it to 409 ABORTED. +var ErrEtagMismatch = cerrors.New(cerrors.FailedPrecondition, "etag does not match the current backup vault etag") + +// EncryptionConfig mirrors the vault's optional customer-managed encryption key. +type EncryptionConfig struct { + KmsKeyName string +} + +// BackupVault is one backup vault. Name components are stored separately so +// the full resource name and location scoping can be rebuilt without re-parsing. +// Every output-only field (State, ServiceAccount, UID, Etag, BackupCount, +// TotalStoredBytes, CreateTime, UpdateTime) is minted by the provider. +type BackupVault struct { + Project string + Location string + ID string + + Description string + Labels map[string]string + Annotations map[string]string + BackupMinimumEnforcedRetentionDuration string + BackupRetentionInheritance string + EffectiveTime string + AccessRestriction string + EncryptionConfig *EncryptionConfig + + State string + ServiceAccount string + UID string + Etag string + Revision int64 + BackupCount int64 + TotalStoredBytes int64 + CreateTime time.Time + UpdateTime time.Time +} + +// Deletable reports whether the vault holds no backups, the condition real +// Backup and DR exposes as the output-only `deletable` field. +func (v *BackupVault) Deletable() bool { return v.BackupCount == 0 } + +// BackupVaultConfig is the input to a create or patch. Etag is only consulted +// by a patch: when non-empty it must match the stored vault's etag. +// ValidateOnly runs every check without mutating state. +type BackupVaultConfig struct { + Project string + Location string + ID string + + Description string + Labels map[string]string + Annotations map[string]string + BackupMinimumEnforcedRetentionDuration string + BackupRetentionInheritance string + EffectiveTime string + AccessRestriction string + EncryptionConfig *EncryptionConfig + + Etag string + ValidateOnly bool +} + +// DeleteBackupVaultRequest carries the identity and the delete options real +// Backup and DR accepts. Force deletes a vault that still holds backups; +// AllowMissing turns a delete of an absent vault into a no-op success; Etag, +// when non-empty, must match the stored vault's etag. +type DeleteBackupVaultRequest struct { + Project string + Location string + ID string + Etag string + Force bool + AllowMissing bool + ValidateOnly bool +} + +// Operation is a completed long-running operation. Every CloudEmu mutation +// finishes synchronously, so Done is always true; the shared poller replays it +// so an SDK or Terraform LRO wait terminates on the first poll. +type Operation struct { + Name string // projects/{p}/locations/{region}/operations/{op} + Done bool + TargetName string // the backup vault the operation acted on + Type string // create | update | delete +} + +// BackupDR is the control-plane interface a provider implements for the backup +// vaults collection. +type BackupDR interface { + // CreateBackupVault creates a vault; the retention duration is required. + CreateBackupVault(ctx context.Context, cfg *BackupVaultConfig) (*BackupVault, *Operation, error) + // GetBackupVault returns one vault by identity. + GetBackupVault(ctx context.Context, project, location, id string) (*BackupVault, error) + // ListBackupVaults lists a project's vaults in a location ("-" for every location). + ListBackupVaults(ctx context.Context, project, location string) ([]BackupVault, error) + // UpdateBackupVault applies a field-masked update and rotates the etag. + UpdateBackupVault(ctx context.Context, cfg *BackupVaultConfig, mask []string) (*BackupVault, *Operation, error) + // DeleteBackupVault deletes a vault, honoring force, allowMissing and etag. + DeleteBackupVault(ctx context.Context, req *DeleteBackupVaultRequest) (*Operation, error) + + // GetOperation resolves a (done) long-running operation by name, for a + // standalone package server's own operations poll. + GetOperation(ctx context.Context, name string) (*Operation, error) +} From 8aa02872cca216b82ea26e2332908fa4804a0084 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sat, 26 Sep 2026 20:43:13 +0530 Subject: [PATCH 03/24] feat(azure): API Management service control-plane (Microsoft.ApiManagement/service) Service CRUD via the real armapimanagement/v3 ServiceClient: CreateOrUpdate, Get, Update (PATCH: properties merged key-by-key, tags/zones replaced when sent, sku/identity re-resolved when sent), Delete, ListByResourceGroup and List (subscription). LROs complete synchronously: PUT 201/200 and PATCH 200 carry provisioningState=Succeeded with no polling headers, DELETE is 200/204. Validation (400 InvalidParameter): location, sku.name (SDK SKUType enum), sku.capacity (Consumption = 0, other tiers >= 1), properties.publisherEmail and publisherName, service name (1-50 chars, starts with a letter, letters/ digits/hyphens, no trailing hyphen). A PATCH is re-validated after merge. Computed, stable fields: etag, createdAtUtc, provisioningState, the system-assigned identity principalId/tenantId, and the gateway / portal / developer portal / management / scm URLs derived from the name (Consumption reports only the gateway). Wired into the RG-purge cascade, snapshot/restore, resource discovery and Resource Graph. Out of scope: child resources (apis, products, subscriptions, policies, backends, ...), the gateway data plane, backup/restore, network configuration updates, soft-deleted services, and global name uniqueness. --- docs/coverage/README.md | 1 + docs/coverage/azure/README.md | 1 + docs/coverage/azure/apimanagement.md | 23 + docs/coverage/coverage.json | 39 ++ docs/sdk-server.md | 1 + go.mod | 1 + go.sum | 2 + .../azure/apimanagement/apimanagement.go | 428 +++++++++++++++++ .../azure/apimanagement/apimanagement_test.go | 298 ++++++++++++ providers/azure/apimanagement/snapshot.go | 61 +++ providers/azure/apimanagement/validate.go | 144 ++++++ providers/azure/apimanagement_discovery.go | 47 ++ providers/azure/azure.go | 4 + .../apimanagement/apimanagement_sdk_test.go | 431 ++++++++++++++++++ server/azure/apimanagement/handler.go | 213 +++++++++ server/azure/apimanagement/types.go | 241 ++++++++++ server/azure/azure.go | 20 + server/azure/from_provider.go | 1 + server/azure/resourcegraph/handler.go | 1 + server/azure/resourcegraph/kql.go | 3 + services/resourcediscovery/walkers.go | 8 + 21 files changed, 1968 insertions(+) create mode 100644 docs/coverage/azure/apimanagement.md create mode 100644 providers/azure/apimanagement/apimanagement.go create mode 100644 providers/azure/apimanagement/apimanagement_test.go create mode 100644 providers/azure/apimanagement/snapshot.go create mode 100644 providers/azure/apimanagement/validate.go create mode 100644 providers/azure/apimanagement_discovery.go create mode 100644 server/azure/apimanagement/apimanagement_sdk_test.go create mode 100644 server/azure/apimanagement/handler.go create mode 100644 server/azure/apimanagement/types.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index ea4d4ba4a..ab223d9f8 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -18,6 +18,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `apigateway` | [APIGateway](./aws/apigateway.md) | - | - | - | 29 | | `apigatewaygcp` | - | - | [APIGateway](./gcp/apigateway.md) | - | 16 | | `apigatewayv2` | [APIGatewayV2](./aws/apigatewayv2.md) | - | - | - | 20 | +| `apimanagement` | - | [APIManagement](./azure/apimanagement.md) | - | - | 10 | | `appconfiguration` | - | [AppConfiguration](./azure/appconfiguration.md) | - | - | 9 | | `appflow` | [AppFlow](./aws/appflow.md) | - | - | - | 14 | | `appinsights` | - | [Appinsights](./azure/appinsights.md) | - | - | 6 | diff --git a/docs/coverage/azure/README.md b/docs/coverage/azure/README.md index 8f0b55897..62735d62d 100644 --- a/docs/coverage/azure/README.md +++ b/docs/coverage/azure/README.md @@ -8,6 +8,7 @@ Services cloudemu emulates for Azure, by native name. Back to the [cross-provide | [ACR](./acr.md) | `containerregistry` | 15 | | [AI](./ai.md) | `azureai` | 92 | | [AKS](./aks.md) | (provider-native) | 18 | +| [APIManagement](./apimanagement.md) | (provider-native) | 10 | | [Aad](./aad.md) | (provider-native) | 2 | | [AppConfiguration](./appconfiguration.md) | (provider-native) | 9 | | [AppGateway](./appgateway.md) | `applicationgateway` | 4 | diff --git a/docs/coverage/azure/apimanagement.md b/docs/coverage/azure/apimanagement.md new file mode 100644 index 000000000..2fb1c9c00 --- /dev/null +++ b/docs/coverage/azure/apimanagement.md @@ -0,0 +1,23 @@ + +# APIManagement + +provider-native `apimanagement` wire service (Azure-only) · no portable driver · [Azure index](./README.md) + +## Operations (10) + +| Operation | Description | +| --- | --- | +| `ARMID` | | +| `CreateOrUpdateService` | | +| `DeleteService` | | +| `DiscoverServices` | | +| `Endpoints` | | +| `GetService` | | +| `ListServicesByResourceGroup` | | +| `ListServicesBySubscription` | | +| `PurgeResourceGroup` | | +| `UpdateService` | | + +## Not in scope + +_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index bc507f626..03614c172 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -494,6 +494,45 @@ "aws": "APIGatewayV2" } }, + { + "service": "apimanagement", + "interface": "provider-native", + "operations": [ + { + "name": "ARMID" + }, + { + "name": "CreateOrUpdateService" + }, + { + "name": "DeleteService" + }, + { + "name": "DiscoverServices" + }, + { + "name": "Endpoints" + }, + { + "name": "GetService" + }, + { + "name": "ListServicesByResourceGroup" + }, + { + "name": "ListServicesBySubscription" + }, + { + "name": "PurgeResourceGroup" + }, + { + "name": "UpdateService" + } + ], + "providers": { + "azure": "APIManagement" + } + }, { "service": "appconfiguration", "interface": "provider-native", diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 84257550b..2a36153bf 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -208,6 +208,7 @@ All handlers speak ARM JSON over HTTPS unless noted. | **IAM (armauthorization)** | `Microsoft.Authorization`: RoleDefinitions (CreateOrUpdate, Get, List, Delete) and RoleAssignments (Create, Get, ListForScope, Delete) at any scope (subscription, resource group, resource, management group). Real `armauthorization` SDK clients round-trip end-to-end. Microsoft Graph (users/groups) is not implemented yet. | | **Resource Graph** | `Microsoft.ResourceGraph`: `POST /providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01` with a KQL-shaped query over the cross-service inventory; supports `subscriptions[]` scoping and `$top`/`$skipToken` pagination. Rows carry the fixed columns (`id` [ARM-shaped], `name`, `type`, `location`, `resourceGroup`, `subscriptionId`, `tags`) plus resource-shape columns emitted when present; `sku.name`, `properties`, `managedBy`, `kind`, `zones`; so SKU/tier/size-sensitive consumers (e.g. a discovery + cost engine) can read a VM's size, a managed disk's tier/`diskSizeGB`/owning VM, or a flexible server's compute SKU. `project`/`summarize`/`join` are tolerated but ignored (the full row is always returned). | | **Databricks (ARM control plane)** | `Microsoft.Databricks/workspaces`: CreateOrUpdate, Get, Delete, UpdateTags, List / ListByResourceGroup. Real `armdatabricks` SDK clients round-trip end-to-end. | +| **API Management** | `Microsoft.ApiManagement/service`: CreateOrUpdate, Get, Update (PATCH: properties merged, tags replaced when sent), Delete, ListByResourceGroup, List. SKU (Developer/Basic/Standard/Premium/Consumption/Isolated/BasicV2/StandardV2) + capacity validated (Consumption = 0, others >= 1), `publisherEmail`/`publisherName` required, service name 1-50 chars; `gatewayUrl`/`portalUrl`/`developerPortalUrl`/`managementApiUrl`/`scmUrl` derived from the name, stable `createdAtUtc`/etag/system-assigned identity. LROs complete synchronously. Real `armapimanagement/v3` `ServiceClient` round-trips end-to-end. Child resources (apis, products, policies, ...) and backup/restore are out of scope. | | **Databricks (workspace data plane)** *(`databricks-sdk-go`, `/api/2.x`)* | Point the real `WorkspaceClient` at `Config.Host`. Clusters (create/edit/start/restart/resize/pin/unpin/delete + list-node-types / spark-versions / zones), instance pools, jobs + runs (submit / run-now / get / list / cancel / cancel-all / repair / output / delete), cluster policies, libraries (install / uninstall / status), and object permissions. Self-contained families: secrets (scopes / secrets / ACLs), tokens, git credentials, repos, DBFS (incl. block upload), workspace notebooks/directories, SQL warehouses, pipelines, serving endpoints, SCIM identity (users / groups / service principals), and Unity Catalog (catalogs / schemas / tables + metastores / external locations / storage credentials / volumes). Also serves `GET /.well-known/databricks-config` so the SDK's host-metadata resolution succeeds (workspace-host stub) instead of logging a warning. | ### GCP (`server/gcp/`) diff --git a/go.mod b/go.mod index f72168235..9cdebd0b5 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/containers/azcontainerregistry v0.2.3 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.4.2 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v3 v3.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.3 diff --git a/go.sum b/go.sum index db65c75d3..444cf6647 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 h1:j0hhYS006eJ54vusoa github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1/go.mod h1:AdtInaXmK8eYmbjezRWgLz+Qs46nc9Up9GWGwteWNfw= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0 h1:5aCs3yc/Ftlv/Le+Dr0P+oOuo/clB6fsfFwrm+DbNms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0/go.mod h1:4/JvtEOgU0r2lkVz8BImrFd0e7ZNQ74x+3k9swJzSmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 h1:1PD0CnFSl1m1TCwudP3cIiyTABCWVzHXtYc6Vi5J0JY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0/go.mod h1:xCGT95xV5ei4ahSgJWy31pPGE3xWfaWpr9uRzwTzsmg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v3 v3.0.0 h1:FErSe/vQGefbSuVwBV9JlrRXgG1uOFyW6TCXERX89s4= diff --git a/providers/azure/apimanagement/apimanagement.go b/providers/azure/apimanagement/apimanagement.go new file mode 100644 index 000000000..a85130eeb --- /dev/null +++ b/providers/azure/apimanagement/apimanagement.go @@ -0,0 +1,428 @@ +// Package apimanagement provides an in-memory mock of Azure API Management +// (Microsoft.ApiManagement/service), the ARM control plane only. It manages the +// service lifecycle (create-or-update, get, patch, delete, list-by-group, +// list-by-subscription), the service SKU (name + capacity), availability zones +// and the system/user-assigned managed identity. +// +// The API Management data plane (the gateway that proxies traffic, the +// developer portal) and the service's child resources (apis, products, +// subscriptions, policies, named values, backends, loggers, ...) are out of +// scope, as are backup/restore, network-configuration updates and the +// soft-deleted services (deletedservices) surface. +// +// Every service-minted field stays stable for the lifetime of the resource so +// infrastructure-as-code tools (Terraform's azurerm_api_management) see no drift +// on re-plan: id/name, provisioningState ("Succeeded"), createdAtUtc, etag and +// the system-assigned identity's principalId/tenantId are minted once at create +// and byte-stable across every read and patch. The endpoint host names +// (gateway, portal, developer portal, management, scm) derive from the service +// name, exactly as Azure derives them. +// +// The writable properties block (publisherEmail, publisherName and every other +// caller-set property) is stored as raw JSON and round-trips verbatim. +package apimanagement + +import ( + "context" + "encoding/json" + "maps" + "sort" + "strings" + "sync" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/memstore" +) + +const ( + // providerNamespace is the ARM provider namespace. + providerNamespace = "Microsoft.ApiManagement" + // serviceType is the ARM service resource-type segment. + serviceType = "service" + + // stateSucceeded is the terminal provisioningState a synchronous ARM PUT + // reaches immediately. + stateSucceeded = "Succeeded" + + // emulatorTenantID is the single Azure AD directory (tenant) that all + // system-assigned identities in this emulator belong to. + emulatorTenantID = "11111111-1111-1111-1111-111111111111" + + // hostSuffix is the DNS suffix every API Management endpoint lives under. + hostSuffix = ".azure-api.net" +) + +// ManagedIdentity is a service's top-level managed identity. For a +// system-assigned identity the PrincipalID/TenantID are synthesized once (as +// Azure mints them on assignment) and stay stable; UserAssignedIDs holds the +// assigned user-identity resource ids. +type ManagedIdentity struct { + Type string `json:"type"` + PrincipalID string `json:"principalId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + UserAssignedIDs []string `json:"userAssignedIds,omitempty"` +} + +// Service is a stored Microsoft.ApiManagement/service resource. Subscription, +// ResourceGroup and Name preserve the caller's casing; the computed fields are +// minted at create and never regenerated on a read. Properties holds the +// writable properties block and round-trips verbatim. +type Service struct { + Subscription string `json:"subscription"` + ResourceGroup string `json:"resourceGroup"` + Name string `json:"name"` + Location string `json:"location"` + Tags map[string]string `json:"tags,omitempty"` + Zones []string `json:"zones,omitempty"` + + SkuName string `json:"skuName"` + SkuCapacity int32 `json:"skuCapacity"` + Identity *ManagedIdentity `json:"identity,omitempty"` + + Properties json.RawMessage `json:"properties,omitempty"` + + // Computed, stable fields. + ProvisioningState string `json:"provisioningState"` + Etag string `json:"etag"` + CreatedAt time.Time `json:"createdAt"` +} + +// ARMID returns the fully-qualified ARM resource id of the service. +func (s *Service) ARMID() string { + return idgen.AzureID(s.Subscription, s.ResourceGroup, providerNamespace, serviceType, s.Name) +} + +// Endpoints are the host URLs Azure derives from a service name. +type Endpoints struct { + Gateway string // https://.azure-api.net (the proxy) + Portal string // legacy publisher portal + DeveloperPortal string // developer portal + ManagementAPI string // direct management REST endpoint + Scm string // git configuration (SCM) endpoint +} + +// Endpoints returns the service's endpoint URLs. +func (s *Service) Endpoints() Endpoints { + return Endpoints{ + Gateway: s.endpoint(""), + Portal: s.endpoint(".portal"), + DeveloperPortal: s.endpoint(".developer"), + ManagementAPI: s.endpoint(".management"), + Scm: s.endpoint(".scm"), + } +} + +// endpoint renders https://.azure-api.net. Azure host names are +// lowercase whatever casing the caller used for the service name. +func (s *Service) endpoint(infix string) string { + return "https://" + strings.ToLower(s.Name) + infix + hostSuffix +} + +// ServiceInput carries the mutable fields of a service create/update request. +// A nil pointer/map/slice means "not supplied": on a PATCH the stored value is +// preserved, so the request overlays only what it names. +type ServiceInput struct { + Tags map[string]string + Zones []string + SkuName *string + SkuCapacity *int32 + Identity *ManagedIdentity + Properties json.RawMessage +} + +// Mock is the in-memory backend for API Management services. +type Mock struct { + mu sync.RWMutex + clock config.Clock + services *memstore.Store[*Service] +} + +// New creates an empty API Management mock. It falls back to the real clock +// when opts (or its clock) is nil so the mock stays usable standalone. +func New(opts *config.Options) *Mock { + clock := config.Clock(config.RealClock{}) + if opts != nil && opts.Clock != nil { + clock = opts.Clock + } + + return &Mock{clock: clock, services: memstore.New[*Service]()} +} + +// serviceKey is the case-insensitive store key for a service. +func serviceKey(sub, rg, name string) string { + return strings.ToLower(idgen.AzureID(sub, rg, providerNamespace, serviceType, name)) +} + +// CreateOrUpdateService creates a new service or replaces an existing one (ARM +// PUT semantics: tags, zones, identity and the properties block are replaced +// wholesale). The computed fields (provisioningState, etag, createdAtUtc) are +// minted once at create and preserved across updates; location is immutable in +// real Azure and is preserved on update. It returns the stored service and +// whether it was newly created. +func (m *Mock) CreateOrUpdateService( + _ context.Context, sub, rg, name, location string, in *ServiceInput, +) (Service, bool, error) { + if err := validateCreate(sub, rg, name, location, in); err != nil { + return Service{}, false, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + k := serviceKey(sub, rg, name) + + existing, existed := m.services.Get(k) + + var s Service + if existed { + s = *existing + } else { + s = m.newService(sub, rg, name, location) + } + + s.Tags = maps.Clone(in.Tags) + s.Zones = append([]string(nil), in.Zones...) + s.SkuName = canonicalSKU(*in.SkuName) + s.SkuCapacity = *in.SkuCapacity + s.Identity = resolveIdentity(in.Identity, sub, rg, name) + s.Properties = append(json.RawMessage(nil), in.Properties...) + + m.services.Set(k, &s) + + return cloneService(&s), !existed, nil +} + +// UpdateService applies an ARM PATCH: tags and zones are replaced wholesale +// when supplied, sku/identity are re-resolved only when supplied, and the +// properties block is merged key-by-key onto the stored block. The merged +// result is re-validated, so a PATCH cannot blank the publisher fields or leave +// an invalid SKU/capacity pair. A PATCH on a missing service is a NotFound. +func (m *Mock) UpdateService(_ context.Context, sub, rg, name string, in *ServiceInput) (Service, error) { + m.mu.Lock() + defer m.mu.Unlock() + + k := serviceKey(sub, rg, name) + + existing, ok := m.services.Get(k) + if !ok { + return Service{}, notFound(name) + } + + s := *existing + applyPatch(&s, in, sub, rg, name) + + if err := validateSKU(s.SkuName, s.SkuCapacity); err != nil { + return Service{}, err + } + + if err := validatePublisher(s.Properties); err != nil { + return Service{}, err + } + + m.services.Set(k, &s) + + return cloneService(&s), nil +} + +// applyPatch overlays the supplied PATCH fields onto s. A nil pointer/map/slice +// preserves the stored value. +func applyPatch(s *Service, in *ServiceInput, sub, rg, name string) { + if in.Tags != nil { + s.Tags = maps.Clone(in.Tags) + } + + if in.Zones != nil { + s.Zones = append([]string(nil), in.Zones...) + } + + if in.SkuName != nil { + s.SkuName = canonicalSKU(*in.SkuName) + } + + if in.SkuCapacity != nil { + s.SkuCapacity = *in.SkuCapacity + } + + if in.Identity != nil { + s.Identity = resolveIdentity(in.Identity, sub, rg, name) + } + + if in.Properties != nil { + s.Properties = mergeRaw(s.Properties, in.Properties) + } +} + +// newService seeds a fresh service with its immutable identity and its +// computed, stable fields. The etag derives deterministically from the resource +// id so it is stable yet distinct per service. +func (m *Mock) newService(sub, rg, name, location string) Service { + return Service{ + Subscription: sub, + ResourceGroup: rg, + Name: name, + Location: location, + ProvisioningState: stateSucceeded, + Etag: idgen.SyntheticGUID("apimanagement/etag/" + serviceKey(sub, rg, name)), + CreatedAt: m.clock.Now().UTC().Truncate(time.Second), + } +} + +// GetService returns the service, or a NotFound error. +func (m *Mock) GetService(_ context.Context, sub, rg, name string) (Service, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + s, ok := m.services.Get(serviceKey(sub, rg, name)) + if !ok { + return Service{}, notFound(name) + } + + return cloneService(s), nil +} + +// DeleteService removes the service, reporting whether it existed. +func (m *Mock) DeleteService(_ context.Context, sub, rg, name string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + return m.services.Delete(serviceKey(sub, rg, name)), nil +} + +// ListServicesByResourceGroup returns every service in the group, sorted by +// name. +func (m *Mock) ListServicesByResourceGroup(_ context.Context, sub, rg string) ([]Service, error) { + return m.filterServices(func(s *Service) bool { + return strings.EqualFold(s.Subscription, sub) && strings.EqualFold(s.ResourceGroup, rg) + }), nil +} + +// ListServicesBySubscription returns every service in the subscription, sorted +// by name. +func (m *Mock) ListServicesBySubscription(_ context.Context, sub string) ([]Service, error) { + return m.filterServices(func(s *Service) bool { + return strings.EqualFold(s.Subscription, sub) + }), nil +} + +// DiscoverServices returns every stored service, for the inventory walk. +func (m *Mock) DiscoverServices(_ context.Context) ([]Service, error) { + return m.filterServices(func(*Service) bool { return true }), nil +} + +// PurgeResourceGroup deletes every service under sub/rg, so a resource-group +// delete cascades into its API Management services. +func (m *Mock) PurgeResourceGroup(_ context.Context, sub, rg string) error { + m.mu.Lock() + defer m.mu.Unlock() + + for k, s := range m.services.All() { + if strings.EqualFold(s.Subscription, sub) && strings.EqualFold(s.ResourceGroup, rg) { + m.services.Delete(k) + } + } + + return nil +} + +// filterServices returns the services matching pred, sorted by name. +func (m *Mock) filterServices(pred func(*Service) bool) []Service { + m.mu.RLock() + defer m.mu.RUnlock() + + var out []Service + + for _, s := range m.services.All() { + if pred(s) { + out = append(out, cloneService(s)) + } + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + + return out +} + +// resolveIdentity normalizes an incoming managed identity: for a +// system-assigned identity it synthesizes deterministic principal/tenant GUIDs +// (as Azure does on assignment); a nil or "None" identity resolves to nil. +func resolveIdentity(in *ManagedIdentity, sub, rg, name string) *ManagedIdentity { + if in == nil || in.Type == "" || strings.EqualFold(in.Type, "None") { + return nil + } + + out := &ManagedIdentity{ + Type: in.Type, + UserAssignedIDs: append([]string(nil), in.UserAssignedIDs...), + } + sort.Strings(out.UserAssignedIDs) + + if strings.Contains(strings.ToLower(in.Type), "systemassigned") { + // Keyed on the full resource id so two services with the same name in + // different groups stay distinct, while the value is stable across + // gets/patches/restarts for the same service. + out.PrincipalID = idgen.SyntheticGUID("apimanagement/principal/" + serviceKey(sub, rg, name)) + out.TenantID = emulatorTenantID + } + + return out +} + +// notFound is the NotFound error for a missing service. +func notFound(name string) error { + return cerrors.Newf(cerrors.NotFound, "API Management service %q not found", name) +} + +// cloneService deep-copies a stored service so callers never alias the store. +func cloneService(s *Service) Service { + out := *s + out.Tags = maps.Clone(s.Tags) + out.Zones = append([]string(nil), s.Zones...) + out.Identity = cloneIdentity(s.Identity) + + if s.Properties != nil { + out.Properties = append(json.RawMessage(nil), s.Properties...) + } + + return out +} + +// cloneIdentity deep-copies a managed identity, or returns nil. +func cloneIdentity(id *ManagedIdentity) *ManagedIdentity { + if id == nil { + return nil + } + + out := *id + out.UserAssignedIDs = append([]string(nil), id.UserAssignedIDs...) + + return &out +} + +// mergeRaw overlays the top-level keys of patch onto base and returns the +// merged raw JSON object. A malformed base or patch falls back to whichever +// side parses, so a merge never drops the caller's bytes silently. +func mergeRaw(base, patch json.RawMessage) json.RawMessage { + merged := map[string]json.RawMessage{} + if len(base) > 0 { + if err := json.Unmarshal(base, &merged); err != nil { + merged = map[string]json.RawMessage{} + } + } + + overlay := map[string]json.RawMessage{} + if err := json.Unmarshal(patch, &overlay); err != nil { + return append(json.RawMessage(nil), patch...) + } + + maps.Copy(merged, overlay) + + raw, err := json.Marshal(merged) + if err != nil { + return append(json.RawMessage(nil), patch...) + } + + return raw +} diff --git a/providers/azure/apimanagement/apimanagement_test.go b/providers/azure/apimanagement/apimanagement_test.go new file mode 100644 index 000000000..1df498b96 --- /dev/null +++ b/providers/azure/apimanagement/apimanagement_test.go @@ -0,0 +1,298 @@ +package apimanagement_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" +) + +const ( + sub = "sub" + rg = "rg" +) + +//nolint:gochecknoglobals // fixed test clock origin +var epoch = time.Date(2026, 9, 1, 10, 30, 0, 0, time.UTC) + +func newMock() (*apimanagement.Mock, *config.FakeClock) { + clk := config.NewFakeClock(epoch) + return apimanagement.New(config.NewOptions(config.WithClock(clk))), clk +} + +func sptr(v string) *string { return &v } + +func i32(v int32) *int32 { return &v } + +func devInput() *apimanagement.ServiceInput { + return &apimanagement.ServiceInput{ + Tags: map[string]string{"env": "dev"}, + SkuName: sptr("Developer"), + SkuCapacity: i32(1), + Identity: &apimanagement.ManagedIdentity{Type: "SystemAssigned"}, + Properties: json.RawMessage(`{"publisherEmail":"a@b.test","publisherName":"Contoso","customProperties":{"k":"v"}}`), + } +} + +func create(t *testing.T, m *apimanagement.Mock, name string) apimanagement.Service { + t.Helper() + + s, created, err := m.CreateOrUpdateService(context.Background(), sub, rg, name, "East US", devInput()) + if err != nil || !created { + t.Fatalf("create %s: err=%v created=%v", name, err, created) + } + + return s +} + +func TestCreateComputedFields(t *testing.T) { + m, _ := newMock() + s := create(t, m, "Apim1") + + if s.ProvisioningState != "Succeeded" || s.Etag == "" { + t.Fatalf("computed fields not minted: %+v", s) + } + + if !s.CreatedAt.Equal(epoch) { + t.Errorf("createdAt = %v, want %v", s.CreatedAt, epoch) + } + + if s.Identity == nil || s.Identity.PrincipalID == "" || s.Identity.TenantID == "" { + t.Fatalf("system-assigned identity not synthesized: %+v", s.Identity) + } + + if got := s.Endpoints().Gateway; got != "https://apim1.azure-api.net" { + t.Errorf("gateway url = %q", got) + } + + if got := s.Endpoints().Scm; got != "https://apim1.scm.azure-api.net" { + t.Errorf("scm url = %q", got) + } + + want := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.ApiManagement/service/Apim1" + if s.ARMID() != want { + t.Errorf("ARM id = %q, want %q", s.ARMID(), want) + } +} + +func TestReplaceKeepsComputedFieldsAndLocation(t *testing.T) { + m, clk := newMock() + first := create(t, m, "apim1") + + clk.Advance(time.Hour) + + in := devInput() + in.Tags = nil + in.SkuName = sptr("premium") + in.SkuCapacity = i32(2) + + got, created, err := m.CreateOrUpdateService(context.Background(), sub, rg, "apim1", "West Europe", in) + if err != nil || created { + t.Fatalf("replace: err=%v created=%v", err, created) + } + + if !got.CreatedAt.Equal(first.CreatedAt) || got.Etag != first.Etag || got.Location != "East US" { + t.Errorf("replace changed stable fields: %+v", got) + } + + if got.SkuName != "Premium" || got.SkuCapacity != 2 { + t.Errorf("sku = %s/%d, want Premium/2", got.SkuName, got.SkuCapacity) + } + + if got.Tags != nil { + t.Errorf("PUT without tags must clear them, got %v", got.Tags) + } + + if got.Identity.PrincipalID != first.Identity.PrincipalID { + t.Error("principalId changed across a replace") + } +} + +func TestCreateValidation(t *testing.T) { + cases := map[string]struct { + name string + location string + mutate func(*apimanagement.ServiceInput) + }{ + "missing location": {"svc", "", nil}, + "name starts digit": {"1svc", "eastus", nil}, + "name trailing hyphen": {"svc-", "eastus", nil}, + "name bad char": {"svc_1", "eastus", nil}, + "name too long": {"a" + strings.Repeat("b", 50), "eastus", nil}, + "missing sku": {"svc", "eastus", func(in *apimanagement.ServiceInput) { in.SkuName = nil }}, + "unknown sku": {"svc", "eastus", func(in *apimanagement.ServiceInput) { in.SkuName = sptr("Gold") }}, + "missing capacity": {"svc", "eastus", func(in *apimanagement.ServiceInput) { in.SkuCapacity = nil }}, + "zero capacity dev": {"svc", "eastus", func(in *apimanagement.ServiceInput) { in.SkuCapacity = i32(0) }}, + "consumption capacity1": {"svc", "eastus", func(in *apimanagement.ServiceInput) { in.SkuName = sptr("Consumption") }}, + "missing email": {"svc", "eastus", func(in *apimanagement.ServiceInput) { + in.Properties = json.RawMessage(`{"publisherName":"Contoso"}`) + }}, + "missing publisher name": {"svc", "eastus", func(in *apimanagement.ServiceInput) { + in.Properties = json.RawMessage(`{"publisherEmail":"a@b.test"}`) + }}, + } + + for label, tc := range cases { + t.Run(label, func(t *testing.T) { + m, _ := newMock() + in := devInput() + + if tc.mutate != nil { + tc.mutate(in) + } + + _, _, err := m.CreateOrUpdateService(context.Background(), sub, rg, tc.name, tc.location, in) + if !cerrors.IsInvalidArgument(err) { + t.Fatalf("err = %v, want InvalidArgument", err) + } + }) + } +} + +func TestConsumptionZeroCapacityAccepted(t *testing.T) { + m, _ := newMock() + in := devInput() + in.SkuName = sptr("Consumption") + in.SkuCapacity = i32(0) + + s, _, err := m.CreateOrUpdateService(context.Background(), sub, rg, "serverless", "eastus", in) + if err != nil { + t.Fatalf("create consumption: %v", err) + } + + if s.SkuName != "Consumption" || s.SkuCapacity != 0 { + t.Errorf("sku = %s/%d", s.SkuName, s.SkuCapacity) + } +} + +func TestPatchMergesPropertiesAndReplacesTags(t *testing.T) { + m, _ := newMock() + create(t, m, "apim1") + + got, err := m.UpdateService(context.Background(), sub, rg, "apim1", &apimanagement.ServiceInput{ + Tags: map[string]string{"team": "api"}, + Properties: json.RawMessage(`{"publisherName":"Fabrikam"}`), + }) + if err != nil { + t.Fatalf("patch: %v", err) + } + + if len(got.Tags) != 1 || got.Tags["team"] != "api" { + t.Errorf("tags = %v, want exactly team=api", got.Tags) + } + + var props map[string]any + if err := json.Unmarshal(got.Properties, &props); err != nil { + t.Fatalf("props: %v", err) + } + + if props["publisherName"] != "Fabrikam" || props["publisherEmail"] != "a@b.test" || props["customProperties"] == nil { + t.Errorf("merge lost keys: %v", props) + } + + if got.SkuName != "Developer" || got.SkuCapacity != 1 { + t.Errorf("patch without sku changed it: %s/%d", got.SkuName, got.SkuCapacity) + } +} + +func TestPatchRejectsInvalidResult(t *testing.T) { + m, _ := newMock() + create(t, m, "apim1") + + ctx := context.Background() + + _, err := m.UpdateService(ctx, sub, rg, "apim1", &apimanagement.ServiceInput{SkuName: sptr("Consumption")}) + if !cerrors.IsInvalidArgument(err) { + t.Fatalf("Consumption with capacity 1: err = %v, want InvalidArgument", err) + } + + _, err = m.UpdateService(ctx, sub, rg, "apim1", &apimanagement.ServiceInput{ + Properties: json.RawMessage(`{"publisherEmail":""}`), + }) + if !cerrors.IsInvalidArgument(err) { + t.Fatalf("blank publisherEmail: err = %v, want InvalidArgument", err) + } + + s, err := m.GetService(ctx, sub, rg, "apim1") + if err != nil || s.SkuName != "Developer" { + t.Fatalf("a rejected PATCH must not persist: %+v, %v", s, err) + } + + _, err = m.UpdateService(ctx, sub, rg, "missing", &apimanagement.ServiceInput{}) + if !cerrors.IsNotFound(err) { + t.Fatalf("patch missing: err = %v, want NotFound", err) + } +} + +func TestListDeleteAndPurge(t *testing.T) { + m, _ := newMock() + ctx := context.Background() + + create(t, m, "b-svc") + create(t, m, "a-svc") + + if _, _, err := m.CreateOrUpdateService(ctx, sub, "other", "c-svc", "eastus", devInput()); err != nil { + t.Fatalf("create other: %v", err) + } + + list, _ := m.ListServicesByResourceGroup(ctx, sub, "RG") + if len(list) != 2 || list[0].Name != "a-svc" { + t.Fatalf("list by rg = %+v", list) + } + + all, _ := m.ListServicesBySubscription(ctx, sub) + if len(all) != 3 { + t.Fatalf("list by sub = %d, want 3", len(all)) + } + + if existed, _ := m.DeleteService(ctx, sub, rg, "a-svc"); !existed { + t.Fatal("delete existing reported not existed") + } + + if existed, _ := m.DeleteService(ctx, sub, rg, "a-svc"); existed { + t.Fatal("second delete reported existed") + } + + if _, err := m.GetService(ctx, sub, rg, "a-svc"); !cerrors.IsNotFound(err) { + t.Fatalf("get deleted: err = %v", err) + } + + if err := m.PurgeResourceGroup(ctx, sub, rg); err != nil { + t.Fatalf("purge: %v", err) + } + + left, _ := m.DiscoverServices(ctx) + if len(left) != 1 || left[0].Name != "c-svc" { + t.Fatalf("after purge = %+v, want only c-svc", left) + } +} + +func TestSnapshotRoundTrip(t *testing.T) { + m, _ := newMock() + orig := create(t, m, "apim1") + + data, err := m.Snapshot(context.Background(), false) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + + restored, _ := newMock() + if err := restored.Restore(context.Background(), data); err != nil { + t.Fatalf("restore: %v", err) + } + + got, err := restored.GetService(context.Background(), sub, rg, "apim1") + if err != nil { + t.Fatalf("get restored: %v", err) + } + + if got.Etag != orig.Etag || !got.CreatedAt.Equal(orig.CreatedAt) || + got.Identity.PrincipalID != orig.Identity.PrincipalID || string(got.Properties) != string(orig.Properties) { + t.Errorf("restored service differs:\n got %+v\nwant %+v", got, orig) + } +} diff --git a/providers/azure/apimanagement/snapshot.go b/providers/azure/apimanagement/snapshot.go new file mode 100644 index 000000000..a0bc52a8d --- /dev/null +++ b/providers/azure/apimanagement/snapshot.go @@ -0,0 +1,61 @@ +package apimanagement + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/stackshy/cloudemu/v2/internal/snapshot" +) + +var _ snapshot.Snapshottable = (*Mock)(nil) + +// snapshotState is the on-disk shape: the service store keyed by its +// (lowercased) resource id. +type snapshotState struct { + Services json.RawMessage `json:"services,omitempty"` +} + +// Snapshot captures every API Management service. includeAssets is unused: +// these resources hold no bulk object bodies. +func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + services, err := m.services.Snapshot() + if err != nil { + return nil, fmt.Errorf("apimanagement: snapshot services: %w", err) + } + + data, err := json.Marshal(snapshotState{Services: services}) + if err != nil { + return nil, fmt.Errorf("apimanagement: marshal snapshot: %w", err) + } + + return data, nil +} + +// Restore rebuilds every service under its original id. +func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { + m.mu.Lock() + defer m.mu.Unlock() + + if len(data) == 0 { + return nil + } + + var state snapshotState + if err := json.Unmarshal(data, &state); err != nil { + return fmt.Errorf("apimanagement: unmarshal snapshot: %w", err) + } + + if len(state.Services) == 0 { + return nil + } + + if err := m.services.LoadSnapshot(state.Services); err != nil { + return fmt.Errorf("apimanagement: restore services: %w", err) + } + + return nil +} diff --git a/providers/azure/apimanagement/validate.go b/providers/azure/apimanagement/validate.go new file mode 100644 index 000000000..d163725ce --- /dev/null +++ b/providers/azure/apimanagement/validate.go @@ -0,0 +1,144 @@ +package apimanagement + +import ( + "encoding/json" + "regexp" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +const ( + // maxServiceNameLen is the longest service name Azure accepts. + maxServiceNameLen = 50 + + // skuConsumption is the serverless tier, the only one whose capacity is 0. + skuConsumption = "Consumption" +) + +// serviceNamePattern is Azure's service-name rule: starts with a letter, then +// letters, digits and hyphens, and does not end with a hyphen. +var serviceNamePattern = regexp.MustCompile(`^[A-Za-z]([A-Za-z0-9-]*[A-Za-z0-9])?$`) + +// validSKUs is the armapimanagement v3 SKUType enum, keyed lowercase so the +// lookup is case-insensitive like ARM, mapped to the canonical casing. +// +//nolint:gochecknoglobals // static lookup table +var validSKUs = map[string]string{ + "developer": "Developer", + "basic": "Basic", + "standard": "Standard", + "premium": "Premium", + "consumption": skuConsumption, + "isolated": "Isolated", + "basicv2": "BasicV2", + "standardv2": "StandardV2", +} + +// canonicalSKU returns the canonical casing of a known SKU name, or the input +// unchanged when it is unknown (validation rejects unknown names first). +func canonicalSKU(name string) string { + if c, ok := validSKUs[strings.ToLower(name)]; ok { + return c + } + + return name +} + +// validateCreate rejects a create/replace request with missing or malformed +// required fields: the path identity, location, the SKU block and the two +// publisher properties. +func validateCreate(sub, rg, name, location string, in *ServiceInput) error { + switch { + case sub == "": + return invalid("subscription is required") + case rg == "": + return invalid("resource group is required") + case location == "": + return invalid("location is required") + } + + if err := validateName(name); err != nil { + return err + } + + if in.SkuName == nil || *in.SkuName == "" { + return invalid("sku.name is required") + } + + if in.SkuCapacity == nil { + return invalid("sku.capacity is required") + } + + if err := validateSKU(*in.SkuName, *in.SkuCapacity); err != nil { + return err + } + + return validatePublisher(in.Properties) +} + +// validateName enforces Azure's service-name rule (1-50 characters, starts with +// a letter, letters/digits/hyphens, no trailing hyphen). +func validateName(name string) error { + if name == "" || len(name) > maxServiceNameLen || !serviceNamePattern.MatchString(name) { + return cerrors.Newf(cerrors.InvalidArgument, + "invalid API Management service name %q: it must be 1-%d characters, start with a letter, "+ + "contain only letters, digits and hyphens, and not end with a hyphen", name, maxServiceNameLen) + } + + return nil +} + +// validateSKU checks the SKU name is a known tier and the capacity fits it: the +// Consumption tier must be 0 units, every other tier at least 1. +func validateSKU(name string, capacity int32) error { + canon, ok := validSKUs[strings.ToLower(name)] + if !ok { + return cerrors.Newf(cerrors.InvalidArgument, "invalid sku.name %q", name) + } + + if canon == skuConsumption { + if capacity != 0 { + return cerrors.Newf(cerrors.InvalidArgument, + "sku.capacity must be 0 for the Consumption tier, got %d", capacity) + } + + return nil + } + + if capacity < 1 { + return cerrors.Newf(cerrors.InvalidArgument, + "sku.capacity must be at least 1 for the %s tier, got %d", canon, capacity) + } + + return nil +} + +// validatePublisher requires non-empty properties.publisherEmail and +// properties.publisherName. +func validatePublisher(props json.RawMessage) error { + var p struct { + PublisherEmail string `json:"publisherEmail"` + PublisherName string `json:"publisherName"` + } + + if len(props) > 0 { + if err := json.Unmarshal(props, &p); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "malformed properties: %v", err) + } + } + + switch { + case strings.TrimSpace(p.PublisherEmail) == "": + return invalid("properties.publisherEmail is required") + case strings.TrimSpace(p.PublisherName) == "": + return invalid("properties.publisherName is required") + default: + return nil + } +} + +// invalid is an InvalidArgument error (ARM 400 InvalidParameter). +func invalid(msg string) error { + return cerrors.New(cerrors.InvalidArgument, msg) +} diff --git a/providers/azure/apimanagement_discovery.go b/providers/azure/apimanagement_discovery.go new file mode 100644 index 000000000..10c7edd13 --- /dev/null +++ b/providers/azure/apimanagement_discovery.go @@ -0,0 +1,47 @@ +package azure + +import ( + "context" + + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" + "github.com/stackshy/cloudemu/v2/services/resourcediscovery" +) + +// propProvisioningState is the discovery attribute carrying a resource's ARM +// provisioning state. +const propProvisioningState = "provisioningState" + +// apiManagementDiscovery projects Azure API Management services +// (Microsoft.ApiManagement/service) into the cross-service inventory so they +// surface in Resource Graph / `az resource list`. API Management is Azure-only +// with no shared cross-cloud driver, so this rides the generic projection (like +// recoveryServicesDiscovery) rather than a shared walker. +type apiManagementDiscovery struct{ m *apimanagement.Mock } + +func (d apiManagementDiscovery) DiscoverResources( + ctx context.Context, +) ([]resourcediscovery.DiscoveredResource, error) { + items, err := d.m.DiscoverServices(ctx) + if err != nil { + return nil, err + } + + return projectDiscovery(items, func(s *apimanagement.Service) resourcediscovery.DiscoveredResource { + props := map[string]any{ + propProvisioningState: s.ProvisioningState, + "sku": s.SkuName, + "skuCapacity": s.SkuCapacity, + "gatewayUrl": s.Endpoints().Gateway, + } + + return resourcediscovery.DiscoveredResource{ + Service: resourcediscovery.ServiceAPIManagement, + Type: resourcediscovery.TypeAPIManagementService, + ID: s.Name, + ARN: s.ARMID(), + Region: s.Location, + Tags: s.Tags, + Attrs: resourcediscovery.Attributes{Properties: props}, + } + }), nil +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go index b4dc6c569..a5d268fc1 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -12,6 +12,7 @@ import ( "github.com/stackshy/cloudemu/v2/providers/azure/acr" "github.com/stackshy/cloudemu/v2/providers/azure/ai" "github.com/stackshy/cloudemu/v2/providers/azure/aks" + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" "github.com/stackshy/cloudemu/v2/providers/azure/appconfiguration" "github.com/stackshy/cloudemu/v2/providers/azure/applicationgateway" "github.com/stackshy/cloudemu/v2/providers/azure/bastion" @@ -207,6 +208,7 @@ type Provider struct { RecoveryServices *recoveryservices.Mock IoTHub *iothub.Mock HealthcareApis *healthcareapis.Mock + APIManagement *apimanagement.Mock ResourceDiscovery *resourcediscovery.Engine @@ -289,6 +291,7 @@ func New(opts ...config.Option) *Provider { RecoveryServices: recoveryservices.New(o), IoTHub: iothub.New(o), HealthcareApis: healthcareapis.New(o), + APIManagement: apimanagement.New(o), SubscriptionID: o.AccountID, Region: o.Region, EnforceAuth: o.EnforceAuth, @@ -343,6 +346,7 @@ func New(opts ...config.Option) *Provider { recoveryServicesDiscovery{p.RecoveryServices}, iotHubDiscovery{p.IoTHub}, healthcareApisDiscovery{p.HealthcareApis}, + apiManagementDiscovery{p.APIManagement}, }, }, ) diff --git a/server/azure/apimanagement/apimanagement_sdk_test.go b/server/azure/apimanagement/apimanagement_sdk_test.go new file mode 100644 index 000000000..d616d2a54 --- /dev/null +++ b/server/azure/apimanagement/apimanagement_sdk_test.go @@ -0,0 +1,431 @@ +package apimanagement_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + + "github.com/stackshy/cloudemu/v2" + azureserver "github.com/stackshy/cloudemu/v2/server/azure" +) + +const ( + subID = "00000000-0000-0000-0000-0000000000a1" + rgName = "rg-apim" + rgOther = "rg-apim-other" + svcName = "Contoso-Apim" +) + +// fakeCred is a static-token credential for tests. +type fakeCred struct{} + +func (fakeCred) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "fake", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +type fixture struct { + ts *httptest.Server + svc *armapimanagement.ServiceClient + rgs *armresources.ResourceGroupsClient +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + + ts := httptest.NewTLSServer(azureserver.NewFromProvider(cloudemu.NewAzure())) + t.Cleanup(ts.Close) + + opts := &arm.ClientOptions{ClientOptions: azcore.ClientOptions{ + Cloud: cloud.Configuration{ + ActiveDirectoryAuthorityHost: "https://login.microsoftonline.com/", + Services: map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: {Endpoint: ts.URL, Audience: "https://management.azure.com"}, + }, + }, + Transport: ts.Client(), + Retry: policy.RetryOptions{MaxRetries: -1}, + }} + + cf, err := armapimanagement.NewClientFactory(subID, fakeCred{}, opts) + if err != nil { + t.Fatalf("apim client factory: %v", err) + } + + rgs, err := armresources.NewResourceGroupsClient(subID, fakeCred{}, opts) + if err != nil { + t.Fatalf("resource groups client: %v", err) + } + + f := &fixture{ts: ts, svc: cf.NewServiceClient(), rgs: rgs} + f.ensureRG(t, rgName) + f.ensureRG(t, rgOther) + + return f +} + +func (f *fixture) ensureRG(t *testing.T, name string) { + t.Helper() + + _, err := f.rgs.CreateOrUpdate(context.Background(), name, armresources.ResourceGroup{Location: to.Ptr("eastus")}, nil) + if err != nil { + t.Fatalf("create resource group %s: %v", name, err) + } +} + +func developerService() armapimanagement.ServiceResource { + return armapimanagement.ServiceResource{ + Location: to.Ptr("East US"), + Tags: map[string]*string{"env": to.Ptr("dev")}, + SKU: &armapimanagement.ServiceSKUProperties{ + Name: to.Ptr(armapimanagement.SKUTypeDeveloper), + Capacity: to.Ptr[int32](1), + }, + Identity: &armapimanagement.ServiceIdentity{Type: to.Ptr(armapimanagement.ApimIdentityTypeSystemAssigned)}, + Properties: &armapimanagement.ServiceProperties{ + PublisherEmail: to.Ptr("api@contoso.test"), + PublisherName: to.Ptr("Contoso"), + CustomProperties: map[string]*string{ + "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls10": to.Ptr("false"), + }, + }, + } +} + +func (f *fixture) create( + t *testing.T, rg, name string, body armapimanagement.ServiceResource, +) armapimanagement.ServiceResource { + t.Helper() + + ctx := context.Background() + + poller, err := f.svc.BeginCreateOrUpdate(ctx, rg, name, body, nil) + if err != nil { + t.Fatalf("BeginCreateOrUpdate %s: %v", name, err) + } + + res, err := poller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: time.Millisecond}) + if err != nil { + t.Fatalf("PollUntilDone create %s: %v", name, err) + } + + return res.ServiceResource +} + +// TestSDKServiceLifecycle drives the real armapimanagement ServiceClient +// through create (LRO) -> get -> patch (LRO) -> list (rg + subscription) -> +// delete (LRO) -> get 404. +func TestSDKServiceLifecycle(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + created := f.create(t, rgName, svcName, developerService()) + assertCreated(t, &created) + + got, err := f.svc.Get(ctx, rgName, svcName, nil) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if *got.Etag != *created.Etag || *got.Identity.PrincipalID != *created.Identity.PrincipalID || + !got.Properties.CreatedAtUTC.Equal(*created.Properties.CreatedAtUTC) { + t.Errorf("computed fields drifted between create and get") + } + + patched := patchService(t, f) + assertPatched(t, &patched, &created) + + assertListed(t, f) + + delPoller, err := f.svc.BeginDelete(ctx, rgName, svcName, nil) + if err != nil { + t.Fatalf("BeginDelete: %v", err) + } + + if _, err := delPoller.PollUntilDone(ctx, nil); err != nil { + t.Fatalf("PollUntilDone delete: %v", err) + } + + _, err = f.svc.Get(ctx, rgName, svcName, nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") +} + +func assertCreated(t *testing.T, s *armapimanagement.ServiceResource) { + t.Helper() + + p := s.Properties + + checks := map[string][2]string{ + "type": {deref(s.Type), "Microsoft.ApiManagement/service"}, + "provisioningState": {deref(p.ProvisioningState), "Succeeded"}, + "gatewayUrl": {deref(p.GatewayURL), "https://contoso-apim.azure-api.net"}, + "portalUrl": {deref(p.PortalURL), "https://contoso-apim.portal.azure-api.net"}, + "developerPortalUrl": {deref(p.DeveloperPortalURL), "https://contoso-apim.developer.azure-api.net"}, + "managementApiUrl": {deref(p.ManagementAPIURL), "https://contoso-apim.management.azure-api.net"}, + "scmUrl": {deref(p.ScmURL), "https://contoso-apim.scm.azure-api.net"}, + "publisherEmail": {deref(p.PublisherEmail), "api@contoso.test"}, + "sku": {string(deref(s.SKU.Name)), "Developer"}, + } + + for field, c := range checks { + if c[0] != c[1] { + t.Errorf("%s = %q, want %q", field, c[0], c[1]) + } + } + + if !strings.HasSuffix(deref(s.ID), "/resourceGroups/"+rgName+"/providers/Microsoft.ApiManagement/service/"+svcName) { + t.Errorf("id = %q", deref(s.ID)) + } + + if p.CreatedAtUTC == nil || p.CreatedAtUTC.IsZero() || s.Etag == nil || *s.Etag == "" { + t.Error("createdAtUtc / etag not minted") + } + + if s.Identity == nil || deref(s.Identity.PrincipalID) == "" || deref(s.Identity.TenantID) == "" { + t.Errorf("system-assigned identity not minted: %+v", s.Identity) + } + + if deref(p.CustomProperties["Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls10"]) != "false" { + t.Errorf("customProperties did not round-trip: %v", p.CustomProperties) + } +} + +func patchService(t *testing.T, f *fixture) armapimanagement.ServiceResource { + t.Helper() + + ctx := context.Background() + + poller, err := f.svc.BeginUpdate(ctx, rgName, svcName, armapimanagement.ServiceUpdateParameters{ + Tags: map[string]*string{"team": to.Ptr("api")}, + SKU: &armapimanagement.ServiceSKUProperties{ + Name: to.Ptr(armapimanagement.SKUTypePremium), + Capacity: to.Ptr[int32](2), + }, + Properties: &armapimanagement.ServiceUpdateProperties{PublisherName: to.Ptr("Fabrikam")}, + }, nil) + if err != nil { + t.Fatalf("BeginUpdate: %v", err) + } + + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + t.Fatalf("PollUntilDone update: %v", err) + } + + return res.ServiceResource +} + +func assertPatched(t *testing.T, got, before *armapimanagement.ServiceResource) { + t.Helper() + + if len(got.Tags) != 1 || deref(got.Tags["team"]) != "api" { + t.Errorf("PATCH tags must replace the set, got %v", got.Tags) + } + + if deref(got.SKU.Name) != armapimanagement.SKUTypePremium || deref(got.SKU.Capacity) != 2 { + t.Errorf("sku = %v/%v, want Premium/2", deref(got.SKU.Name), deref(got.SKU.Capacity)) + } + + if deref(got.Properties.PublisherName) != "Fabrikam" || + deref(got.Properties.PublisherEmail) != "api@contoso.test" { + t.Errorf("PATCH properties must merge: name=%q email=%q", + deref(got.Properties.PublisherName), deref(got.Properties.PublisherEmail)) + } + + if len(got.Properties.CustomProperties) != 1 { + t.Errorf("PATCH dropped unnamed properties: %v", got.Properties.CustomProperties) + } + + if deref(got.Identity.PrincipalID) != deref(before.Identity.PrincipalID) || deref(got.Etag) != deref(before.Etag) { + t.Error("PATCH re-minted the identity or etag") + } +} + +func assertListed(t *testing.T, f *fixture) { + t.Helper() + + f.create(t, rgOther, "other-apim", developerService()) + + ctx := context.Background() + + var inRG []string + + for pager := f.svc.NewListByResourceGroupPager(rgName, nil); pager.More(); { + page, err := pager.NextPage(ctx) + if err != nil { + t.Fatalf("list by rg: %v", err) + } + + for _, s := range page.Value { + inRG = append(inRG, *s.Name) + } + } + + if len(inRG) != 1 || inRG[0] != svcName { + t.Errorf("list by rg = %v, want [%s]", inRG, svcName) + } + + total := 0 + + for pager := f.svc.NewListPager(nil); pager.More(); { + page, err := pager.NextPage(ctx) + if err != nil { + t.Fatalf("list by subscription: %v", err) + } + + total += len(page.Value) + } + + if total != 2 { + t.Errorf("list by subscription = %d services, want 2", total) + } +} + +// TestSDKConsumptionTier covers the capacity-0 serverless tier, whose response +// carries only the gateway endpoint. +func TestSDKConsumptionTier(t *testing.T) { + f := newFixture(t) + + body := developerService() + body.SKU = &armapimanagement.ServiceSKUProperties{ + Name: to.Ptr(armapimanagement.SKUTypeConsumption), Capacity: to.Ptr[int32](0), + } + + s := f.create(t, rgName, "serverless", body) + + if deref(s.SKU.Capacity) != 0 || deref(s.Properties.GatewayURL) != "https://serverless.azure-api.net" { + t.Errorf("consumption: capacity=%d gateway=%q", deref(s.SKU.Capacity), deref(s.Properties.GatewayURL)) + } + + if s.Properties.PortalURL != nil || s.Properties.ScmURL != nil { + t.Error("consumption tier must not report portal/scm endpoints") + } +} + +// TestSDKValidationErrors asserts every rejected create surfaces as an ARM 400 +// through the real client. +func TestSDKValidationErrors(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + cases := map[string]struct { + name string + mutate func(*armapimanagement.ServiceResource) + }{ + "missing publisherEmail": {"svc-a", func(s *armapimanagement.ServiceResource) { s.Properties.PublisherEmail = nil }}, + "missing publisherName": {"svc-b", func(s *armapimanagement.ServiceResource) { s.Properties.PublisherName = nil }}, + "missing location": {"svc-c", func(s *armapimanagement.ServiceResource) { s.Location = nil }}, + "consumption capacity 1": {"svc-d", func(s *armapimanagement.ServiceResource) { + s.SKU.Name = to.Ptr(armapimanagement.SKUTypeConsumption) + }}, + "developer capacity 0": {"svc-e", func(s *armapimanagement.ServiceResource) { s.SKU.Capacity = to.Ptr[int32](0) }}, + "unknown sku": {"svc-f", func(s *armapimanagement.ServiceResource) { s.SKU.Name = to.Ptr(armapimanagement.SKUType("Gold")) }}, + "name starts digit": {"1svc", nil}, + "name too long": {"a" + strings.Repeat("b", 50), nil}, + "name trailing hyphen": {"svc-", nil}, + } + + for label, tc := range cases { + t.Run(label, func(t *testing.T) { + body := developerService() + if tc.mutate != nil { + tc.mutate(&body) + } + + _, err := f.svc.BeginCreateOrUpdate(ctx, rgName, tc.name, body, nil) + assertStatus(t, err, http.StatusBadRequest, "InvalidParameter") + }) + } +} + +// TestSDKMissingResourceGroupAndCascade covers the resource-group gate (a +// create in a group that does not exist is 404 ResourceGroupNotFound) and the +// purge cascade (deleting the group deletes its services). +func TestSDKMissingResourceGroupAndCascade(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + _, err := f.svc.BeginCreateOrUpdate(ctx, "rg-missing", svcName, developerService(), nil) + assertStatus(t, err, http.StatusNotFound, "ResourceGroupNotFound") + + f.create(t, rgOther, svcName, developerService()) + + rgPoller, err := f.rgs.BeginDelete(ctx, rgOther, nil) + if err != nil { + t.Fatalf("delete resource group: %v", err) + } + + if _, err := rgPoller.PollUntilDone(ctx, nil); err != nil { + t.Fatalf("poll resource group delete: %v", err) + } + + f.ensureRG(t, rgOther) + + _, err = f.svc.Get(ctx, rgOther, svcName, nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") +} + +// TestResourceGraphListsService asserts the service is projected into the +// discovery inventory and Resource Graph under its ARM type. +func TestResourceGraphListsService(t *testing.T) { + f := newFixture(t) + f.create(t, rgName, svcName, developerService()) + + body := `{"subscriptions":["` + subID + `"],"query":"Resources | where type =~ 'microsoft.apimanagement/service'"}` + + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + f.ts.URL+"/providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01", strings.NewReader(body)) + if err != nil { + t.Fatalf("new request: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := f.ts.Client().Do(req) + if err != nil { + t.Fatalf("resource graph query: %v", err) + } + defer resp.Body.Close() + + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(raw), svcName) || + !strings.Contains(strings.ToLower(string(raw)), "microsoft.apimanagement/service") { + t.Fatalf("resource graph = %d %s, want the service row", resp.StatusCode, raw) + } +} + +func assertStatus(t *testing.T, err error, status int, code string) { + t.Helper() + + var re *azcore.ResponseError + if !errors.As(err, &re) { + t.Fatalf("err = %v, want *azcore.ResponseError %d", err, status) + } + + if re.StatusCode != status || re.ErrorCode != code { + t.Fatalf("got %d %s, want %d %s", re.StatusCode, re.ErrorCode, status, code) + } +} + +// deref returns *p, or the zero value for a nil pointer. +func deref[T any](p *T) T { + var zero T + if p == nil { + return zero + } + + return *p +} diff --git a/server/azure/apimanagement/handler.go b/server/azure/apimanagement/handler.go new file mode 100644 index 000000000..598fe92b8 --- /dev/null +++ b/server/azure/apimanagement/handler.go @@ -0,0 +1,213 @@ +// Package apimanagement serves the Azure API Management service ARM API +// (Microsoft.ApiManagement/service). Real armapimanagement ServiceClient +// requests hit this handler the same way they hit management.azure.com. +// +// Real Azure runs service create, update and delete as long-running operations +// (a Developer-tier create takes 30-45 minutes). The emulator completes them +// synchronously: PUT answers 201/200 and PATCH 200 with a body whose +// properties.provisioningState is already "Succeeded" and no +// Azure-AsyncOperation / Location header, which azcore's Body poller treats as +// terminal on the initial response; DELETE answers 200/204 with no polling +// header, which azcore treats as a completed no-op poll. So PollUntilDone +// returns on the first call. +// +// The service's child resources (apis, products, subscriptions, policies, +// backends, ...), the gateway data plane, backup/restore and the soft-deleted +// services (deletedservices) surface are out of scope. +package apimanagement + +import ( + "context" + "net/http" + "strings" + + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" + "github.com/stackshy/cloudemu/v2/server/wire/azurearm" +) + +const ( + providerName = "Microsoft.ApiManagement" + serviceType = "service" + serviceArmType = providerName + "/" + serviceType +) + +// Store is the minimal API Management backend the handler needs. +// *apimanagement.Mock satisfies it. +type Store interface { + CreateOrUpdateService( + ctx context.Context, sub, rg, name, location string, in *apimanagement.ServiceInput, + ) (apimanagement.Service, bool, error) + UpdateService(ctx context.Context, sub, rg, name string, in *apimanagement.ServiceInput) (apimanagement.Service, error) + GetService(ctx context.Context, sub, rg, name string) (apimanagement.Service, error) + DeleteService(ctx context.Context, sub, rg, name string) (bool, error) + ListServicesByResourceGroup(ctx context.Context, sub, rg string) ([]apimanagement.Service, error) + ListServicesBySubscription(ctx context.Context, sub string) ([]apimanagement.Service, error) + PurgeResourceGroup(ctx context.Context, sub, rg string) error +} + +// Handler serves Microsoft.ApiManagement/service ARM requests. +type Handler struct { + store Store +} + +// New returns an API Management handler backed by store. +func New(store Store) *Handler { + return &Handler{store: store} +} + +// Matches reports whether r targets an API Management service ARM URL. The +// provider and type are matched case-insensitively. +func (*Handler) Matches(r *http.Request) bool { + rp, ok := azurearm.ParsePath(r.URL.Path) + if !ok { + return false + } + + return strings.EqualFold(rp.Provider, providerName) && + strings.EqualFold(rp.ResourceType, serviceType) +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rp, ok := azurearm.ParsePath(r.URL.Path) + if !ok { + azurearm.WriteError(w, http.StatusBadRequest, "InvalidPath", "malformed ARM path") + return + } + + switch { + case rp.ResourceName == "": + h.listServices(w, r, &rp) + case rp.SubResource == "": + h.serveService(w, r, &rp) + default: + azurearm.WriteError(w, http.StatusNotFound, "InvalidResourceType", + "unsupported API Management sub-resource "+rp.SubResource) + } +} + +// PurgeResourceGroup deletes every service under sub/rg so a resource-group +// delete cascades into them. +func (h *Handler) PurgeResourceGroup(ctx context.Context, subscription, resourceGroup string) error { + return h.store.PurgeResourceGroup(ctx, subscription, resourceGroup) +} + +// serveService routes the top-level service CRUD surface. +func (h *Handler) serveService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + switch r.Method { + case http.MethodPut: + h.createService(w, r, rp) + case http.MethodPatch: + h.updateService(w, r, rp) + case http.MethodGet: + h.getService(w, r, rp) + case http.MethodDelete: + h.deleteService(w, r, rp) + default: + azurearm.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + } +} + +func (h *Handler) createService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + if rp.ResourceGroup == "" { + azurearm.WriteError(w, http.StatusBadRequest, "InvalidPath", "missing resourceGroups segment") + return + } + + var req serviceRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + in := serviceInputFromRequest(&req) + + s, created, err := h.store.CreateOrUpdateService( + r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName, req.Location, &in) + if err != nil { + azurearm.WriteCErr(w, err) + return + } + + status := http.StatusOK + if created { + status = http.StatusCreated + } + + azurearm.WriteJSON(w, status, toServiceResponse(&s)) +} + +// updateService applies an ARM PATCH: supplied tags replace the set, sku and +// identity are re-resolved when named, and the properties block is merged. A +// PATCH on a missing service is a 404. +func (h *Handler) updateService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + var req serviceRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + in := serviceInputFromRequest(&req) + + s, err := h.store.UpdateService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName, &in) + if err != nil { + azurearm.WriteCErr(w, err) + return + } + + azurearm.WriteJSON(w, http.StatusOK, toServiceResponse(&s)) +} + +func (h *Handler) getService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + s, err := h.store.GetService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName) + if err != nil { + azurearm.WriteCErr(w, err) + return + } + + azurearm.WriteJSON(w, http.StatusOK, toServiceResponse(&s)) +} + +// deleteService is the idempotent ARM DELETE: 200 when the service existed, +// 204 when it did not. +func (h *Handler) deleteService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + existed, err := h.store.DeleteService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName) + if err != nil { + azurearm.WriteCErr(w, err) + return + } + + if existed { + w.WriteHeader(http.StatusOK) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) listServices(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + if r.Method != http.MethodGet { + azurearm.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + return + } + + var ( + items []apimanagement.Service + err error + ) + + if rp.ResourceGroup != "" { + items, err = h.store.ListServicesByResourceGroup(r.Context(), rp.Subscription, rp.ResourceGroup) + } else { + items, err = h.store.ListServicesBySubscription(r.Context(), rp.Subscription) + } + + if err != nil { + azurearm.WriteCErr(w, err) + return + } + + out := serviceListResponse{Value: make([]serviceResponse, 0, len(items))} + for i := range items { + out.Value = append(out.Value, toServiceResponse(&items[i])) + } + + azurearm.WriteJSON(w, http.StatusOK, out) +} diff --git a/server/azure/apimanagement/types.go b/server/azure/apimanagement/types.go new file mode 100644 index 000000000..13fb25e72 --- /dev/null +++ b/server/azure/apimanagement/types.go @@ -0,0 +1,241 @@ +package apimanagement + +import ( + "encoding/json" + "strings" + "time" + + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" +) + +const ( + // skuConsumption is the serverless tier, which has no dedicated portal, + // management or SCM endpoints and runs on the multi-tenant platform. + skuConsumption = "Consumption" + + // platformDedicated / platformConsumption are the computePlatform versions + // Azure reports for dedicated tiers and for the Consumption tier. + platformDedicated = "stv2" + platformConsumption = "mtv1" + + // notificationSenderDefault is the sender address Azure assigns when the + // caller sets none. + notificationSenderDefault = "apimgmt-noreply@mail.windowsazure.com" +) + +// serviceRequest is the ARM service PUT/PATCH body. location, tags, zones, sku +// and identity are top-level; the writable service properties live under +// properties and round-trip verbatim. +type serviceRequest struct { + Location string `json:"location"` + Tags map[string]string `json:"tags,omitempty"` + Zones []string `json:"zones,omitempty"` + Sku *skuWire `json:"sku,omitempty"` + Identity *identityWire `json:"identity,omitempty"` + Properties json.RawMessage `json:"properties,omitempty"` +} + +// skuWire is the service SKU block. Capacity is a pointer so a request that +// omits it is distinguishable from an explicit 0 (the Consumption tier). +type skuWire struct { + Name string `json:"name,omitempty"` + Capacity *int32 `json:"capacity,omitempty"` +} + +// skuResponse is the SKU block as returned: capacity is always present. +type skuResponse struct { + Name string `json:"name"` + Capacity int32 `json:"capacity"` +} + +// identityWire is the managed-identity request/response block. On a request +// only type and userAssignedIdentities are read; on a response principalId and +// tenantId are the computed, stable values. +type identityWire struct { + Type string `json:"type,omitempty"` + PrincipalID string `json:"principalId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + UserAssignedIdentities map[string]json.RawMessage `json:"userAssignedIdentities,omitempty"` +} + +// serviceResponse is the ARM representation of an API Management service. +type serviceResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Location string `json:"location"` + Tags map[string]string `json:"tags,omitempty"` + Zones []string `json:"zones,omitempty"` + Sku skuResponse `json:"sku"` + Identity *identityWire `json:"identity,omitempty"` + Etag string `json:"etag"` + Properties json.RawMessage `json:"properties"` +} + +// serviceListResponse is the ARM service list envelope. nextLink is omitted: +// the emulator returns a single page. +type serviceListResponse struct { + Value []serviceResponse `json:"value"` +} + +// serviceInputFromRequest builds a service create/update Input from a request +// body. +func serviceInputFromRequest(req *serviceRequest) apimanagement.ServiceInput { + in := apimanagement.ServiceInput{Tags: req.Tags, Zones: req.Zones, Properties: req.Properties} + + if req.Sku != nil { + if req.Sku.Name != "" { + name := req.Sku.Name + in.SkuName = &name + } + + in.SkuCapacity = req.Sku.Capacity + } + + if req.Identity != nil { + in.Identity = &apimanagement.ManagedIdentity{ + Type: req.Identity.Type, + UserAssignedIDs: userAssignedKeys(req.Identity.UserAssignedIdentities), + } + } + + return in +} + +// userAssignedKeys extracts the user-assigned identity resource ids from the +// request map. +func userAssignedKeys(m map[string]json.RawMessage) []string { + if len(m) == 0 { + return nil + } + + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + + return out +} + +// toServiceResponse projects a stored service onto the ARM wire +// representation, injecting the computed read-only properties. +func toServiceResponse(s *apimanagement.Service) serviceResponse { + return serviceResponse{ + ID: s.ARMID(), + Name: s.Name, + Type: serviceArmType, + Location: s.Location, + Tags: s.Tags, + Zones: s.Zones, + Sku: skuResponse{Name: s.SkuName, Capacity: s.SkuCapacity}, + Identity: toIdentityWire(s.Identity), + Etag: s.Etag, + Properties: responseProperties(s), + } +} + +// responseProperties overlays the computed read-only fields onto the stored, +// verbatim properties block, filling Azure's defaults for the few writable +// fields the caller left unset. +func responseProperties(s *apimanagement.Service) json.RawMessage { + obj := map[string]any{} + if len(s.Properties) > 0 { + if err := json.Unmarshal(s.Properties, &obj); err != nil { + obj = map[string]any{} + } + } + + for k, v := range map[string]any{ + "virtualNetworkType": "None", + "publicNetworkAccess": "Enabled", + "notificationSenderEmail": notificationSenderDefault, + "disableGateway": false, + } { + if _, set := obj[k]; !set { + obj[k] = v + } + } + + for k, v := range computedProperties(s) { + obj[k] = v + } + + raw, err := json.Marshal(obj) + if err != nil { + return s.Properties + } + + return raw +} + +// computedProperties returns the read-only properties Azure mints: the +// provisioning state, creation time, platform version and the endpoint URLs. +// The Consumption tier has only a gateway, so its portal, management and SCM +// endpoints are absent. +func computedProperties(s *apimanagement.Service) map[string]any { + out := map[string]any{ + "provisioningState": s.ProvisioningState, + "targetProvisioningState": "", + "createdAtUtc": s.CreatedAt.UTC().Format(time.RFC3339), + "gatewayUrl": s.Endpoints().Gateway, + "publicIPAddresses": []string{}, + "platformVersion": platformDedicated, + } + + if s.SkuName == skuConsumption { + out["platformVersion"] = platformConsumption + + return out + } + + ep := s.Endpoints() + out["gatewayRegionalUrl"] = regionalGatewayURL(s) + out["portalUrl"] = ep.Portal + out["developerPortalUrl"] = ep.DeveloperPortal + out["managementApiUrl"] = ep.ManagementAPI + out["scmUrl"] = ep.Scm + + return out +} + +// regionalGatewayURL renders the primary region's gateway endpoint, +// https://--01.regional.azure-api.net. +func regionalGatewayURL(s *apimanagement.Service) string { + region := strings.ToLower(strings.ReplaceAll(s.Location, " ", "")) + + return "https://" + strings.ToLower(s.Name) + "-" + region + "-01.regional.azure-api.net" +} + +// toIdentityWire projects a stored managed identity onto the wire block, +// synthesizing per-identity principal/client ids for each user-assigned entry. +func toIdentityWire(id *apimanagement.ManagedIdentity) *identityWire { + if id == nil { + return nil + } + + out := &identityWire{Type: id.Type, PrincipalID: id.PrincipalID, TenantID: id.TenantID} + + if len(id.UserAssignedIDs) > 0 { + out.UserAssignedIdentities = make(map[string]json.RawMessage, len(id.UserAssignedIDs)) + for _, uaID := range id.UserAssignedIDs { + out.UserAssignedIdentities[uaID] = userAssignedValue(uaID) + } + } + + return out +} + +// userAssignedValue synthesizes the deterministic {principalId, clientId} +// block Azure returns for an assigned user identity. +func userAssignedValue(uaID string) json.RawMessage { + principal := idgen.SyntheticGUID("apimanagement/ua-principal/" + strings.ToLower(uaID)) + client := idgen.SyntheticGUID("apimanagement/ua-client/" + strings.ToLower(uaID)) + + raw, err := json.Marshal(map[string]string{"principalId": principal, "clientId": client}) + if err != nil { + return json.RawMessage(`{}`) + } + + return raw +} diff --git a/server/azure/azure.go b/server/azure/azure.go index 5d64df0ff..bad0d963a 100644 --- a/server/azure/azure.go +++ b/server/azure/azure.go @@ -16,6 +16,7 @@ import ( "github.com/stackshy/cloudemu/v2/server/azure/acr" azureaiserver "github.com/stackshy/cloudemu/v2/server/azure/ai" aksserver "github.com/stackshy/cloudemu/v2/server/azure/aks" + apimanagementsrv "github.com/stackshy/cloudemu/v2/server/azure/apimanagement" appconfigsrv "github.com/stackshy/cloudemu/v2/server/azure/appconfiguration" appinsightssrv "github.com/stackshy/cloudemu/v2/server/azure/appinsights" appgatewaysrv "github.com/stackshy/cloudemu/v2/server/azure/applicationgateway" @@ -219,6 +220,9 @@ type Drivers struct { // IoTHub serves Microsoft.Devices/IotHubs plus its listkeys / // getKeysForKeyName actions and the nested event-hub consumer groups. IoTHub iothubsrv.Store + // APIManagement serves Microsoft.ApiManagement/service (the API Management + // service control plane). + APIManagement apimanagementsrv.Store // SQLVirtualMachine serves Microsoft.SqlVirtualMachine/sqlVirtualMachines: // the SQL-management overlay on a compute VM. SQLVirtualMachine sqlvirtualmachinesrv.Store @@ -640,6 +644,15 @@ func New(d Drivers) http.Handler { rgPurgers = append(rgPurgers, iotHubHandler) } + // API Management: a resource-group-scoped resource, so its handler joins the + // purge cascade. Deleting the group tears down every API Management service. + // Registered further below. + var apiManagementHandler *apimanagementsrv.Handler + if d.APIManagement != nil { + apiManagementHandler = apimanagementsrv.New(d.APIManagement) + rgPurgers = append(rgPurgers, apiManagementHandler) + } + // SQL virtual machines: a resource-group-scoped resource, so its handler // joins the purge cascade. Deleting the group tears down the SQL-management // overlay records but never the paired compute VMs. Registered further below. @@ -1114,6 +1127,13 @@ func New(d Drivers) http.Handler { srv.Register(iotHubHandler) } + // API Management claims Microsoft.ApiManagement/service: a distinct ARM + // provider name from every other Azure handler, so registration order is + // unconstrained. + if apiManagementHandler != nil { + srv.Register(apiManagementHandler) + } + // SQL virtual machines claim Microsoft.SqlVirtualMachine/sqlVirtualMachines: // a distinct ARM provider name from every other Azure handler, so registration // order is unconstrained. Registered before the BlobStorage fallback. diff --git a/server/azure/from_provider.go b/server/azure/from_provider.go index 84abec4a8..b07a85764 100644 --- a/server/azure/from_provider.go +++ b/server/azure/from_provider.go @@ -58,6 +58,7 @@ func DriversFrom(p *azureprovider.Provider) Drivers { StreamAnalytics: p.StreamAnalytics, RecoveryServices: p.RecoveryServices, IoTHub: p.IoTHub, + APIManagement: p.APIManagement, SQLVirtualMachine: p.SQLVirtualMachine, ContainerApps: p.ContainerApps, IAM: p.IAM, diff --git a/server/azure/resourcegraph/handler.go b/server/azure/resourcegraph/handler.go index ba73005e1..71b23ed2f 100644 --- a/server/azure/resourcegraph/handler.go +++ b/server/azure/resourcegraph/handler.go @@ -494,6 +494,7 @@ var portableToAzureTypeMap = map[string]string{ //nolint:gochecknoglobals // sta "streamanalytics/StreamingJob": "microsoft.streamanalytics/streamingjobs", "recoveryservices/Vault": "microsoft.recoveryservices/vaults", "iothub/IotHub": "microsoft.devices/iothubs", + "apimanagement/Service": "microsoft.apimanagement/service", } func portableToAzureType(service, typ string) string { diff --git a/server/azure/resourcegraph/kql.go b/server/azure/resourcegraph/kql.go index 8167a3a88..6a59fa1b5 100644 --- a/server/azure/resourcegraph/kql.go +++ b/server/azure/resourcegraph/kql.go @@ -100,6 +100,7 @@ const ( azureTypeStreamAnl = "microsoft.streamanalytics/streamingjobs" azureTypeRecovery = "microsoft.recoveryservices/vaults" azureTypeIoTHub = "microsoft.devices/iothubs" + azureTypeAPIM = "microsoft.apimanagement/service" ) // Portable service identifiers as emitted by the resourcediscovery walkers. @@ -145,6 +146,7 @@ const ( portableStreamAnl = "streamanalytics" portableRecovery = "recoveryservices" portableIoTHub = "iothub" + portableAPIM = "apimanagement" ) // parsedKQL is the result of KQL parsing: an engine Query plus the limit @@ -445,6 +447,7 @@ var azureToPortableType = map[string]portableResourceType{ //nolint:gochecknoglo azureTypeStreamAnl: {portableStreamAnl, "StreamingJob"}, azureTypeRecovery: {portableRecovery, "Vault"}, azureTypeIoTHub: {portableIoTHub, "IotHub"}, + azureTypeAPIM: {portableAPIM, "Service"}, } // mapAzureType translates a fully-qualified Azure resource type to the diff --git a/services/resourcediscovery/walkers.go b/services/resourcediscovery/walkers.go index a90c5b294..aa86a5740 100644 --- a/services/resourcediscovery/walkers.go +++ b/services/resourcediscovery/walkers.go @@ -111,6 +111,9 @@ const ( // ServiceIoTHub buckets Azure IoT Hub resources // (Microsoft.Devices/IotHubs). ServiceIoTHub = "iothub" + // ServiceAPIManagement buckets Azure API Management resources + // (Microsoft.ApiManagement/service). + ServiceAPIManagement = "apimanagement" ) // Resource type constants emitted by the walkers. @@ -266,6 +269,11 @@ const TypeRecoveryVault = "Vault" // (Microsoft.Devices/IotHubs). Azure-only, so it lives in its own line. const TypeIoTHub = "IotHub" +// TypeAPIManagementService is the portable type for an Azure API Management +// service (Microsoft.ApiManagement/service). Azure-only, so it lives in its own +// line. +const TypeAPIManagementService = "Service" + // sqlVMOptInTagKey and sqlVMOptInTagValue mark a compute VM as opting in to a // paired Microsoft.SqlVirtualMachine overlay row in discovery. Only Azure VMs // carrying the tag get the overlay, so plain VMs, and every AWS/GCP VM, are From 25036cc1e40215a598b2710b03fdd8ff9fc0b86b Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sat, 26 Sep 2026 20:47:13 +0530 Subject: [PATCH 04/24] feat(gcp): Cloud CDN backendBuckets on the compute load-balancing handler Operations (compute/v1 global/backendBuckets): insert, get, list (maxResults/pageToken, name filter), patch (RFC 7386 merge: cdnPolicy merges member-by-member), update (full replace; output-only edgeSecurityPolicy kept), delete, setEdgeSecurityPolicy. Every mutation returns a DONE global compute#operation recorded in the shared OperationRegistry and polled via global/operations. Storage: new optional driver capability GCPBackendBucketStore (services/loadbalancer/driver/gcp.go), implemented by the GCP LB mock over the existing opaque GCP resource store, so records snapshot with the other LB resources. AWS/Azure are untouched (type-asserted capability). Validation: RFC 1035 name; bucketName required and, when the GCS driver is wired (Drivers.Storage), must name an existing bucket; compressionMode and cdnPolicy.cacheMode enums; defaultTtl/maxTtl/clientTtl in 0..31622400 and defaultTtl <= maxTtl; serveWhileStale <= 604800; <= 5 bypass headers; negativeCachingPolicy requires negativeCaching. Patch/update validate the merged result under the store lock. Duplicate 409, missing 404. Reference integrity: url-map defaultService / pathMatchers[].defaultService / pathRules[].service naming a missing backend bucket is rejected (400); deleting a backend bucket a url-map routes to returns 400 resourceInUseByAnotherResource. Out of scope: signed URL keys, IAM policy, validating the edge security policy reference (no securityPolicies resource), mode-dependent TTL rules. --- docs/coverage/coverage.json | 26 + docs/coverage/gcp/lb.md | 12 + docs/sdk-server.md | 1 + providers/gcp/loadbalancer/backend_buckets.go | 82 ++++ .../gcp/loadbalancer/backend_buckets_test.go | 55 +++ server/gcp/gcp.go | 6 + server/gcp/loadbalancer/backendbuckets.go | 457 +++++++++++++++++ .../loadbalancer/backendbuckets_sdk_test.go | 462 ++++++++++++++++++ .../loadbalancer/backendbuckets_validate.go | 210 ++++++++ server/gcp/loadbalancer/handler.go | 8 +- server/gcp/loadbalancer/l7frontend.go | 2 + .../referenceintegrity_sdk_test.go | 8 +- server/gcp/loadbalancer/refvalidation.go | 66 ++- server/gcp/loadbalancer/resources.go | 9 + services/loadbalancer/driver/gcp.go | 30 ++ 15 files changed, 1413 insertions(+), 21 deletions(-) create mode 100644 providers/gcp/loadbalancer/backend_buckets.go create mode 100644 providers/gcp/loadbalancer/backend_buckets_test.go create mode 100644 server/gcp/loadbalancer/backendbuckets.go create mode 100644 server/gcp/loadbalancer/backendbuckets_sdk_test.go create mode 100644 server/gcp/loadbalancer/backendbuckets_validate.go diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index 03614c172..a6b684c70 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -9620,6 +9620,32 @@ } ] }, + { + "name": "GCPBackendBucketStore", + "doc": "GCPBackendBucketStore is an OPTIONAL, type-asserted capability implemented", + "operations": [ + { + "name": "DeleteGCPBackendBucket", + "doc": "DeleteGCPBackendBucket removes the named backend bucket, returning" + }, + { + "name": "GetGCPBackendBucket", + "doc": "GetGCPBackendBucket returns the named backend bucket, or NotFound." + }, + { + "name": "InsertGCPBackendBucket", + "doc": "InsertGCPBackendBucket stores res, returning AlreadyExists when a backend" + }, + { + "name": "ListGCPBackendBuckets", + "doc": "ListGCPBackendBuckets returns every backend bucket." + }, + { + "name": "UpdateGCPBackendBucket", + "doc": "UpdateGCPBackendBucket applies mutate to the named backend bucket under" + } + ] + }, { "name": "GCPBackendServicePatcher", "doc": "GCPBackendServicePatcher is an OPTIONAL, type-asserted capability implemented", diff --git a/docs/coverage/gcp/lb.md b/docs/coverage/gcp/lb.md index 3db008ba5..ebe0e7bab 100644 --- a/docs/coverage/gcp/lb.md +++ b/docs/coverage/gcp/lb.md @@ -31,6 +31,18 @@ GCP's `loadbalancer` service · portable interface `driver.LoadBalancer` · [GCP Discovered by type assertion; only some providers implement these. +### GCPBackendBucketStore + +GCPBackendBucketStore is an OPTIONAL, type-asserted capability implemented + +| Operation | Description | +| --- | --- | +| `DeleteGCPBackendBucket` | DeleteGCPBackendBucket removes the named backend bucket, returning | +| `GetGCPBackendBucket` | GetGCPBackendBucket returns the named backend bucket, or NotFound. | +| `InsertGCPBackendBucket` | InsertGCPBackendBucket stores res, returning AlreadyExists when a backend | +| `ListGCPBackendBuckets` | ListGCPBackendBuckets returns every backend bucket. | +| `UpdateGCPBackendBucket` | UpdateGCPBackendBucket applies mutate to the named backend bucket under | + ### GCPBackendServicePatcher GCPBackendServicePatcher is an OPTIONAL, type-asserted capability implemented diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 2a36153bf..46fbb654f 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -231,6 +231,7 @@ All handlers speak REST + JSON. | **IAM (iam.googleapis.com v1)** | ServiceAccounts (Create/Get/List/Delete/Patch), custom Roles (Create/Get/List/Delete/Patch), ServiceAccountKeys (Create/Get/List/Delete). Real `google.golang.org/api/iam/v1` clients round-trip end-to-end; errors surface as typed `*googleapi.Error`. Resource-level `getIamPolicy`/`setIamPolicy` bindings on individual GCP resources are out of scope. | | **Managed Service for Apache Kafka (managedkafka.googleapis.com v1)** | Clusters (Create/Update/Delete as LROs polled through the shared location-operations route, Get, List with `pageToken`/`pageSize`; `updateMask` required, only masked fields change, unknown/immutable/output-only paths are 400) and Topics (Create/Get/List/Patch/Delete, synchronous; `partitionCount` can only increase, `replicationFactor` immutable). Validates `vcpuCount >= 3`, 1-8 GiB memory per vCPU, 1-10 `networkConfigs[].subnet`, and the RFC 1035 `clusterId`. Shares the `/clusters` path with GKE/AlloyDB: it claims only Kafka-shaped creates and clusters it owns. Real `google.golang.org/api/managedkafka/v1` clients round-trip end-to-end. Consumer groups, ACLs, Connect and Schema Registry are out of scope. | | **Backup and DR (backupdr.googleapis.com v1)** | Backup vaults: create (LRO; `backupVaultId`, required `backupMinimumEnforcedRetentionDuration`, `validateOnly`), get, list (`pageSize`/`pageToken`, `locations/-` wildcard; `filter`/`orderBy` accepted and ignored), patch (LRO; required `updateMask`, output-only/unknown paths 400, stale body `etag` 409 ABORTED), delete (LRO; `force`, `allowMissing`, `etag`, `validateOnly`; a vault holding backups without `force` is 400 FAILED_PRECONDITION). Output-only `state` ACTIVE, `deletable`, `etag` (rotates per update), `uid`, deterministic `serviceAccount` (`service-{12-digit number derived from the project id}@gcp-sa-backupdr-pr.iam.gserviceaccount.com`), `backupCount`/`totalStoredBytes` "0". Operations poll through the shared location-scoped LRO poller. Data sources, backups, backup plans, management servers and restores are out of scope. Real `google.golang.org/api/backupdr/v1` clients round-trip end-to-end. | +| **Cloud CDN backend buckets** *(`compute/v1` `global/backendBuckets`)* | Insert, Get, List (`maxResults`/`pageToken`, `name` filter), Patch (JSON merge patch: `cdnPolicy` merges member-by-member), Update (full replace), Delete, `setEdgeSecurityPolicy`; every mutation returns a global `compute#operation` the SDK polls to DONE. Validates the RFC 1035 name, a required `bucketName` that must name an existing GCS bucket when the GCS backend is wired, `compressionMode`, `cdnPolicy.cacheMode`, TTL ranges (0-31,622,400s, `defaultTtl` <= `maxTtl`), `serveWhileStale` <= 604,800s, <= 5 bypass headers, and `negativeCachingPolicy` requiring `negativeCaching`. Duplicate -> 409, missing -> 404. Url-map `defaultService` / `pathMatchers[].defaultService` / `pathRules[].service` references to a missing backend bucket are rejected (400), and deleting a backend bucket a url-map routes to returns 400 `resourceInUseByAnotherResource`. Signed URL keys, IAM policy, and the edge security policy reference itself (no `securityPolicies` resource) are not modeled. | An operation cloudemu doesn't implement returns `501 Not Implemented` or the provider's native `UnknownOperation` / `NotImplemented` / `NOT_FOUND` error. diff --git a/providers/gcp/loadbalancer/backend_buckets.go b/providers/gcp/loadbalancer/backend_buckets.go new file mode 100644 index 000000000..55ab29996 --- /dev/null +++ b/providers/gcp/loadbalancer/backend_buckets.go @@ -0,0 +1,82 @@ +package loadbalancer + +import ( + "context" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +// Compile-time check that Mock implements the backend-bucket capability. +var _ driver.GCPBackendBucketStore = (*Mock)(nil) + +// backendBucketScope is the store scope of a backend bucket; they are global. +const backendBucketScope = "global" + +// InsertGCPBackendBucket stores a Cloud CDN backend bucket, returning +// AlreadyExists when the name is taken. Collection and Scope are forced so a +// caller can't file a backend bucket under another collection. +// +//nolint:gocritic // hugeParam: interface method signature is fixed. +func (m *Mock) InsertGCPBackendBucket(ctx context.Context, res driver.GCPResource) error { + res.Collection = driver.GCPBackendBucketCollection + res.Scope = backendBucketScope + + err := m.PutGCPResource(ctx, res) + if cerrors.IsAlreadyExists(err) { + return cerrors.Newf(cerrors.AlreadyExists, "The resource 'backendBuckets/%s' already exists", res.Name) + } + + return err +} + +// GetGCPBackendBucket returns the named backend bucket, or NotFound. +func (m *Mock) GetGCPBackendBucket(_ context.Context, name string) (*driver.GCPResource, error) { + res, ok := m.gcpResources.Get(gcpResourceKey(driver.GCPBackendBucketCollection, backendBucketScope, name)) + if !ok { + return nil, backendBucketNotFound(name) + } + + return &res, nil +} + +// ListGCPBackendBuckets returns every backend bucket. +func (m *Mock) ListGCPBackendBuckets(ctx context.Context) ([]driver.GCPResource, error) { + return m.ListGCPResources(ctx, driver.GCPBackendBucketCollection, backendBucketScope) +} + +// UpdateGCPBackendBucket applies mutate to the named backend bucket under the +// store lock. A mutate error leaves the stored record unchanged. +func (m *Mock) UpdateGCPBackendBucket(_ context.Context, name string, mutate func(*driver.GCPResource) error) error { + var mutateErr error + + updated := m.gcpResources.Update(gcpResourceKey(driver.GCPBackendBucketCollection, backendBucketScope, name), + func(res driver.GCPResource) driver.GCPResource { + next := res + if err := mutate(&next); err != nil { + mutateErr = err + return res + } + + return next + }) + if !updated { + return backendBucketNotFound(name) + } + + return mutateErr +} + +// DeleteGCPBackendBucket removes the named backend bucket, or NotFound. +func (m *Mock) DeleteGCPBackendBucket(_ context.Context, name string) error { + if !m.gcpResources.Delete(gcpResourceKey(driver.GCPBackendBucketCollection, backendBucketScope, name)) { + return backendBucketNotFound(name) + } + + return nil +} + +// backendBucketNotFound renders compute's not-found message for a backend bucket. +func backendBucketNotFound(name string) error { + return cerrors.Newf(cerrors.NotFound, "The resource 'backendBuckets/%s' was not found", name) +} diff --git a/providers/gcp/loadbalancer/backend_buckets_test.go b/providers/gcp/loadbalancer/backend_buckets_test.go new file mode 100644 index 000000000..186127640 --- /dev/null +++ b/providers/gcp/loadbalancer/backend_buckets_test.go @@ -0,0 +1,55 @@ +package loadbalancer + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +func TestGCPBackendBucketStore(t *testing.T) { + ctx := context.Background() + m := newTestMock() + + bb := driver.GCPResource{Name: "bb", ID: "1", Body: map[string]any{"bucketName": "assets"}} + require.NoError(t, m.InsertGCPBackendBucket(ctx, bb)) + assert.True(t, cerrors.IsAlreadyExists(m.InsertGCPBackendBucket(ctx, bb))) + + got, err := m.GetGCPBackendBucket(ctx, "bb") + require.NoError(t, err) + assert.Equal(t, driver.GCPBackendBucketCollection, got.Collection) + assert.Equal(t, "global", got.Scope) + + // A mutate error leaves the stored record unchanged. + errReject := cerrors.New(cerrors.InvalidArgument, "rejected") + err = m.UpdateGCPBackendBucket(ctx, "bb", func(res *driver.GCPResource) error { + res.Body = map[string]any{"bucketName": "other"} + return errReject + }) + require.ErrorIs(t, err, errReject) + + got, err = m.GetGCPBackendBucket(ctx, "bb") + require.NoError(t, err) + assert.Equal(t, "assets", got.Body["bucketName"]) + + require.NoError(t, m.UpdateGCPBackendBucket(ctx, "bb", func(res *driver.GCPResource) error { + res.Body = map[string]any{"bucketName": "other"} + return nil + })) + + list, err := m.ListGCPBackendBuckets(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + assert.Equal(t, "other", list[0].Body["bucketName"]) + + require.NoError(t, m.DeleteGCPBackendBucket(ctx, "bb")) + assert.True(t, cerrors.IsNotFound(m.DeleteGCPBackendBucket(ctx, "bb"))) + + _, err = m.GetGCPBackendBucket(ctx, "bb") + assert.True(t, cerrors.IsNotFound(err)) + assert.True(t, cerrors.IsNotFound(m.UpdateGCPBackendBucket(ctx, "bb", func(*driver.GCPResource) error { return nil }))) +} diff --git a/server/gcp/gcp.go b/server/gcp/gcp.go index ebc285110..f098f49c9 100644 --- a/server/gcp/gcp.go +++ b/server/gcp/gcp.go @@ -485,6 +485,12 @@ func New(d Drivers) *server.Server { if d.LB != nil { lbH := lbsrv.New(d.LB) lbH.SetOperationRegistry(computeOps) + + if d.Storage != nil { + // backendBuckets reject a bucketName naming no existing GCS bucket. + lbH.SetBucketLister(d.Storage) + } + srv.Register(lbH) } diff --git a/server/gcp/loadbalancer/backendbuckets.go b/server/gcp/loadbalancer/backendbuckets.go new file mode 100644 index 000000000..64cf908cd --- /dev/null +++ b/server/gcp/loadbalancer/backendbuckets.go @@ -0,0 +1,457 @@ +package loadbalancer + +// Cloud CDN backend buckets (compute.backendBuckets): a global load-balancer +// backend that serves a Cloud Storage bucket, referenced from a url-map's +// defaultService / pathMatchers[].defaultService / pathRules[].service the same +// way a backend service is. Records live in the GCP provider's opaque resource +// store through the GCPBackendBucketStore optional capability, so every field +// the client sent round-trips and the record snapshots with the other LB +// resources. +// +// Surface: insert, get, list, patch (JSON merge patch), update (full replace), +// delete, setEdgeSecurityPolicy. Every mutation answers a DONE compute#operation +// recorded in the shared OperationRegistry, polled at +// /compute/v1/projects/{p}/global/operations/{op}. + +import ( + "context" + "net/http" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + lbdriver "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" + storagedriver "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +const ( + resourceBackendBuckets = lbdriver.GCPBackendBucketCollection + + // actionSetEdgeSecurityPolicy is the POST action that attaches (or, with an + // empty securityPolicy, detaches) an edge security policy. + actionSetEdgeSecurityPolicy = "setEdgeSecurityPolicy" + + fieldID = "id" + fieldKind = "kind" + fieldSelfLink = "selfLink" + fieldCreationTimestamp = "creationTimestamp" + fieldUsedBy = "usedBy" + fieldParams = "params" + fieldBucketName = "bucketName" + fieldEdgeSecurityPolicy = "edgeSecurityPolicy" + + opInsert = "insert" + opPatch = "patch" + opUpdate = "update" + opDelete = "delete" +) + +// backendBucketOutputOnly are members a client may echo back from a get but the +// server owns: they are dropped from insert/patch/update bodies. params is +// input-only and never persisted; edgeSecurityPolicy is output-only and set +// solely through setEdgeSecurityPolicy. +// +//nolint:gochecknoglobals // immutable lookup table, not mutable state +var backendBucketOutputOnly = []string{ + fieldID, fieldKind, fieldSelfLink, fieldCreationTimestamp, fieldUsedBy, fieldParams, fieldEdgeSecurityPolicy, +} + +// BucketLister is the slice of the storage driver the handler needs to check +// that a backend bucket's bucketName names an existing Cloud Storage bucket. +type BucketLister interface { + ListBuckets(ctx context.Context) ([]storagedriver.BucketInfo, error) +} + +// SetBucketLister wires the Cloud Storage backend so backendBuckets insert, +// patch and update reject a bucketName that names no existing bucket. Without +// it bucketName is only required to be present. +func (h *Handler) SetBucketLister(b BucketLister) { h.buckets = b } + +// backendBucketStore returns the backend-bucket capability, or false when the +// backing driver does not implement it (non-GCP driver). +func (h *Handler) backendBucketStore() (lbdriver.GCPBackendBucketStore, bool) { + s, ok := h.lb.(lbdriver.GCPBackendBucketStore) + + return s, ok +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeBackendBuckets(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { + store, ok := h.backendBucketStore() + if !ok || rp.Scope != gcprest.ScopeGlobal { + gcprest.WriteError(w, http.StatusNotFound, "notFound", "backendBuckets are global only") + return + } + + switch { + case rp.ResourceName == "": + h.routeBackendBucketCollection(w, r, rp, store) + case rp.Action != "": + h.backendBucketAction(w, r, rp, store) + default: + h.routeBackendBucketItem(w, r, rp, store) + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeBackendBucketCollection(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + switch r.Method { + case http.MethodPost: + h.insertBackendBucket(w, r, rp, store) + case http.MethodGet: + listBackendBuckets(w, r, rp, store) + default: + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeBackendBucketItem(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + switch r.Method { + case http.MethodGet: + getBackendBucket(w, r, rp, store) + case http.MethodPatch: + h.mutateBackendBucket(w, r, rp, store, true) + case http.MethodPut: + h.mutateBackendBucket(w, r, rp, store, false) + case http.MethodDelete: + h.deleteBackendBucket(w, r, rp, store) + default: + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) backendBucketAction(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + if r.Method != http.MethodPost || rp.Action != actionSetEdgeSecurityPolicy { + gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", + "backendBuckets."+rp.Action+" is not implemented") + + return + } + + var req struct { + SecurityPolicy string `json:"securityPolicy"` + } + + if !gcprest.DecodeJSON(w, r, &req) { + return + } + + // The emulator has no securityPolicies resource, so the reference is stored + // as given; an empty reference detaches the policy. + err := store.UpdateGCPBackendBucket(r.Context(), rp.ResourceName, func(res *lbdriver.GCPResource) error { + body := deepCopyMap(res.Body) + if req.SecurityPolicy == "" { + delete(body, fieldEdgeSecurityPolicy) + } else { + body[fieldEdgeSecurityPolicy] = req.SecurityPolicy + } + + res.Body = body + + return nil + }) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeBackendBucketOp(w, r, rp, rp.ResourceName, actionSetEdgeSecurityPolicy) +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) insertBackendBucket(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + var body map[string]any + if !gcprest.DecodeJSON(w, r, &body) { + return + } + + name, _ := body["name"].(string) + if err := validateRFC1035Name(name); err != nil { + gcprest.WriteCErr(w, err) + return + } + + stripOutputOnly(body) + applyBackendBucketDefaults(body) + + if err := h.validateBackendBucket(r.Context(), body, true); err != nil { + gcprest.WriteCErr(w, err) + return + } + + res := lbdriver.GCPResource{ + Name: name, + ID: numericID(resourceBackendBuckets + "/" + name), + CreationTimestamp: time.Now().UTC().Format(time.RFC3339), + Body: body, + } + + if err := store.InsertGCPBackendBucket(r.Context(), res); err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeBackendBucketOp(w, r, rp, name, opInsert) +} + +// mutateBackendBucket serves backendBuckets.patch (merge=true, RFC 7386 JSON +// merge patch: only members present in the body change, nested objects such as +// cdnPolicy merge member-by-member, null removes) and backendBuckets.update +// (merge=false, full replace). The merged result is validated under the store +// lock, so a rejected change leaves the stored record untouched. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) mutateBackendBucket(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, merge bool, +) { + var body map[string]any + if !gcprest.DecodeJSON(w, r, &body) { + return + } + + stripOutputOnly(body) + + // The GCS lookup happens outside the store lock; only a changed bucketName + // needs it. + if _, present := body[fieldBucketName]; present { + if err := h.requireStorageBucket(r.Context(), body[fieldBucketName]); err != nil { + gcprest.WriteCErr(w, err) + return + } + } + + err := store.UpdateGCPBackendBucket(r.Context(), rp.ResourceName, func(res *lbdriver.GCPResource) error { + next := nextBackendBucketBody(res.Body, body, merge) + next["name"] = res.Name + + if err := h.validateBackendBucket(r.Context(), next, false); err != nil { + return err + } + + res.Body = next + + return nil + }) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + verb := opUpdate + if merge { + verb = opPatch + } + + h.writeBackendBucketOp(w, r, rp, rp.ResourceName, verb) +} + +// nextBackendBucketBody computes the body a patch or update produces from the +// stored body, without touching the stored map. +func nextBackendBucketBody(stored, req map[string]any, merge bool) map[string]any { + if merge { + next := deepCopyMap(stored) + mergePatch(next, req) + applyBackendBucketDefaults(next) + + return next + } + + next := deepCopyMap(req) + // edgeSecurityPolicy is output-only: a full replace keeps the attached policy. + if policy, ok := stored[fieldEdgeSecurityPolicy]; ok { + next[fieldEdgeSecurityPolicy] = policy + } + + applyBackendBucketDefaults(next) + + return next +} + +//nolint:gocritic // rp is a request-scoped value +func getBackendBucket(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, store lbdriver.GCPBackendBucketStore) { + res, err := store.GetGCPBackendBucket(r.Context(), rp.ResourceName) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, gcpResourceJSON(res, rp, hostOf(r))) +} + +//nolint:gocritic // rp is a request-scoped value +func listBackendBuckets(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, store lbdriver.GCPBackendBucketStore) { + items, err := store.ListGCPBackendBuckets(r.Context()) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + writeGCPResourceList(w, r, rp, items) +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) deleteBackendBucket(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + // Real GCP refuses to delete a backend bucket a url-map still routes to (400 + // resourceInUseByAnotherResource), so the url-map is never left dangling. + if ref := h.urlMapRefBackendBucket(r.Context(), rp.ResourceName); ref != "" { + gcprest.WriteError(w, http.StatusBadRequest, reasonResourceInUse, + "The "+singularOf(resourceBackendBuckets)+" resource '"+rp.ResourceName+"' is already being used by '"+ref+"'") + + return + } + + if err := store.DeleteGCPBackendBucket(r.Context(), rp.ResourceName); err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeBackendBucketOp(w, r, rp, rp.ResourceName, opDelete) +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) writeBackendBucketOp(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, name, opType string) { + op := h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, resourceBackendBuckets, name, opType) + gcprest.WriteJSON(w, http.StatusOK, op) +} + +// urlMapRefBackendBucket returns the name of a global url-map whose +// defaultService / pathMatchers[].defaultService / pathRules[].service / +// routeRules[].service names the backend bucket, or "" when none does. +func (h *Handler) urlMapRefBackendBucket(ctx context.Context, name string) string { + store, ok := h.gcpStore() + if !ok { + return "" + } + + maps, err := store.ListGCPResources(ctx, resourceURLMaps, gcprest.ScopeGlobal) + if err != nil { + return "" + } + + for i := range maps { + var refs []namedRef + + collectRefs(maps[i].Body, urlMapServiceFields, &refs) + + for _, ref := range refs { + if isBackendBucketRef(ref.value) && lastPathSegment(ref.value) == name { + return maps[i].Name + } + } + } + + return "" +} + +// requireStorageBucket rejects a bucketName that names no existing Cloud +// Storage bucket. It is a no-op when no storage backend is wired. +func (h *Handler) requireStorageBucket(ctx context.Context, v any) error { + name, _ := v.(string) + if name == "" || h.buckets == nil { + return nil + } + + buckets, err := h.buckets.ListBuckets(ctx) + if err != nil { + return err + } + + for i := range buckets { + if buckets[i].Name == name { + return nil + } + } + + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.bucketName': '%s'. The referenced Cloud Storage bucket cannot be found.", name) +} + +// validateBackendBucket checks a complete backend-bucket body: bucketName is +// required (and, on insert, must name an existing Cloud Storage bucket), and +// compressionMode / cdnPolicy must hold values the real API accepts. +func (h *Handler) validateBackendBucket(ctx context.Context, body map[string]any, checkBucket bool) error { + bucket, _ := body[fieldBucketName].(string) + if bucket == "" { + return cerrors.New(cerrors.InvalidArgument, "Invalid value for field 'resource.bucketName': ''. Required.") + } + + if checkBucket { + if err := h.requireStorageBucket(ctx, bucket); err != nil { + return err + } + } + + if err := validateCompressionMode(body["compressionMode"]); err != nil { + return err + } + + return validateCDNPolicy(body["cdnPolicy"]) +} + +// stripOutputOnly drops server-owned members from a request body. +func stripOutputOnly(body map[string]any) { + for _, k := range backendBucketOutputOnly { + delete(body, k) + } +} + +// mergePatch applies an RFC 7386 JSON merge patch: objects merge recursively, +// a null member removes the target member, anything else (including arrays) +// replaces it. +func mergePatch(dst, patch map[string]any) { + for k, v := range patch { + if v == nil { + delete(dst, k) + continue + } + + sub, isMap := v.(map[string]any) + cur, curIsMap := dst[k].(map[string]any) + + if isMap && curIsMap { + mergePatch(cur, sub) + continue + } + + dst[k] = deepCopyValue(v) + } +} + +// deepCopyMap copies a decoded JSON object so the copy shares no nested map or +// slice with the original. +func deepCopyMap(m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = deepCopyValue(v) + } + + return out +} + +func deepCopyValue(v any) any { + switch t := v.(type) { + case map[string]any: + return deepCopyMap(t) + case []any: + out := make([]any, len(t)) + for i := range t { + out[i] = deepCopyValue(t[i]) + } + + return out + default: + return v + } +} diff --git a/server/gcp/loadbalancer/backendbuckets_sdk_test.go b/server/gcp/loadbalancer/backendbuckets_sdk_test.go new file mode 100644 index 000000000..d7481e2a9 --- /dev/null +++ b/server/gcp/loadbalancer/backendbuckets_sdk_test.go @@ -0,0 +1,462 @@ +package loadbalancer_test + +import ( + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + + gcpcompute "cloud.google.com/go/compute/apiv1" + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + + "github.com/stackshy/cloudemu/v2" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +// gcsBucket is the Cloud Storage bucket the backend buckets in these tests serve. +const gcsBucket = "static-assets" + +// newCDNServer serves the LB + Compute handlers with the GCS backend wired, so +// backendBuckets.bucketName is checked against real Cloud Storage buckets. +func newCDNServer(t *testing.T, buckets ...string) *httptest.Server { + t.Helper() + + cloudP := cloudemu.NewGCP() + + for _, b := range buckets { + if err := cloudP.GCS.CreateBucket(context.Background(), b); err != nil { + t.Fatalf("CreateBucket %s: %v", b, err) + } + } + + srv := gcpserver.New(gcpserver.Drivers{LB: cloudP.LB, Compute: cloudP.GCE, Storage: cloudP.GCS}) + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + return ts +} + +func newBackendBucketsClient(t *testing.T, ts *httptest.Server) *gcpcompute.BackendBucketsClient { + t.Helper() + + c, err := gcpcompute.NewBackendBucketsRESTClient(context.Background(), clientOpts(ts)...) + if err != nil { + t.Fatalf("NewBackendBucketsRESTClient: %v", err) + } + + t.Cleanup(func() { _ = c.Close() }) + + return c +} + +// bbRef renders a relative backend-bucket reference. +func bbRef(name string) string { + return "projects/" + testProject + "/global/backendBuckets/" + name +} + +// insertBB inserts bb and waits for the operation, failing the test on error. +func insertBB(ctx context.Context, t *testing.T, c *gcpcompute.BackendBucketsClient, bb *computepb.BackendBucket) { + t.Helper() + + waitOp(ctx, t, "BackendBucket Insert "+bb.GetName(), func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertBackendBucketRequest{Project: testProject, BackendBucketResource: bb}) + }) +} + +// callOp runs a mutating call and waits for its operation, returning the first error. +func callOp(ctx context.Context, call func() (*gcpcompute.Operation, error)) error { + op, err := call() + if err != nil { + return err + } + + return op.Wait(ctx) +} + +func getBB(ctx context.Context, t *testing.T, c *gcpcompute.BackendBucketsClient, name string) *computepb.BackendBucket { + t.Helper() + + got, err := c.Get(ctx, &computepb.GetBackendBucketRequest{Project: testProject, BackendBucket: name}) + if err != nil { + t.Fatalf("BackendBucket Get %s: %v", name, err) + } + + return got +} + +// assertHTTPCode fails unless err is a googleapi.Error carrying code. +func assertHTTPCode(t *testing.T, err error, code int) { + t.Helper() + + var gerr *googleapi.Error + if !errors.As(err, &gerr) { + t.Fatalf("error = %v, want a googleapi.Error with code %d", err, code) + } + + if gerr.Code != code { + t.Fatalf("error code = %d, want %d (%v)", gerr.Code, code, err) + } +} + +func cdnBucket(name string) *computepb.BackendBucket { + return &computepb.BackendBucket{ + Name: ptrStr(name), + BucketName: ptrStr(gcsBucket), + Description: ptrStr("static site"), + EnableCdn: ptrBool(true), + CompressionMode: ptrStr("AUTOMATIC"), + CustomResponseHeaders: []string{"X-Cache-Status: {cdn_cache_status}"}, + CdnPolicy: &computepb.BackendBucketCdnPolicy{ + CacheMode: ptrStr("FORCE_CACHE_ALL"), + DefaultTtl: ptrI32(60), + MaxTtl: ptrI32(600), + ClientTtl: ptrI32(30), + NegativeCaching: ptrBool(true), + SignedUrlCacheMaxAgeSec: func() *int64 { v := int64(7200); return &v }(), + RequestCoalescing: ptrBool(true), + }, + } +} + +// TestSDKGCPBackendBucketLifecycle drives insert → get → patch → update → +// setEdgeSecurityPolicy → delete → get(404) through the real BackendBuckets +// client, waiting on every global operation. +func TestSDKGCPBackendBucketLifecycle(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + insertBB(ctx, t, c, cdnBucket("site-bb")) + assertInsertedBB(t, getBB(ctx, t, c, "site-bb")) + + // PATCH: only members present change; cdnPolicy merges member-by-member. + waitOp(ctx, t, "Patch", func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendBucketRequest{ + Project: testProject, BackendBucket: "site-bb", + BackendBucketResource: &computepb.BackendBucket{ + Description: ptrStr("patched"), + CdnPolicy: &computepb.BackendBucketCdnPolicy{DefaultTtl: ptrI32(120)}, + }, + }) + }) + assertPatchedBB(t, getBB(ctx, t, c, "site-bb")) + + waitOp(ctx, t, "SetEdgeSecurityPolicy", func() (*gcpcompute.Operation, error) { + return c.SetEdgeSecurityPolicy(ctx, &computepb.SetEdgeSecurityPolicyBackendBucketRequest{ + Project: testProject, BackendBucket: "site-bb", + SecurityPolicyReferenceResource: &computepb.SecurityPolicyReference{SecurityPolicy: ptrStr("edge-policy")}, + }) + }) + + // PUT: full replace; omitted members are gone, output-only edgeSecurityPolicy stays. + waitOp(ctx, t, "Update", func() (*gcpcompute.Operation, error) { + return c.Update(ctx, &computepb.UpdateBackendBucketRequest{ + Project: testProject, BackendBucket: "site-bb", + BackendBucketResource: &computepb.BackendBucket{Name: ptrStr("site-bb"), BucketName: ptrStr(gcsBucket)}, + }) + }) + assertReplacedBB(t, getBB(ctx, t, c, "site-bb")) + + waitOp(ctx, t, "Delete", func() (*gcpcompute.Operation, error) { + return c.Delete(ctx, &computepb.DeleteBackendBucketRequest{Project: testProject, BackendBucket: "site-bb"}) + }) + + _, err := c.Get(ctx, &computepb.GetBackendBucketRequest{Project: testProject, BackendBucket: "site-bb"}) + assertHTTPCode(t, err, 404) + + _, err = c.Delete(ctx, &computepb.DeleteBackendBucketRequest{Project: testProject, BackendBucket: "site-bb"}) + assertHTTPCode(t, err, 404) +} + +func assertInsertedBB(t *testing.T, got *computepb.BackendBucket) { + t.Helper() + + if got.GetKind() != "compute#backendBucket" || got.GetId() == 0 || got.GetCreationTimestamp() == "" { + t.Errorf("identity: kind=%q id=%d created=%q", got.GetKind(), got.GetId(), got.GetCreationTimestamp()) + } + + if !strings.HasSuffix(got.GetSelfLink(), "/compute/v1/"+bbRef("site-bb")) { + t.Errorf("selfLink = %q", got.GetSelfLink()) + } + + if got.GetBucketName() != gcsBucket || !got.GetEnableCdn() || got.GetCompressionMode() != "AUTOMATIC" || + got.GetDescription() != "static site" || len(got.GetCustomResponseHeaders()) != 1 { + t.Errorf("fields did not round-trip: %v", got) + } + + p := got.GetCdnPolicy() + if p.GetCacheMode() != "FORCE_CACHE_ALL" || p.GetDefaultTtl() != 60 || p.GetMaxTtl() != 600 || + p.GetClientTtl() != 30 || !p.GetNegativeCaching() || p.GetSignedUrlCacheMaxAgeSec() != 7200 || + !p.GetRequestCoalescing() { + t.Errorf("cdnPolicy did not round-trip: %v", p) + } +} + +func assertPatchedBB(t *testing.T, got *computepb.BackendBucket) { + t.Helper() + + if got.GetDescription() != "patched" { + t.Errorf("description = %q, want patched", got.GetDescription()) + } + + if got.GetBucketName() != gcsBucket || !got.GetEnableCdn() || len(got.GetCustomResponseHeaders()) != 1 { + t.Errorf("patch clobbered members it did not name: %v", got) + } + + p := got.GetCdnPolicy() + if p.GetDefaultTtl() != 120 || p.GetCacheMode() != "FORCE_CACHE_ALL" || p.GetMaxTtl() != 600 { + t.Errorf("cdnPolicy after patch = %v, want defaultTtl=120 with cacheMode/maxTtl kept", p) + } +} + +func assertReplacedBB(t *testing.T, got *computepb.BackendBucket) { + t.Helper() + + if got.Description != nil || got.EnableCdn != nil || got.CdnPolicy != nil || len(got.GetCustomResponseHeaders()) != 0 { + t.Errorf("update did not replace the resource: %v", got) + } + + if got.GetEdgeSecurityPolicy() != "edge-policy" { + t.Errorf("edgeSecurityPolicy = %q, want edge-policy (output-only, kept across update)", got.GetEdgeSecurityPolicy()) + } +} + +// TestSDKGCPBackendBucketListPaging lists with maxResults=1 (the iterator +// follows nextPageToken) and with a name filter. +func TestSDKGCPBackendBucketListPaging(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + for _, n := range []string{"bb-c", "bb-a", "bb-b"} { + insertBB(ctx, t, c, &computepb.BackendBucket{Name: ptrStr(n), BucketName: ptrStr(gcsBucket)}) + } + + all := listBBNames(ctx, t, c, &computepb.ListBackendBucketsRequest{Project: testProject, MaxResults: func() *uint32 { + v := uint32(1) + return &v + }()}) + if strings.Join(all, ",") != "bb-a,bb-b,bb-c" { + t.Errorf("paged list = %v, want [bb-a bb-b bb-c]", all) + } + + if pages := countBBPages(t, c.List(ctx, &computepb.ListBackendBucketsRequest{Project: testProject})); pages != 3 { + t.Errorf("pages at page size 1 = %d, want 3 (server must honor maxResults + pageToken)", pages) + } + + filtered := listBBNames(ctx, t, c, &computepb.ListBackendBucketsRequest{Project: testProject, Filter: ptrStr("name = bb-b")}) + if strings.Join(filtered, ",") != "bb-b" { + t.Errorf("filtered list = %v, want [bb-b]", filtered) + } +} + +// countBBPages walks it one item per page, returning how many pages the server served. +func countBBPages(t *testing.T, it *gcpcompute.BackendBucketIterator) int { + t.Helper() + + pager := iterator.NewPager(it, 1, "") + + for pages := 0; ; pages++ { + var page []*computepb.BackendBucket + + token, err := pager.NextPage(&page) + if err != nil { + t.Fatalf("NextPage: %v", err) + } + + if len(page) > 1 { + t.Fatalf("page holds %d items, want <= 1", len(page)) + } + + if token == "" { + return pages + 1 + } + } +} + +func listBBNames(ctx context.Context, t *testing.T, c *gcpcompute.BackendBucketsClient, req *computepb.ListBackendBucketsRequest) []string { + t.Helper() + + var names []string + + it := c.List(ctx, req) + + for { + bb, err := it.Next() + if errors.Is(err, iterator.Done) { + return names + } + + if err != nil { + t.Fatalf("List: %v", err) + } + + names = append(names, bb.GetName()) + } +} + +// TestSDKGCPBackendBucketInsertValidation covers the insert-time 400s and the +// duplicate 409. +func TestSDKGCPBackendBucketInsertValidation(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + withPolicy := func(p *computepb.BackendBucketCdnPolicy) *computepb.BackendBucket { + return &computepb.BackendBucket{Name: ptrStr("bad-bb"), BucketName: ptrStr(gcsBucket), CdnPolicy: p} + } + + cases := map[string]*computepb.BackendBucket{ + "missing bucketName": {Name: ptrStr("bad-bb")}, + "unknown GCS bucket": {Name: ptrStr("bad-bb"), BucketName: ptrStr("no-such-bucket")}, + "non-RFC1035 name": {Name: ptrStr("Bad_Name"), BucketName: ptrStr(gcsBucket)}, + "compressionMode": {Name: ptrStr("bad-bb"), BucketName: ptrStr(gcsBucket), CompressionMode: ptrStr("ZSTD")}, + "cacheMode": withPolicy(&computepb.BackendBucketCdnPolicy{CacheMode: ptrStr("BOGUS")}), + "defaultTtl>maxTtl": withPolicy(&computepb.BackendBucketCdnPolicy{DefaultTtl: ptrI32(900), MaxTtl: ptrI32(60)}), + "maxTtl over 1 year": withPolicy(&computepb.BackendBucketCdnPolicy{MaxTtl: ptrI32(31622401)}), + "negative clientTtl": withPolicy(&computepb.BackendBucketCdnPolicy{ClientTtl: ptrI32(-1)}), + "negativeCachingPolicy without negativeCaching": withPolicy(&computepb.BackendBucketCdnPolicy{ + NegativeCachingPolicy: []*computepb.BackendBucketCdnPolicyNegativeCachingPolicy{{Code: ptrI32(404), Ttl: ptrI32(60)}}, + }), + } + + for name, bb := range cases { + t.Run(name, func(t *testing.T) { + err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertBackendBucketRequest{Project: testProject, BackendBucketResource: bb}) + }) + assertHTTPCode(t, err, 400) + }) + } + + insertBB(ctx, t, c, &computepb.BackendBucket{Name: ptrStr("dup-bb"), BucketName: ptrStr(gcsBucket)}) + + err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertBackendBucketRequest{Project: testProject, + BackendBucketResource: &computepb.BackendBucket{Name: ptrStr("dup-bb"), BucketName: ptrStr(gcsBucket)}}) + }) + assertHTTPCode(t, err, 409) + + _, err = c.Get(ctx, &computepb.GetBackendBucketRequest{Project: testProject, BackendBucket: "ghost-bb"}) + assertHTTPCode(t, err, 404) +} + +// TestSDKGCPBackendBucketMutationValidation covers patch/update 400s (checked +// against the merged result) and that a rejected change leaves the record as it was. +func TestSDKGCPBackendBucketMutationValidation(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + insertBB(ctx, t, c, cdnBucket("site-bb")) + + patch := func(bb *computepb.BackendBucket) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendBucketRequest{ + Project: testProject, BackendBucket: "site-bb", BackendBucketResource: bb, + }) + }) + } + + // defaultTtl alone is in range, but exceeds the stored maxTtl (600) once merged. + assertHTTPCode(t, patch(&computepb.BackendBucket{CdnPolicy: &computepb.BackendBucketCdnPolicy{DefaultTtl: ptrI32(900)}}), 400) + assertHTTPCode(t, patch(&computepb.BackendBucket{BucketName: ptrStr("no-such-bucket")}), 400) + assertHTTPCode(t, patch(&computepb.BackendBucket{CdnPolicy: &computepb.BackendBucketCdnPolicy{CacheMode: ptrStr("NOPE")}}), 400) + + err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Update(ctx, &computepb.UpdateBackendBucketRequest{ + Project: testProject, BackendBucket: "site-bb", + BackendBucketResource: &computepb.BackendBucket{Name: ptrStr("site-bb")}, + }) + }) + assertHTTPCode(t, err, 400) + + assertInsertedBB(t, getBB(ctx, t, c, "site-bb")) + + assertHTTPCode(t, callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendBucketRequest{ + Project: testProject, BackendBucket: "ghost-bb", + BackendBucketResource: &computepb.BackendBucket{Description: ptrStr("x")}, + }) + }), 404) +} + +// TestSDKGCPURLMapBackendBucketReferences proves url-map references to backend +// buckets are validated (defaultService, pathMatchers[].defaultService, +// pathRules[].service; full URL or relative path) and that a referenced backend +// bucket cannot be deleted until the url-map is gone. +func TestSDKGCPURLMapBackendBucketReferences(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + bb := newBackendBucketsClient(t, ts) + + um, err := gcpcompute.NewUrlMapsRESTClient(ctx, clientOpts(ts)...) + if err != nil { + t.Fatalf("NewUrlMapsRESTClient: %v", err) + } + + t.Cleanup(func() { _ = um.Close() }) + + insertMap := func(m *computepb.UrlMap) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return um.Insert(ctx, &computepb.InsertUrlMapRequest{Project: testProject, UrlMapResource: m}) + }) + } + + assertInvalidArgument(t, insertMap(&computepb.UrlMap{Name: ptrStr("cdn-map"), DefaultService: ptrStr(bbRef("ghost-bb"))})) + + insertBB(ctx, t, bb, &computepb.BackendBucket{Name: ptrStr("real-bb"), BucketName: ptrStr(gcsBucket)}) + + assertInvalidArgument(t, insertMap(&computepb.UrlMap{ + Name: ptrStr("cdn-map"), DefaultService: ptrStr(bbRef("real-bb")), + PathMatchers: []*computepb.PathMatcher{{ + Name: ptrStr("pm"), DefaultService: ptrStr(bbRef("real-bb")), + PathRules: []*computepb.PathRule{{Paths: []string{"/img/*"}, Service: ptrStr(bbRef("ghost-bb"))}}, + }}, + })) + + fullURL := ts.URL + "/compute/v1/" + bbRef("real-bb") + if err := insertMap(&computepb.UrlMap{ + Name: ptrStr("cdn-map"), DefaultService: ptrStr(fullURL), + PathMatchers: []*computepb.PathMatcher{{ + Name: ptrStr("pm"), DefaultService: ptrStr(bbRef("real-bb")), + PathRules: []*computepb.PathRule{{Paths: []string{"/img/*"}, Service: ptrStr(bbRef("real-bb"))}}, + }}, + }); err != nil { + t.Fatalf("UrlMap Insert referencing an existing backend bucket: %v", err) + } + + deleteBB := func() error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return bb.Delete(ctx, &computepb.DeleteBackendBucketRequest{Project: testProject, BackendBucket: "real-bb"}) + }) + } + + assertResourceInUse(t, deleteBB()) + + waitOp(ctx, t, "UrlMap Delete", func() (*gcpcompute.Operation, error) { + return um.Delete(ctx, &computepb.DeleteUrlMapRequest{Project: testProject, UrlMap: "cdn-map"}) + }) + + if err := deleteBB(); err != nil { + t.Fatalf("Delete after url-map removed: %v", err) + } +} + +// TestSDKGCPBackendBucketWithoutStorage: with no GCS backend wired, bucketName is +// only required to be present. +func TestSDKGCPBackendBucketWithoutStorage(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + insertBB(ctx, t, c, &computepb.BackendBucket{Name: ptrStr("any-bb"), BucketName: ptrStr("unchecked-bucket")}) + + if got := getBB(ctx, t, c, "any-bb"); got.GetBucketName() != "unchecked-bucket" { + t.Errorf("bucketName = %q", got.GetBucketName()) + } +} diff --git a/server/gcp/loadbalancer/backendbuckets_validate.go b/server/gcp/loadbalancer/backendbuckets_validate.go new file mode 100644 index 000000000..bbc56d62d --- /dev/null +++ b/server/gcp/loadbalancer/backendbuckets_validate.go @@ -0,0 +1,210 @@ +package loadbalancer + +import ( + "math" + "regexp" + "strconv" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +// Cloud CDN limits, from the BackendBucketCdnPolicy field docs in +// cloud.google.com/go/compute/apiv1/computepb (compute v1.60.0). +const ( + // maxCDNTTLSeconds is the largest defaultTtl / maxTtl / clientTtl (1 year). + maxCDNTTLSeconds = 31622400 + // maxServeWhileStaleSeconds is the largest serveWhileStale (1 week). + maxServeWhileStaleSeconds = 604800 + // maxBypassCacheHeaders is how many bypassCacheOnRequestHeaders are allowed. + maxBypassCacheHeaders = 5 + + cacheModeCacheAllStatic = "CACHE_ALL_STATIC" + + fieldCacheMode = "cacheMode" + fieldDefaultTTL = "defaultTtl" + fieldMaxTTL = "maxTtl" + + fieldService = "service" + fieldDefaultService = "defaultService" +) + +// rfc1035Name is the compute resource-name grammar: 1-63 characters, a +// lowercase letter first, then lowercase letters, digits or dashes, not ending +// in a dash. +var rfc1035Name = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + +// validCacheModes are the cdnPolicy.cacheMode values the API accepts. +// +//nolint:gochecknoglobals // immutable lookup table, not mutable state +var validCacheModes = map[string]bool{ + cacheModeCacheAllStatic: true, + "USE_ORIGIN_HEADERS": true, + "FORCE_CACHE_ALL": true, +} + +// validCompressionModes are the compressionMode values the API accepts. +// +//nolint:gochecknoglobals // immutable lookup table, not mutable state +var validCompressionModes = map[string]bool{ + "AUTOMATIC": true, + "DISABLED": true, +} + +// urlMapServiceFields are the url-map members that name a backend service or +// backend bucket. +// +//nolint:gochecknoglobals // immutable lookup table, not mutable state +var urlMapServiceFields = map[string]bool{fieldService: true, fieldDefaultService: true} + +// isBackendBucketRef reports whether ref is a backendBuckets self-link or +// relative path (full URL or "projects/{p}/global/backendBuckets/{name}"). +func isBackendBucketRef(ref string) bool { + return strings.Contains(ref, "/"+resourceBackendBuckets+"/") +} + +// validateRFC1035Name rejects a missing or malformed resource name. +func validateRFC1035Name(name string) error { + if !rfc1035Name.MatchString(name) { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.name': '%s'. Must be a match of regex '(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)'", name) + } + + return nil +} + +// applyBackendBucketDefaults fills cdnPolicy.cacheMode with the documented +// default (CACHE_ALL_STATIC) when Cloud CDN is enabled or a cdnPolicy is given +// without one. +func applyBackendBucketDefaults(body map[string]any) { + enabled, _ := body["enableCdn"].(bool) + policy, hasPolicy := body["cdnPolicy"].(map[string]any) + + if !enabled && !hasPolicy { + return + } + + if !hasPolicy { + policy = map[string]any{} + body["cdnPolicy"] = policy + } + + if _, ok := policy[fieldCacheMode]; !ok { + policy[fieldCacheMode] = cacheModeCacheAllStatic + } +} + +// validateCompressionMode rejects an unrecognized compressionMode. +func validateCompressionMode(v any) error { + if v == nil { + return nil + } + + mode, _ := v.(string) + if !validCompressionModes[mode] { + return cerrors.Newf(cerrors.InvalidArgument, "Invalid value for field 'resource.compressionMode': '%v'.", v) + } + + return nil +} + +// validateCDNPolicy checks the cdnPolicy members with documented constraints; +// every other member passes through untouched. +func validateCDNPolicy(v any) error { + if v == nil { + return nil + } + + policy, ok := v.(map[string]any) + if !ok { + return cerrors.New(cerrors.InvalidArgument, "Invalid value for field 'resource.cdnPolicy': must be an object.") + } + + if mode, present := policy[fieldCacheMode]; present { + if s, _ := mode.(string); !validCacheModes[s] { + return cerrors.Newf(cerrors.InvalidArgument, "Invalid value for field 'resource.cdnPolicy.cacheMode': '%v'.", mode) + } + } + + if err := validateCDNRanges(policy); err != nil { + return err + } + + return validateCDNLists(policy) +} + +// validateCDNRanges checks the TTL bounds and that defaultTtl <= maxTtl. +func validateCDNRanges(policy map[string]any) error { + limits := []struct { + field string + max int64 + }{ + {fieldDefaultTTL, maxCDNTTLSeconds}, + {fieldMaxTTL, maxCDNTTLSeconds}, + {"clientTtl", maxCDNTTLSeconds}, + {"serveWhileStale", maxServeWhileStaleSeconds}, + {"signedUrlCacheMaxAgeSec", math.MaxInt64}, + } + + for _, l := range limits { + raw, present := policy[l.field] + if !present { + continue + } + + n, ok := jsonInt(raw) + if !ok || n < 0 || n > l.max { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.%s': '%v'. Must be between 0 and %d.", l.field, raw, l.max) + } + } + + defTTL, hasDef := jsonInt(policy[fieldDefaultTTL]) + maxTTL, hasMax := jsonInt(policy[fieldMaxTTL]) + + if hasDef && hasMax && defTTL > maxTTL { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.defaultTtl': '%d'. defaultTtl cannot be greater than maxTtl (%d).", + defTTL, maxTTL) + } + + return nil +} + +// validateCDNLists checks the list-valued cdnPolicy members. +func validateCDNLists(policy map[string]any) error { + if headers, _ := policy["bypassCacheOnRequestHeaders"].([]any); len(headers) > maxBypassCacheHeaders { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.bypassCacheOnRequestHeaders': at most %d headers are allowed.", + maxBypassCacheHeaders) + } + + negPolicy, _ := policy["negativeCachingPolicy"].([]any) + negEnabled, _ := policy["negativeCaching"].(bool) + + if len(negPolicy) > 0 && !negEnabled { + return cerrors.New(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.negativeCachingPolicy': negativeCaching must be enabled.") + } + + return nil +} + +// jsonInt reads an integral JSON value: a number, or a decimal string (proto +// JSON encodes int64 fields such as signedUrlCacheMaxAgeSec as strings). +func jsonInt(v any) (int64, bool) { + switch t := v.(type) { + case float64: + if t != math.Trunc(t) || t < math.MinInt64 || t > math.MaxInt64 { + return 0, false + } + + return int64(t), true + case string: + n, err := strconv.ParseInt(t, 10, 64) + + return n, err == nil + default: + return 0, false + } +} diff --git a/server/gcp/loadbalancer/handler.go b/server/gcp/loadbalancer/handler.go index f3ee193ea..1918ed477 100644 --- a/server/gcp/loadbalancer/handler.go +++ b/server/gcp/loadbalancer/handler.go @@ -62,6 +62,9 @@ type Handler struct { // resolves a real operation and 404s a bogus one. Nil in a package-level // server (every operation poll answered DONE, legacy behavior). ops *gcprest.OperationRegistry + // buckets, when set, lets backendBuckets reject a bucketName that names no + // existing Cloud Storage bucket. + buckets BucketLister } // New returns a GCP load balancer handler backed by lb. @@ -75,6 +78,7 @@ func New(lb lbdriver.LoadBalancer) *Handler { func (h *Handler) SetOperationRegistry(reg *gcprest.OperationRegistry) { h.ops = reg } // Matches returns true for the load-balancing resource types: backendServices, +// backendBuckets (Cloud CDN), // forwardingRules, healthChecks, targetPools, urlMaps, the L7 front-end chain // (targetHttpProxies, targetHttpsProxies, sslCertificates) and instanceGroups / // regionInstanceGroups. Disjoint from the compute (instances/operations/disks/…) @@ -94,7 +98,7 @@ func (*Handler) Matches(r *http.Request) bool { } switch rp.ResourceType { - case resourceBackendServices, resourceForwardingRules, + case resourceBackendServices, resourceForwardingRules, resourceBackendBuckets, resourceHealthChecks, resourceTargetPools, resourceURLMaps, resourceTargetHTTPProxies, resourceTargetHTTPSProxies, resourceSslCertificates, resourceInstanceGroups, resourceRegionInstanceGroups: @@ -117,6 +121,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.routeBackendServices(w, r, rp) case resourceForwardingRules: h.routeForwardingRules(w, r, rp) + case resourceBackendBuckets: + h.routeBackendBuckets(w, r, rp) case resourceHealthChecks, resourceTargetPools, resourceURLMaps: h.routeGCPResource(w, r, rp) case resourceTargetHTTPProxies, resourceTargetHTTPSProxies, resourceSslCertificates, diff --git a/server/gcp/loadbalancer/l7frontend.go b/server/gcp/loadbalancer/l7frontend.go index ccc8a9d6a..efdc22894 100644 --- a/server/gcp/loadbalancer/l7frontend.go +++ b/server/gcp/loadbalancer/l7frontend.go @@ -318,6 +318,8 @@ func singularOf(collection string) string { return "health_check" case resourceURLMaps: return "url_map" + case resourceBackendBuckets: + return "backend_bucket" case resourceSslCertificates: return "ssl_certificate" case resourceTargetHTTPProxies: diff --git a/server/gcp/loadbalancer/referenceintegrity_sdk_test.go b/server/gcp/loadbalancer/referenceintegrity_sdk_test.go index be61ae3f9..781c6c515 100644 --- a/server/gcp/loadbalancer/referenceintegrity_sdk_test.go +++ b/server/gcp/loadbalancer/referenceintegrity_sdk_test.go @@ -591,9 +591,8 @@ func TestSDKGCPBackendServiceInvalidBalancingMode(t *testing.T) { // TestSDKGCPURLMapBackendBucketRefNotRejected guards against a false-reject: a // url-map's defaultService may legitimately name a backendBuckets/{name} // self-link (standard CDN/static-content routing, e.g. -// google_compute_backend_bucket.self_link). Backend buckets have no driver -// model here, so the reference must be left unvalidated rather than rejected -// as a dangling backend-service reference. +// google_compute_backend_bucket.self_link). An existing backend bucket must be +// accepted, not misread as a dangling backend-service reference. func TestSDKGCPURLMapBackendBucketRefNotRejected(t *testing.T) { ts := newGCPLBServer(t) ctx := context.Background() @@ -605,6 +604,9 @@ func TestSDKGCPURLMapBackendBucketRefNotRejected(t *testing.T) { t.Cleanup(func() { _ = um.Close() }) + insertBB(ctx, t, newBackendBucketsClient(t, ts), + &computepb.BackendBucket{Name: ptrStr("static-assets"), BucketName: ptrStr("static-assets")}) + bucketRef := "projects/" + testProject + "/global/backendBuckets/static-assets" insOp, err := um.Insert(ctx, &computepb.InsertUrlMapRequest{ diff --git a/server/gcp/loadbalancer/refvalidation.go b/server/gcp/loadbalancer/refvalidation.go index 650670b3c..b32acec3a 100644 --- a/server/gcp/loadbalancer/refvalidation.go +++ b/server/gcp/loadbalancer/refvalidation.go @@ -7,7 +7,7 @@ package loadbalancer // l7frontend.go so no create/update in the chain can point at a resource that // isn't there: // -// urlMap.defaultService / pathMatchers[].defaultService / pathRules[].service → backendService +// urlMap.defaultService / pathMatchers[].defaultService / pathRules[].service → backendService / backendBucket // targetHttp(s)Proxy.urlMap → urlMap // targetHttpsProxy.sslCertificates[] → sslCertificate // forwardingRule.target → targetHttp(s)Proxy / targetPool @@ -80,29 +80,31 @@ func collectRefs(v any, fields map[string]bool, out *[]namedRef) { // validateURLMapServiceRefs rejects a url-map body whose defaultService, or any // nested pathMatchers[].defaultService / pathRules[].service, names a backend -// service that does not exist in the same scope. The same fields can also -// legitimately name a backendBuckets/{name} self-link (standard CDN/static- -// content routing, e.g. google_compute_backend_bucket.self_link). Backend -// buckets have no driver model here, so any ref that doesn't resolve to the -// backendServices collection is left unvalidated rather than falsely rejected, -// mirroring targetCollectionFor's allowlist for forwarding-rule targets. +// service or backend bucket that does not exist. The same fields name either +// collection: a backendBuckets/{name} self-link (Cloud CDN / static-content +// routing, e.g. google_compute_backend_bucket.self_link) must resolve to an +// existing global backend bucket; a bare name or backendServices reference must +// resolve to a backend service in the url-map's scope. A reference to any other +// collection is left unvalidated, mirroring targetCollectionFor's allowlist for +// forwarding-rule targets. // //nolint:gocritic // rp is a request-scoped value func (h *Handler) validateURLMapServiceRefs(ctx context.Context, rp gcprest.ResourcePath, body map[string]any) error { var refs []namedRef - collectRefs(body, map[string]bool{"service": true, "defaultService": true}, &refs) + collectRefs(body, urlMapServiceFields, &refs) for _, ref := range refs { - if !isBackendServiceRef(ref.value) { - continue - } + var err error - if _, err := h.findTGByName(ctx, rp, backendServiceName(ref.value)); err != nil { - if cerrors.IsNotFound(err) { - return invalidRefErr(ref.field, ref.value, "backend service") - } + switch { + case isBackendBucketRef(ref.value): + err = h.requireBackendBucket(ctx, ref) + case isBackendServiceRef(ref.value): + err = h.requireBackendService(ctx, rp, ref) + } + if err != nil { return err } } @@ -110,11 +112,41 @@ func (h *Handler) validateURLMapServiceRefs(ctx context.Context, rp gcprest.Reso return nil } +// requireBackendService rejects a url-map reference naming a backend service +// that does not exist in the url-map's scope. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) requireBackendService(ctx context.Context, rp gcprest.ResourcePath, ref namedRef) error { + _, err := h.findTGByName(ctx, rp, backendServiceName(ref.value)) + if cerrors.IsNotFound(err) { + return invalidRefErr(ref.field, ref.value, "backend service") + } + + return err +} + +// requireBackendBucket rejects a url-map reference naming a backend bucket that +// does not exist. Backend buckets are global. A driver without the +// backend-bucket capability leaves the reference unvalidated. +func (h *Handler) requireBackendBucket(ctx context.Context, ref namedRef) error { + store, ok := h.backendBucketStore() + if !ok { + return nil + } + + _, err := store.GetGCPBackendBucket(ctx, lastPathSegment(ref.value)) + if cerrors.IsNotFound(err) { + return invalidRefErr(ref.field, ref.value, "backend bucket") + } + + return err +} + // isBackendServiceRef reports whether ref names the backendServices // collection: either a bare name (no path separators, the common case for a // same-scope reference) or a self-link/relative path containing -// "/backendServices/". Anything else (e.g. a backendBuckets self-link) is left -// unvalidated. +// "/backendServices/". A backendBuckets self-link is matched first by +// isBackendBucketRef; any other collection is left unvalidated. func isBackendServiceRef(ref string) bool { return !strings.Contains(ref, "/") || strings.Contains(ref, "/backendServices/") } diff --git a/server/gcp/loadbalancer/resources.go b/server/gcp/loadbalancer/resources.go index 2012df65d..d14d7cb72 100644 --- a/server/gcp/loadbalancer/resources.go +++ b/server/gcp/loadbalancer/resources.go @@ -31,6 +31,7 @@ var resourceKind = map[string]string{ resourceHealthChecks: "compute#healthCheck", resourceTargetPools: "compute#targetPool", resourceURLMaps: "compute#urlMap", + resourceBackendBuckets: "compute#backendBucket", resourceTargetHTTPProxies: "compute#targetHttpProxy", resourceTargetHTTPSProxies: "compute#targetHttpsProxy", resourceSslCertificates: "compute#sslCertificate", @@ -215,6 +216,14 @@ func (h *Handler) listGCPResource(w http.ResponseWriter, r *http.Request, rp gcp return } + writeGCPResourceList(w, r, rp, items) +} + +// writeGCPResourceList filters (name), sorts, paginates (maxResults/pageToken) +// and writes a compute#…List envelope over items of rp's collection. +// +//nolint:gocritic // rp is a request-scoped value +func writeGCPResourceList(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, items []lbdriver.GCPResource) { filter := r.URL.Query().Get("filter") matched := make([]lbdriver.GCPResource, 0, len(items)) diff --git a/services/loadbalancer/driver/gcp.go b/services/loadbalancer/driver/gcp.go index 246729fb1..4e04bcc59 100644 --- a/services/loadbalancer/driver/gcp.go +++ b/services/loadbalancer/driver/gcp.go @@ -48,3 +48,33 @@ type GCPBackendServicePatcher interface { // service exists. PatchGCPBackendService(ctx context.Context, name string, mutate func(*TargetGroupInfo)) error } + +// GCPBackendBucketCollection is the Collection a Cloud CDN backend bucket is +// stored under; backend buckets are always global. +const GCPBackendBucketCollection = "backendBuckets" + +// GCPBackendBucketStore is an OPTIONAL, type-asserted capability implemented +// only by the GCP load-balancer provider. It persists Cloud CDN backend buckets +// (compute.backendBuckets): a global load-balancer backend that serves a Cloud +// Storage bucket. Records are GCPResource values (Collection +// GCPBackendBucketCollection, Scope "global") living alongside the other +// opaque GCP resources, so they snapshot and restore with them. Non-GCP +// providers do not implement it. +type GCPBackendBucketStore interface { + // InsertGCPBackendBucket stores res, returning AlreadyExists when a backend + // bucket with the same name already exists. + InsertGCPBackendBucket(ctx context.Context, res GCPResource) error + // GetGCPBackendBucket returns the named backend bucket, or NotFound. + GetGCPBackendBucket(ctx context.Context, name string) (*GCPResource, error) + // ListGCPBackendBuckets returns every backend bucket. + ListGCPBackendBuckets(ctx context.Context) ([]GCPResource, error) + // UpdateGCPBackendBucket applies mutate to the named backend bucket under + // the store lock (compute backendBuckets.patch / update / + // setEdgeSecurityPolicy). When mutate returns an error the stored record is + // left unchanged and that error is returned; mutate must therefore replace + // Body rather than edit the stored map in place. Returns NotFound when absent. + UpdateGCPBackendBucket(ctx context.Context, name string, mutate func(*GCPResource) error) error + // DeleteGCPBackendBucket removes the named backend bucket, returning + // NotFound when absent. + DeleteGCPBackendBucket(ctx context.Context, name string) error +} From e42ed3cc8fc1641ba17307540b92197420f1efdf Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sat, 26 Sep 2026 21:52:33 +0530 Subject: [PATCH 05/24] feat(gcp): compute addresses and globalAddresses setLabels with labelFingerprint and label list filters POST .../regions/{r}/addresses/{name}/setLabels and .../global/addresses/{name}/setLabels previously answered 405 methodNotAllowed. They now replace the address's label set under labelFingerprint optimistic concurrency: the caller must send the current fingerprint (read from Get), and a missing or stale one is rejected 412 conditionNotMet with no change applied. Success returns a DONE compute Operation recorded in the shared gcprest OperationRegistry, so regionOperations/globalOperations polls resolve it. Get now always carries labelFingerprint (stamped at insert, recomputed on every setLabels), so a new label set reads back under a new fingerprint and an empty set clears the labels. Address list and aggregatedList filters now understand labels.= (and !=, eq, ne) in addition to name, instead of treating a label filter as match-all. Tested with the google.golang.org/api/compute/v1 AddressesService and GlobalAddressesService SetLabels clients. --- docs/sdk-server.md | 2 +- server/gcp/vpc/address_labels.go | 176 ++++++++++++++++++ server/gcp/vpc/address_labels_sdk_test.go | 213 ++++++++++++++++++++++ server/gcp/vpc/addresses.go | 15 +- 4 files changed, 403 insertions(+), 3 deletions(-) create mode 100644 server/gcp/vpc/address_labels.go create mode 100644 server/gcp/vpc/address_labels_sdk_test.go diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 46fbb654f..8b9ca4e60 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -218,7 +218,7 @@ All handlers speak REST + JSON. | Service | Operations | |---------|-----------| | **Compute Engine** | Instances + Disks + Snapshots + Images: insert/get/list/delete with LRO envelopes | -| **Networks** | VPCs, Subnetworks, Firewalls, Routes | +| **Networks** | VPCs, Subnetworks, Firewalls, Routes; regional + global Addresses (insert/get/list/aggregatedList/delete, `setLabels` replacing the label set under `labelFingerprint` with 412 `conditionNotMet` on a missing/stale fingerprint, list `filter` on `name` and `labels.=`) | | **Cloud Storage (GCS)** | Buckets + Objects: create/get/list/delete, upload, download, copy | | **Firestore** | Documents + Collections via `:commit`, `:batchGet`, `:runQuery` | | **Cloud Monitoring** | Time-series ingest/read, alert policies | diff --git a/server/gcp/vpc/address_labels.go b/server/gcp/vpc/address_labels.go new file mode 100644 index 000000000..a26b2b270 --- /dev/null +++ b/server/gcp/vpc/address_labels.go @@ -0,0 +1,176 @@ +package vpc + +import ( + "encoding/json" + "net/http" + "sort" + "strings" + + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" +) + +// setLabelsAction is the custom verb for POST .../addresses/{name}/setLabels +// (regional) and .../global/addresses/{name}/setLabels. +const setLabelsAction = "setLabels" + +// labelsFilterPrefix is the field prefix a compute list filter uses to match a +// label value, e.g. `labels.env=prod`. +const labelsFilterPrefix = "labels." + +// filterOpNotEqual is the inequality operator a compute list filter uses. +const filterOpNotEqual = "!=" + +// addressSetLabelsRequest is the RegionSetLabelsRequest / GlobalSetLabelsRequest +// body. Both carry the same two fields. +type addressSetLabelsRequest struct { + Labels map[string]string `json:"labels"` + LabelFingerprint string `json:"labelFingerprint"` +} + +// addressLabelFingerprint returns the fingerprint of an address's label set. +// It is a pure function of the labels, so it changes exactly when they do and +// an address with no labels still has a (stable, non-empty) fingerprint the +// caller must echo back, as real Compute Engine requires. +func addressLabelFingerprint(labels map[string]string) string { + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + + sort.Strings(keys) + + parts := make([]string, 0, len(keys)*2) //nolint:mnd // key and value per label + for _, k := range keys { + parts = append(parts, k, labels[k]) + } + + return fingerprintOf(append([]string{"labels"}, parts...)...) +} + +// addressLabels extracts the labels map from a stored address body. +func addressLabels(body json.RawMessage) map[string]string { + var withLabels struct { + Labels map[string]string `json:"labels"` + } + + _ = json.Unmarshal(body, &withLabels) + + return withLabels.Labels +} + +// setAddressLabels handles setLabels on a regional or global address. The +// request's labels REPLACE the whole set; the caller must send the current +// labelFingerprint (read from a Get), and a missing or stale one is rejected +// 412 conditionNotMet with no change applied. Success returns a DONE compute +// Operation recorded in the shared registry, and a later Get shows the new +// labels under a new labelFingerprint. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) setAddressLabels(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { + var req addressSetLabelsRequest + if !gcprest.DecodeJSON(w, r, &req) { + return + } + + scope := scopeOf(rp) + + status := h.addresses.replaceLabels(rp.Project, scope, rp.ResourceName, req) + + switch status { + case labelsNotFound: + gcprest.WriteError(w, http.StatusNotFound, "notFound", "address "+rp.ResourceName+" not found") + return + case labelsConditionNotMet: + gcprest.WriteError(w, http.StatusPreconditionFailed, "conditionNotMet", + "Labels fingerprint either invalid or resource labels have changed") + + return + case labelsInvalid: + gcprest.WriteError(w, http.StatusBadRequest, "invalid", "address body is not a JSON object") + return + case labelsOK: + } + + gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, + rp.Scope, rp.ScopeName, resourceAddresses, rp.ResourceName, setLabelsAction)) +} + +// labelsResult is the outcome of a replaceLabels attempt. +type labelsResult int + +const ( + labelsOK labelsResult = iota + labelsNotFound + labelsConditionNotMet + labelsInvalid +) + +// replaceLabels swaps an address's label set under the store lock, so the +// fingerprint check and the write are atomic against a concurrent setLabels. +func (s *addressStore) replaceLabels( + project, scope, name string, req addressSetLabelsRequest, +) labelsResult { + s.mu.Lock() + defer s.mu.Unlock() + + body, ok := s.addresses[s.key(project, scope)][name] + if !ok { + return labelsNotFound + } + + if req.LabelFingerprint == "" || req.LabelFingerprint != addressLabelFingerprint(addressLabels(body)) { + return labelsConditionNotMet + } + + var obj map[string]any + if err := json.Unmarshal(body, &obj); err != nil || obj == nil { + return labelsInvalid + } + + if len(req.Labels) == 0 { + delete(obj, "labels") + } else { + obj["labels"] = req.Labels + } + + obj["labelFingerprint"] = addressLabelFingerprint(req.Labels) + + out, err := json.Marshal(obj) + if err != nil { + return labelsInvalid + } + + s.addresses[s.key(project, scope)][name] = out + + return labelsOK +} + +// addressMatches applies a compute list filter to a stored address. It extends +// the shared name-only matcher with `labels.=` (and `!=`, `eq`, +// `ne`) so a label-scoped list returns only the addresses carrying that label. +// Any other field falls through to the shared matcher, which matches by name +// and treats fields it does not understand as match-all. +func addressMatches(filter string, body json.RawMessage) bool { + f := strings.TrimSpace(filter) + if !strings.HasPrefix(f, labelsFilterPrefix) { + return nameMatches(filter, rawName(body)) + } + + for _, cand := range []string{filterOpNotEqual, "=", " ne ", " eq "} { + idx := strings.Index(f, cand) + if idx < 0 { + continue + } + + key := strings.TrimPrefix(strings.TrimSpace(f[:idx]), labelsFilterPrefix) + want := strings.Trim(strings.TrimSpace(f[idx+len(cand):]), `"'`) + op := strings.TrimSpace(cand) + negate := op == filterOpNotEqual || op == "ne" + + got, has := addressLabels(body)[key] + + return (has && got == want) != negate + } + + return true +} diff --git a/server/gcp/vpc/address_labels_sdk_test.go b/server/gcp/vpc/address_labels_sdk_test.go new file mode 100644 index 000000000..66f9ce73b --- /dev/null +++ b/server/gcp/vpc/address_labels_sdk_test.go @@ -0,0 +1,213 @@ +package vpc_test + +import ( + "context" + "errors" + "net/http" + "testing" + + computev1 "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" +) + +const labelsRegion = "us-central1" + +func newComputeV1(t *testing.T) *computev1.Service { + t.Helper() + + ts := newGCPNetServer(t) + + svc, err := computev1.NewService(context.Background(), + option.WithEndpoint(ts.URL+"/compute/v1/"), + option.WithoutAuthentication(), + option.WithHTTPClient(ts.Client()), + ) + if err != nil { + t.Fatalf("compute.NewService: %v", err) + } + + return svc +} + +// wantStatus asserts err is a googleapi error with the given HTTP status. +func wantStatus(t *testing.T, what string, err error, code int) { + t.Helper() + + var gerr *googleapi.Error + if !errors.As(err, &gerr) || gerr.Code != code { + t.Fatalf("%s: err=%v, want HTTP %d", what, err, code) + } +} + +// TestSDKRegionalAddressSetLabels drives AddressesService.SetLabels through the +// real compute/v1 client: labels replace the set under the current +// labelFingerprint, a stale or missing fingerprint is 412, the returned +// operation resolves DONE through regionOperations, Get shows the new labels +// under a new fingerprint, and a labels.= list filter narrows the list. +func TestSDKRegionalAddressSetLabels(t *testing.T) { + ctx := context.Background() + svc := newComputeV1(t) + + for _, name := range []string{"addr-a", "addr-b"} { + if _, err := svc.Addresses.Insert(testProject, labelsRegion, &computev1.Address{ + Name: name, Labels: map[string]string{"team": "net"}, + }).Context(ctx).Do(); err != nil { + t.Fatalf("Insert %s: %v", name, err) + } + } + + before, err := svc.Addresses.Get(testProject, labelsRegion, "addr-a").Context(ctx).Do() + if err != nil { + t.Fatalf("Get: %v", err) + } + + if before.LabelFingerprint == "" { + t.Fatal("labelFingerprint empty on a freshly inserted address") + } + + _, err = svc.Addresses.SetLabels(testProject, labelsRegion, "addr-a", &computev1.RegionSetLabelsRequest{ + Labels: map[string]string{"env": "prod"}, LabelFingerprint: "c3RhbGU=", + }).Context(ctx).Do() + wantStatus(t, "SetLabels(stale fingerprint)", err, http.StatusPreconditionFailed) + + _, err = svc.Addresses.SetLabels(testProject, labelsRegion, "addr-a", &computev1.RegionSetLabelsRequest{ + Labels: map[string]string{"env": "prod"}, + }).Context(ctx).Do() + wantStatus(t, "SetLabels(no fingerprint)", err, http.StatusPreconditionFailed) + + op, err := svc.Addresses.SetLabels(testProject, labelsRegion, "addr-a", &computev1.RegionSetLabelsRequest{ + Labels: map[string]string{"env": "prod", "tier": "edge"}, LabelFingerprint: before.LabelFingerprint, + }).Context(ctx).Do() + if err != nil { + t.Fatalf("SetLabels: %v", err) + } + + if op.Status != "DONE" || op.OperationType != "setLabels" { + t.Fatalf("op status=%q type=%q, want DONE/setLabels", op.Status, op.OperationType) + } + + polled, err := svc.RegionOperations.Get(testProject, labelsRegion, op.Name).Context(ctx).Do() + if err != nil || polled.Status != "DONE" { + t.Fatalf("RegionOperations.Get: op=%+v err=%v", polled, err) + } + + after, err := svc.Addresses.Get(testProject, labelsRegion, "addr-a").Context(ctx).Do() + if err != nil { + t.Fatalf("Get after: %v", err) + } + + if len(after.Labels) != 2 || after.Labels["env"] != "prod" || after.Labels["tier"] != "edge" { + t.Fatalf("labels=%v, want exactly env=prod tier=edge (team replaced away)", after.Labels) + } + + if after.LabelFingerprint == "" || after.LabelFingerprint == before.LabelFingerprint { + t.Fatalf("labelFingerprint %q did not change from %q", after.LabelFingerprint, before.LabelFingerprint) + } + + // The old fingerprint is now stale. + _, err = svc.Addresses.SetLabels(testProject, labelsRegion, "addr-a", &computev1.RegionSetLabelsRequest{ + Labels: map[string]string{}, LabelFingerprint: before.LabelFingerprint, + }).Context(ctx).Do() + wantStatus(t, "SetLabels(superseded fingerprint)", err, http.StatusPreconditionFailed) + + _, err = svc.Addresses.SetLabels(testProject, labelsRegion, "missing", &computev1.RegionSetLabelsRequest{ + LabelFingerprint: before.LabelFingerprint, + }).Context(ctx).Do() + wantStatus(t, "SetLabels(missing address)", err, http.StatusNotFound) + + assertFilter(ctx, t, svc, "labels.env=prod", []string{"addr-a"}) + assertFilter(ctx, t, svc, "labels.team=net", []string{"addr-b"}) + assertFilter(ctx, t, svc, "labels.env!=prod", []string{"addr-b"}) + assertFilter(ctx, t, svc, "name=addr-b", []string{"addr-b"}) +} + +func assertFilter(ctx context.Context, t *testing.T, svc *computev1.Service, filter string, want []string) { + t.Helper() + + list, err := svc.Addresses.List(testProject, labelsRegion).Filter(filter).Context(ctx).Do() + if err != nil { + t.Fatalf("List(%q): %v", filter, err) + } + + got := make([]string, 0, len(list.Items)) + for _, a := range list.Items { + got = append(got, a.Name) + } + + if len(got) != len(want) { + t.Fatalf("List(%q) = %v, want %v", filter, got, want) + } + + for i := range want { + if got[i] != want[i] { + t.Fatalf("List(%q) = %v, want %v", filter, got, want) + } + } +} + +// TestSDKGlobalAddressSetLabels covers GlobalAddressesService.SetLabels: the +// same replace + fingerprint semantics on a global address, with the operation +// resolving through globalOperations and an empty label set clearing labels. +func TestSDKGlobalAddressSetLabels(t *testing.T) { + ctx := context.Background() + svc := newComputeV1(t) + + if _, err := svc.GlobalAddresses.Insert(testProject, &computev1.Address{ + Name: "g-addr", Purpose: "VPC_PEERING", AddressType: "INTERNAL", PrefixLength: 16, + }).Context(ctx).Do(); err != nil { + t.Fatalf("Insert: %v", err) + } + + before, err := svc.GlobalAddresses.Get(testProject, "g-addr").Context(ctx).Do() + if err != nil { + t.Fatalf("Get: %v", err) + } + + _, err = svc.GlobalAddresses.SetLabels(testProject, "g-addr", &computev1.GlobalSetLabelsRequest{ + Labels: map[string]string{"env": "dev"}, + }).Context(ctx).Do() + wantStatus(t, "SetLabels(no fingerprint)", err, http.StatusPreconditionFailed) + + op, err := svc.GlobalAddresses.SetLabels(testProject, "g-addr", &computev1.GlobalSetLabelsRequest{ + Labels: map[string]string{"env": "dev"}, LabelFingerprint: before.LabelFingerprint, + }).Context(ctx).Do() + if err != nil { + t.Fatalf("SetLabels: %v", err) + } + + polled, err := svc.GlobalOperations.Get(testProject, op.Name).Context(ctx).Do() + if err != nil || polled.Status != "DONE" { + t.Fatalf("GlobalOperations.Get: op=%+v err=%v", polled, err) + } + + mid, err := svc.GlobalAddresses.Get(testProject, "g-addr").Context(ctx).Do() + if err != nil { + t.Fatalf("Get mid: %v", err) + } + + if mid.Labels["env"] != "dev" || mid.LabelFingerprint == before.LabelFingerprint { + t.Fatalf("labels=%v fp=%q (before %q), want env=dev under a new fingerprint", + mid.Labels, mid.LabelFingerprint, before.LabelFingerprint) + } + + if mid.Purpose != "VPC_PEERING" || mid.PrefixLength != 16 { + t.Fatalf("setLabels clobbered other fields: purpose=%q prefixLength=%d", mid.Purpose, mid.PrefixLength) + } + + if _, err := svc.GlobalAddresses.SetLabels(testProject, "g-addr", &computev1.GlobalSetLabelsRequest{ + LabelFingerprint: mid.LabelFingerprint, + }).Context(ctx).Do(); err != nil { + t.Fatalf("SetLabels(clear): %v", err) + } + + cleared, err := svc.GlobalAddresses.Get(testProject, "g-addr").Context(ctx).Do() + if err != nil { + t.Fatalf("Get cleared: %v", err) + } + + if len(cleared.Labels) != 0 || cleared.LabelFingerprint != before.LabelFingerprint { + t.Fatalf("labels=%v fp=%q, want no labels and the empty-set fingerprint %q", + cleared.Labels, cleared.LabelFingerprint, before.LabelFingerprint) + } +} diff --git a/server/gcp/vpc/addresses.go b/server/gcp/vpc/addresses.go index dffbc0f5d..916a90030 100644 --- a/server/gcp/vpc/addresses.go +++ b/server/gcp/vpc/addresses.go @@ -166,6 +166,16 @@ func (h *Handler) routeAddresses(w http.ResponseWriter, r *http.Request, rp gcpr return } + if rp.Action != "" { + if rp.Action == setLabelsAction && r.Method == http.MethodPost { + h.setAddressLabels(w, r, rp) + } else { + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } + + return + } + switch r.Method { case http.MethodGet: h.getAddress(w, r, rp) @@ -223,6 +233,7 @@ func (h *Handler) enrichAddress(raw json.RawMessage, rp gcprest.ResourcePath, ho body["status"] = "RESERVED" body["selfLink"] = gcprest.SelfLink(host, rp.Project, rp.Scope, rp.ScopeName, resourceAddresses, name) body["creationTimestamp"] = nowRFC3339() + body["labelFingerprint"] = addressLabelFingerprint(addressLabels(raw)) if addr, ok := body["address"].(string); !ok || addr == "" { body["address"] = h.addresses.allocIP() @@ -264,7 +275,7 @@ func (h *Handler) listAddresses(w http.ResponseWriter, r *http.Request, rp gcpre items := make([]json.RawMessage, 0, len(all)) for _, body := range all { - if nameMatches(filter, rawName(body)) { + if addressMatches(filter, body) { items = append(items, reflectAddressUsage(body, usersByIP)) } } @@ -307,7 +318,7 @@ func (h *Handler) aggregatedListAddresses(w http.ResponseWriter, r *http.Request list := make([]json.RawMessage, 0, len(bodies)) for _, b := range bodies { - if nameMatches(filter, rawName(b)) { + if addressMatches(filter, b) { list = append(list, reflectAddressUsage(b, usersByIP)) } } From efae655bfc23847342c029278e6ddad2a08af13f Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 00:16:38 +0530 Subject: [PATCH 06/24] fix(ecs): Describe* include=TAGS reflects TagResource/UntagResource DescribeServices, DescribeClusters, DescribeTaskDefinition and DescribeTasks serialised the entity's own Tags field, a create-time snapshot, while TagResource/UntagResource write only the ARN-keyed tag store that ListTagsForResource reads. After the first tag write the describe calls and ListTagsForResource disagreed. Every create path (CreateCluster, CreateService, RegisterTaskDefinition, RunTask/StartTask) already seeds that store via recordTags, so it is made the single authority: a new liveTags(arn, fallback) helper reads it, and describeCluster, DescribeServices, observedTask and DescribeTaskDefinition overlay the live set onto the returned copy. The entity field is used only when the ARN was never recorded. include gating is unchanged: DescribeClusters still omits tags without include=TAGS; the service, task and task-definition describes return tags unconditionally, as they did before. Test: SDK round-trip (aws-sdk-go-v2 ecs) per resource kind: create with tags, TagResource adds one, UntagResource removes one, describe with include=TAGS and ListTagsForResource both return exactly the live set. --- providers/aws/ecs/clusters.go | 1 + providers/aws/ecs/services.go | 4 +- providers/aws/ecs/tags.go | 15 ++ providers/aws/ecs/taskdefs.go | 1 + providers/aws/ecs/tasks.go | 1 + server/aws/ecs/describe_live_tags_test.go | 227 ++++++++++++++++++++++ 6 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 server/aws/ecs/describe_live_tags_test.go diff --git a/providers/aws/ecs/clusters.go b/providers/aws/ecs/clusters.go index 52ddc2ef4..83111a06e 100644 --- a/providers/aws/ecs/clusters.go +++ b/providers/aws/ecs/clusters.go @@ -93,6 +93,7 @@ func (m *Mock) DescribeClusters(ctx context.Context, ids []string) ([]driver.Clu // resource counts computed from the task, service, and instance stores. func (m *Mock) describeCluster(c *driver.Cluster) driver.Cluster { out := cloneCluster(c) + out.Tags = m.liveTags(c.ARN, c.Tags) out.ActiveServicesCount, out.RunningTasksCount, out.PendingTasksCount, out.RegisteredContainerInstancesCount = m.clusterCounts(c.Name) diff --git a/providers/aws/ecs/services.go b/providers/aws/ecs/services.go index 895f7a802..6585d30e5 100644 --- a/providers/aws/ecs/services.go +++ b/providers/aws/ecs/services.go @@ -762,7 +762,9 @@ func (m *Mock) DescribeServices(ctx context.Context, cluster string, ids []strin for _, id := range ids { if s, ok := m.resolveService(want, id); ok { - found = append(found, cloneService(s)) + out := cloneService(s) + out.Tags = m.liveTags(s.ARN, s.Tags) + found = append(found, out) continue } diff --git a/providers/aws/ecs/tags.go b/providers/aws/ecs/tags.go index ad5a0890b..65a982c68 100644 --- a/providers/aws/ecs/tags.go +++ b/providers/aws/ecs/tags.go @@ -75,6 +75,21 @@ func (m *Mock) ListTagsForResource(_ context.Context, resourceARN string) ([]dri return nil, apiErrf(errors.NotFound, excClient, "resource %q not found", resourceARN) } +// liveTags returns a resource's current tags. The tag store keyed by ARN is +// the single authority: every create path seeds it via recordTags and +// TagResource/UntagResource mutate only it, so the entity's own Tags field is a +// create-time snapshot that goes stale after the first tag write. Describe +// paths read through here so they agree with ListTagsForResource. fallback is +// used only when the ARN was never recorded (e.g. a snapshot taken before the +// resource's tags were tracked). +func (m *Mock) liveTags(arn string, fallback []driver.Tag) []driver.Tag { + if tags, ok := m.tags.Get(arn); ok { + return copyTags(tags) + } + + return copyTags(fallback) +} + // mergeTags upserts add into base: existing keys are overwritten in place and // new keys are appended, preserving order for determinism. func mergeTags(base, add []driver.Tag) []driver.Tag { diff --git a/providers/aws/ecs/taskdefs.go b/providers/aws/ecs/taskdefs.go index e53ee1f5e..3bc92d733 100644 --- a/providers/aws/ecs/taskdefs.go +++ b/providers/aws/ecs/taskdefs.go @@ -241,6 +241,7 @@ func (m *Mock) DescribeTaskDefinition(_ context.Context, id string) (*driver.Tas } out := cloneTaskDef(td) + out.Tags = m.liveTags(td.ARN, td.Tags) return &out, nil } diff --git a/providers/aws/ecs/tasks.go b/providers/aws/ecs/tasks.go index 2c3dfe817..ff4abbea8 100644 --- a/providers/aws/ecs/tasks.go +++ b/providers/aws/ecs/tasks.go @@ -382,6 +382,7 @@ func (m *Mock) overlayStatus(t *driver.Task) { // aws-sdk-go-v2 TasksRunning/TasksStopped waiters. func (m *Mock) observedTask(t *driver.Task) driver.Task { out := cloneTask(t) + out.Tags = m.liveTags(t.ARN, t.Tags) m.overlayStatus(&out) return out diff --git a/server/aws/ecs/describe_live_tags_test.go b/server/aws/ecs/describe_live_tags_test.go new file mode 100644 index 000000000..e64c59cd7 --- /dev/null +++ b/server/aws/ecs/describe_live_tags_test.go @@ -0,0 +1,227 @@ +package ecs_test + +import ( + "context" + "maps" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" + ecstypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" +) + +// liveTagsCase creates one ECS resource with create-time tags and returns its +// ARN, plus a describe func that reads its tags back with include=[TAGS]. +type liveTagsCase struct { + name string + create func(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string + describe func(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag +} + +// TestSDKDescribeIncludeTagsReflectsTagWrites guards that every Describe* call +// with include=[TAGS] returns the live tag set: create-time tags, plus what +// TagResource added, minus what UntagResource removed. The describe paths used +// to serialise the entity's create-time Tags field while TagResource and +// UntagResource wrote only the separate ARN-keyed tag store, so describe and +// ListTagsForResource disagreed after the first tag write. +func TestSDKDescribeIncludeTagsReflectsTagWrites(t *testing.T) { + for _, tc := range liveTagsCases() { + t.Run(tc.name, func(t *testing.T) { + client, cloud := newECSServer(t) + ctx := context.Background() + + if _, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + registerNginx(t, client, ctx) + cloud.ECS.SeedContainerInstance("prod", "i-0livetags") + + arn := tc.create(t, client, ctx, []ecstypes.Tag{ + {Key: aws.String("env"), Value: aws.String("prod")}, + {Key: aws.String("owner"), Value: aws.String("alice")}, + }) + + if _, err := client.TagResource(ctx, &awsecs.TagResourceInput{ + ResourceArn: aws.String(arn), + Tags: []ecstypes.Tag{{Key: aws.String("team"), Value: aws.String("platform")}}, + }); err != nil { + t.Fatalf("TagResource: %v", err) + } + + if _, err := client.UntagResource(ctx, &awsecs.UntagResourceInput{ + ResourceArn: aws.String(arn), + TagKeys: []string{"owner"}, + }); err != nil { + t.Fatalf("UntagResource: %v", err) + } + + want := map[string]string{"env": "prod", "team": "platform"} + + if got := tagMap(tc.describe(t, client, ctx, arn)); !maps.Equal(got, want) { + t.Fatalf("describe include=TAGS = %v, want %v", got, want) + } + + list, err := client.ListTagsForResource(ctx, &awsecs.ListTagsForResourceInput{ResourceArn: aws.String(arn)}) + if err != nil { + t.Fatalf("ListTagsForResource: %v", err) + } + + if got := tagMap(list.Tags); !maps.Equal(got, want) { + t.Fatalf("ListTagsForResource = %v, want %v", got, want) + } + }) + } +} + +func liveTagsCases() []liveTagsCase { + return []liveTagsCase{ + {name: "service", create: createTaggedService, describe: describeServiceTags}, + {name: "cluster", create: createTaggedCluster, describe: describeClusterTags}, + {name: "task-definition", create: createTaggedTaskDef, describe: describeTaskDefTags}, + {name: "task", create: createTaggedTask, describe: describeTaskTags}, + } +} + +func createTaggedService(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), + ServiceName: aws.String("web-svc"), + TaskDefinition: aws.String("web"), + DesiredCount: aws.Int32(1), + Tags: tags, + }) + if err != nil { + t.Fatalf("CreateService: %v", err) + } + + return aws.ToString(out.Service.ServiceArn) +} + +func describeServiceTags(t *testing.T, client *awsecs.Client, ctx context.Context, _ string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{ + Cluster: aws.String("prod"), + Services: []string{"web-svc"}, + Include: []ecstypes.ServiceField{ecstypes.ServiceFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeServices: %v", err) + } + + if len(out.Services) != 1 { + t.Fatalf("DescribeServices = %d services, want 1", len(out.Services)) + } + + return out.Services[0].Tags +} + +func createTaggedCluster(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("tagged"), Tags: tags}) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + return aws.ToString(out.Cluster.ClusterArn) +} + +func describeClusterTags(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeClusters(ctx, &awsecs.DescribeClustersInput{ + Clusters: []string{arn}, + Include: []ecstypes.ClusterField{ecstypes.ClusterFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeClusters: %v", err) + } + + if len(out.Clusters) != 1 { + t.Fatalf("DescribeClusters = %d clusters, want 1", len(out.Clusters)) + } + + return out.Clusters[0].Tags +} + +func createTaggedTaskDef(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.RegisterTaskDefinition(ctx, &awsecs.RegisterTaskDefinitionInput{ + Family: aws.String("tagged"), + ContainerDefinitions: []ecstypes.ContainerDefinition{{ + Name: aws.String("app"), Image: aws.String("nginx:latest"), Memory: aws.Int32(512), + }}, + Tags: tags, + }) + if err != nil { + t.Fatalf("RegisterTaskDefinition: %v", err) + } + + return aws.ToString(out.TaskDefinition.TaskDefinitionArn) +} + +func describeTaskDefTags(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeTaskDefinition(ctx, &awsecs.DescribeTaskDefinitionInput{ + TaskDefinition: aws.String(arn), + Include: []ecstypes.TaskDefinitionField{ecstypes.TaskDefinitionFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeTaskDefinition: %v", err) + } + + return out.Tags +} + +func createTaggedTask(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.RunTask(ctx, &awsecs.RunTaskInput{ + Cluster: aws.String("prod"), + TaskDefinition: aws.String("web"), + Tags: tags, + }) + if err != nil { + t.Fatalf("RunTask: %v", err) + } + + if len(out.Tasks) != 1 { + t.Fatalf("RunTask = %d tasks, want 1", len(out.Tasks)) + } + + return aws.ToString(out.Tasks[0].TaskArn) +} + +func describeTaskTags(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeTasks(ctx, &awsecs.DescribeTasksInput{ + Cluster: aws.String("prod"), + Tasks: []string{arn}, + Include: []ecstypes.TaskField{ecstypes.TaskFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeTasks: %v", err) + } + + if len(out.Tasks) != 1 { + t.Fatalf("DescribeTasks = %d tasks, want 1", len(out.Tasks)) + } + + return out.Tasks[0].Tags +} + +func tagMap(tags []ecstypes.Tag) map[string]string { + out := make(map[string]string, len(tags)) + for _, tg := range tags { + out[aws.ToString(tg.Key)] = aws.ToString(tg.Value) + } + + return out +} From 1d2fcd6d37de0986914bffc3a2acdd944124b6d7 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 01:30:34 +0530 Subject: [PATCH 07/24] fix(ec2): VPC endpoint groupSet items carry groupId DescribeVpcEndpoints and CreateVpcEndpoint rendered groupSet as a list of bare strings. Real EC2 types VpcEndpoint.Groups as SecurityGroupIdentifier, so every carries (and when the group exists); the AWS SDK decoded the bare form as an empty GroupId. Name resolution now looks ids up one at a time when the batch lookup fails, so one dangling id no longer strips the names of the others. --- server/aws/ec2/endpoint.go | 41 +++++++++------ server/aws/ec2/operations.go | 15 ++++-- server/aws/ec2/vpc_endpoint_test.go | 81 +++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 18 deletions(-) diff --git a/server/aws/ec2/endpoint.go b/server/aws/ec2/endpoint.go index e9f9416ca..242cee3b5 100644 --- a/server/aws/ec2/endpoint.go +++ b/server/aws/ec2/endpoint.go @@ -1,6 +1,7 @@ package ec2 import ( + "context" "encoding/xml" "net/http" @@ -14,17 +15,17 @@ import ( const defaultVPCEndpointType = "Gateway" type vpcEndpointXML struct { - VpcEndpointID string `xml:"vpcEndpointId"` - VpcEndpointType string `xml:"vpcEndpointType"` - VpcID string `xml:"vpcId"` - ServiceName string `xml:"serviceName"` - State string `xml:"state"` - RouteTableIDs []string `xml:"routeTableIdSet>item,omitempty"` - SubnetIDs []string `xml:"subnetIdSet>item,omitempty"` - Groups []string `xml:"groupSet>item,omitempty"` - NetworkInterfaceIDs []string `xml:"networkInterfaceIdSet>item,omitempty"` - CreationTime string `xml:"creationTimestamp,omitempty"` - Tags []tagItem `xml:"tagSet>item,omitempty"` + VpcEndpointID string `xml:"vpcEndpointId"` + VpcEndpointType string `xml:"vpcEndpointType"` + VpcID string `xml:"vpcId"` + ServiceName string `xml:"serviceName"` + State string `xml:"state"` + RouteTableIDs []string `xml:"routeTableIdSet>item,omitempty"` + SubnetIDs []string `xml:"subnetIdSet>item,omitempty"` + Groups []groupItem `xml:"groupSet>item,omitempty"` + NetworkInterfaceIDs []string `xml:"networkInterfaceIdSet>item,omitempty"` + CreationTime string `xml:"creationTimestamp,omitempty"` + Tags []tagItem `xml:"tagSet>item,omitempty"` } func (h *Handler) routeVPCEndpoints(w http.ResponseWriter, r *http.Request, action string) bool { @@ -69,7 +70,7 @@ func (h *Handler) createVPCEndpoint(w http.ResponseWriter, r *http.Request) { Xmlns string `xml:"xmlns,attr"` Req string `xml:"requestId"` Endpoint vpcEndpointXML `xml:"vpcEndpoint"` - }{Xmlns: awsquery.Namespace, Req: awsquery.RequestID, Endpoint: toVPCEndpointXML(ep)}) + }{Xmlns: awsquery.Namespace, Req: awsquery.RequestID, Endpoint: h.toVPCEndpointXML(r.Context(), ep)}) } // deleteVPCEndpoints is idempotent: like real EC2 it always returns HTTP 200 @@ -108,7 +109,7 @@ func (h *Handler) describeVPCEndpoints(w http.ResponseWriter, r *http.Request) { for i := range items { if vpcEndpointMatchesFilters(&items[i], filters) { - out = append(out, toVPCEndpointXML(&items[i])) + out = append(out, h.toVPCEndpointXML(r.Context(), &items[i])) } } @@ -214,7 +215,17 @@ func vpcEndpointMatchesFilter(ep *netdriver.VPCEndpoint, f awsquery.Filter) bool } } -func toVPCEndpointXML(ep *netdriver.VPCEndpoint) vpcEndpointXML { +// toVPCEndpointXML renders ep. Real EC2 types VpcEndpoint.Groups as +// SecurityGroupIdentifier, so each carries and, +// when the group exists, its resolved . +func (h *Handler) toVPCEndpointXML(ctx context.Context, ep *netdriver.VPCEndpoint) vpcEndpointXML { + names := h.securityGroupNames(ctx, ep.SecurityGroupIDs) + + groups := make([]groupItem, 0, len(ep.SecurityGroupIDs)) + for _, sg := range ep.SecurityGroupIDs { + groups = append(groups, groupItem{GroupID: sg, GroupName: names[sg]}) + } + return vpcEndpointXML{ VpcEndpointID: ep.ID, VpcEndpointType: nonEmpty(ep.EndpointType, defaultVPCEndpointType), @@ -223,7 +234,7 @@ func toVPCEndpointXML(ep *netdriver.VPCEndpoint) vpcEndpointXML { State: nonEmpty(ep.State, stateAvailable), RouteTableIDs: ep.RouteTableIDs, SubnetIDs: ep.SubnetIDs, - Groups: ep.SecurityGroupIDs, + Groups: groups, NetworkInterfaceIDs: ep.NetworkInterfaceIDs, CreationTime: ep.CreatedAt, Tags: toTagItems(ep.Tags), diff --git a/server/aws/ec2/operations.go b/server/aws/ec2/operations.go index 363af38ef..490b2920e 100644 --- a/server/aws/ec2/operations.go +++ b/server/aws/ec2/operations.go @@ -1140,8 +1140,9 @@ func collectSecurityGroups(instances []computedriver.Instance) []string { } // securityGroupNames resolves security-group ids to their names via the -// networking driver. It returns an empty map when no networking driver is wired -// or the lookup fails, so name resolution is best-effort (ids still render). +// networking driver. Unknown ids (and every id, when no networking driver is +// wired) are absent from the map, so name resolution is best-effort and the +// ids still render. func (h *Handler) securityGroupNames(ctx context.Context, ids []string) map[string]string { names := make(map[string]string) if h.vpc == nil || len(ids) == 0 { @@ -1150,7 +1151,15 @@ func (h *Handler) securityGroupNames(ctx context.Context, ids []string) map[stri groups, err := h.vpc.DescribeSecurityGroups(ctx, ids) if err != nil { - return names + // The batch lookup fails as a whole when any id is unknown; resolve + // one at a time so a dangling id does not strip the others' names. + groups = nil + + for _, id := range ids { + if one, oneErr := h.vpc.DescribeSecurityGroups(ctx, []string{id}); oneErr == nil { + groups = append(groups, one...) + } + } } for i := range groups { diff --git a/server/aws/ec2/vpc_endpoint_test.go b/server/aws/ec2/vpc_endpoint_test.go index d8ac6beec..e4759ba1e 100644 --- a/server/aws/ec2/vpc_endpoint_test.go +++ b/server/aws/ec2/vpc_endpoint_test.go @@ -175,3 +175,84 @@ func TestDescribeVpcEndpointsUnknownIDNotFound(t *testing.T) { t.Fatalf("error = %v, want InvalidVpcEndpointId.NotFound", err) } } + +// assertEndpointGroups fails unless groups is exactly one SecurityGroupIdentifier +// carrying wantID and wantName. +func assertEndpointGroups(t *testing.T, label string, groups []ec2types.SecurityGroupIdentifier, wantID, wantName string) { + t.Helper() + + if len(groups) != 1 { + t.Fatalf("%s: Groups = %+v, want exactly [%s]", label, groups, wantID) + } + + if got := aws.ToString(groups[0].GroupId); got != wantID { + t.Errorf("%s: Groups[0].GroupId = %q, want %q", label, got, wantID) + } + + if got := aws.ToString(groups[0].GroupName); got != wantName { + t.Errorf("%s: Groups[0].GroupName = %q, want %q", label, got, wantName) + } +} + +// TestVPCEndpointGroupsAreSecurityGroupIdentifiers pins that VpcEndpoint.Groups +// decodes as SecurityGroupIdentifier (groupId + groupName) through the real SDK, +// and that ModifyVpcEndpoint Add/RemoveSecurityGroupId edits the stored set. +func TestVPCEndpointGroupsAreSecurityGroupIdentifiers(t *testing.T) { + ctx := context.Background() + client := newEC2(t) + + vpc, err := client.CreateVpc(ctx, &ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}) + if err != nil { + t.Fatalf("CreateVpc: %v", err) + } + vpcID := aws.ToString(vpc.Vpc.VpcId) + + subnet, err := client.CreateSubnet(ctx, &ec2.CreateSubnetInput{ + VpcId: aws.String(vpcID), CidrBlock: aws.String("10.0.1.0/24"), + }) + if err != nil { + t.Fatalf("CreateSubnet: %v", err) + } + + sgIDs := make([]string, 0, 2) + for _, name := range []string{"ep-sg-one", "ep-sg-two"} { + sg, sgErr := client.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(name), Description: aws.String(name), VpcId: aws.String(vpcID), + }) + if sgErr != nil { + t.Fatalf("CreateSecurityGroup(%s): %v", name, sgErr) + } + sgIDs = append(sgIDs, aws.ToString(sg.GroupId)) + } + + create, err := client.CreateVpcEndpoint(ctx, &ec2.CreateVpcEndpointInput{ + VpcId: aws.String(vpcID), + ServiceName: aws.String("com.amazonaws.us-east-1.ssm"), + VpcEndpointType: ec2types.VpcEndpointTypeInterface, + SubnetIds: []string{aws.ToString(subnet.Subnet.SubnetId)}, + SecurityGroupIds: []string{sgIDs[0]}, + }) + if err != nil { + t.Fatalf("CreateVpcEndpoint: %v", err) + } + assertEndpointGroups(t, "CreateVpcEndpoint", create.VpcEndpoint.Groups, sgIDs[0], "ep-sg-one") + + epID := aws.ToString(create.VpcEndpoint.VpcEndpointId) + + if _, err := client.ModifyVpcEndpoint(ctx, &ec2.ModifyVpcEndpointInput{ + VpcEndpointId: aws.String(epID), + AddSecurityGroupIds: []string{sgIDs[1]}, + RemoveSecurityGroupIds: []string{sgIDs[0]}, + }); err != nil { + t.Fatalf("ModifyVpcEndpoint: %v", err) + } + + desc, err := client.DescribeVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{VpcEndpointIds: []string{epID}}) + if err != nil { + t.Fatalf("DescribeVpcEndpoints: %v", err) + } + if len(desc.VpcEndpoints) != 1 { + t.Fatalf("DescribeVpcEndpoints = %d endpoints, want 1", len(desc.VpcEndpoints)) + } + assertEndpointGroups(t, "DescribeVpcEndpoints after modify", desc.VpcEndpoints[0].Groups, sgIDs[1], "ep-sg-two") +} From b0ebd3d38efbda545522da1416eb8db223bd905c Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 01:16:59 +0530 Subject: [PATCH 08/24] fix(gcp-lb): Private Service Connect consumer forwarding rules A PSC consumer rule targets a producer's serviceAttachment (regional) or a Google APIs bundle, all-apis or vpc-sc (global only). The emulator dropped the rule's network and subnetwork and synthesized an EXTERNAL scheme for a rule that has none. - network and subnetwork are stored and returned - a PSC rule sent without a scheme reports none - pscConnectionStatus (ACCEPTED) and a stable pscConnectionId are returned - a bundle target on a regional rule is refused, as GCP does --- .../gcp/loadbalancer/forwardingrules_psc.go | 70 ++++++ .../forwardingrules_psc_sdk_test.go | 215 ++++++++++++++++++ server/gcp/loadbalancer/operations.go | 21 +- server/gcp/loadbalancer/refvalidation.go | 7 + server/gcp/loadbalancer/types.go | 6 + 5 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 server/gcp/loadbalancer/forwardingrules_psc.go create mode 100644 server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go diff --git a/server/gcp/loadbalancer/forwardingrules_psc.go b/server/gcp/loadbalancer/forwardingrules_psc.go new file mode 100644 index 000000000..576414b55 --- /dev/null +++ b/server/gcp/loadbalancer/forwardingrules_psc.go @@ -0,0 +1,70 @@ +package loadbalancer + +import ( + "strconv" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + lbdriver "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +// Private Service Connect (PSC) consumer forwarding rules. +// +// A PSC consumer rule targets either a producer's service attachment (a +// regional rule whose target is a .../serviceAttachments/{name} self-link) or +// a Google APIs bundle (a global rule whose target is "all-apis" or "vpc-sc"). +// Such a rule carries no loadBalancingScheme, so the EXTERNAL default applied +// to ordinary rules must not be synthesized for it, and GCP reports the +// connection it established through pscConnectionStatus / pscConnectionId. +const ( + pscTargetAllAPIs = "all-apis" + pscTargetVPCSC = "vpc-sc" + + pscServiceAttachmentsSegment = "/serviceAttachments/" + + // pscStatusAccepted is the connection status of a PSC rule the emulator + // created: there is no producer-side acceptance list to reject it. + pscStatusAccepted = "ACCEPTED" +) + +// isGoogleAPIsBundle reports whether target names a PSC Google APIs bundle. +func isGoogleAPIsBundle(target string) bool { + return target == pscTargetAllAPIs || target == pscTargetVPCSC +} + +// isPSCTarget reports whether a forwarding rule with this target is a PSC +// consumer rule. +func isPSCTarget(target string) bool { + return isGoogleAPIsBundle(target) || strings.Contains(target, pscServiceAttachmentsSegment) +} + +// validatePSCTarget rejects a Google APIs bundle target on a regional rule: +// all-apis / vpc-sc are only valid on global forwarding rules. +// +//nolint:gocritic // rp is a request-scoped value +func validatePSCTarget(rp gcprest.ResourcePath, target string) error { + if isGoogleAPIsBundle(target) && rp.Scope != gcprest.ScopeGlobal { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.target': '%s'. A Google APIs bundle target is only valid on a global forwarding rule.", + target) + } + + return nil +} + +// applyPSCFields sets pscConnectionStatus and a stable pscConnectionId on a +// PSC consumer rule's response; other rules are left untouched. +func applyPSCFields(out *forwardingRuleResponse, lb *lbdriver.LBInfo) { + if !isPSCTarget(lb.Tags[frTargetTag]) { + return + } + + id := fnvHash("psc:" + lb.ID) + if id == 0 { + id = 1 + } + + out.PscConnectionStatus = pscStatusAccepted + out.PscConnectionID = strconv.FormatUint(id, 10) +} diff --git a/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go b/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go new file mode 100644 index 000000000..737ff08e7 --- /dev/null +++ b/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go @@ -0,0 +1,215 @@ +package loadbalancer_test + +import ( + "context" + "errors" + "net/http" + "testing" + + gcpcompute "cloud.google.com/go/compute/apiv1" + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" +) + +const ( + pscNetwork = "projects/" + testProject + "/global/networks/consumer-vpc" + pscSubnetwork = "projects/" + testProject + "/regions/" + testRegion + "/subnetworks/consumer-subnet" + pscAttachment = "projects/producer-proj/regions/" + testRegion + "/serviceAttachments/producer-sa" + pscAddress = "projects/" + testProject + "/regions/" + testRegion + "/addresses/psc-endpoint-ip" + pscAccepted = "ACCEPTED" +) + +func newRegionalForwardingRulesClient(t *testing.T, url string, httpc option.ClientOption) *gcpcompute.ForwardingRulesClient { + t.Helper() + + client, err := gcpcompute.NewForwardingRulesRESTClient(context.Background(), + option.WithEndpoint(url), option.WithoutAuthentication(), httpc) + if err != nil { + t.Fatalf("NewForwardingRulesRESTClient: %v", err) + } + + t.Cleanup(func() { _ = client.Close() }) + + return client +} + +// assertPSCRule checks the fields a PSC consumer rule must carry on Get. +func assertPSCRule(t *testing.T, got *computepb.ForwardingRule, wantIP, wantTarget string) { + t.Helper() + + if got.GetNetwork() != pscNetwork { + t.Errorf("network = %q, want %q (dropped on insert)", got.GetNetwork(), pscNetwork) + } + + if got.GetIPAddress() != wantIP { + t.Errorf("IPAddress = %q, want %q", got.GetIPAddress(), wantIP) + } + + if got.GetTarget() != wantTarget { + t.Errorf("target = %q, want %q", got.GetTarget(), wantTarget) + } + + if got.LoadBalancingScheme != nil { + t.Errorf("loadBalancingScheme = %q, want unset for a PSC rule", got.GetLoadBalancingScheme()) + } + + if got.GetPscConnectionStatus() != pscAccepted { + t.Errorf("pscConnectionStatus = %q, want %s", got.GetPscConnectionStatus(), pscAccepted) + } + + if got.GetPscConnectionId() == 0 { + t.Error("pscConnectionId = 0, want a non-zero connection id") + } +} + +// TestSDKGCPForwardingRulePSCServiceAttachment drives a regional Private +// Service Connect consumer rule (target = a producer's serviceAttachments +// self-link) through the real ForwardingRulesRESTClient. The attachment lives +// in another project the emulator does not model, so it must not be rejected. +func TestSDKGCPForwardingRulePSCServiceAttachment(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + op, err := client.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, + Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("psc-endpoint"), + Network: ptrStr(pscNetwork), + Subnetwork: ptrStr(pscSubnetwork), + IPAddress: ptrStr(pscAddress), + Target: ptrStr(pscAttachment), + }, + }) + if err != nil { + t.Fatalf("Insert: %v", err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("Insert wait: %v", err) + } + + get := &computepb.GetForwardingRuleRequest{Project: testProject, Region: testRegion, ForwardingRule: "psc-endpoint"} + + got, err := client.Get(ctx, get) + if err != nil { + t.Fatalf("Get: %v", err) + } + + assertPSCRule(t, got, pscAddress, pscAttachment) + + if got.GetSubnetwork() != pscSubnetwork { + t.Errorf("subnetwork = %q, want %q", got.GetSubnetwork(), pscSubnetwork) + } + + again, err := client.Get(ctx, get) + if err != nil { + t.Fatalf("second Get: %v", err) + } + + if again.GetPscConnectionId() != got.GetPscConnectionId() { + t.Errorf("pscConnectionId changed between reads: %d then %d", got.GetPscConnectionId(), again.GetPscConnectionId()) + } +} + +// TestSDKGCPForwardingRulePSCGoogleAPIs drives a global PSC endpoint for +// Google APIs (target "all-apis") through the real +// GlobalForwardingRulesRESTClient, with a literal IP address. +func TestSDKGCPForwardingRulePSCGoogleAPIs(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + client := newForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + const endpointIP = "10.3.0.5" + + op, err := client.Insert(ctx, &computepb.InsertGlobalForwardingRuleRequest{ + Project: testProject, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("pscgoogleapis"), + Network: ptrStr(pscNetwork), + IPAddress: ptrStr(endpointIP), + Target: ptrStr("all-apis"), + }, + }) + if err != nil { + t.Fatalf("Insert: %v", err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("Insert wait: %v", err) + } + + got, err := client.Get(ctx, &computepb.GetGlobalForwardingRuleRequest{Project: testProject, ForwardingRule: "pscgoogleapis"}) + if err != nil { + t.Fatalf("Get: %v", err) + } + + assertPSCRule(t, got, endpointIP, "all-apis") +} + +// TestSDKGCPForwardingRuleGoogleAPIsBundleRegionalRejected: all-apis / vpc-sc +// are only valid targets on a global forwarding rule. +func TestSDKGCPForwardingRuleGoogleAPIsBundleRegionalRejected(t *testing.T) { + ts := newGCPLBServer(t) + client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + _, err := client.Insert(context.Background(), &computepb.InsertForwardingRuleRequest{ + Project: testProject, + Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("regional-vpcsc"), + Network: ptrStr(pscNetwork), + Target: ptrStr("vpc-sc"), + }, + }) + + var gerr *googleapi.Error + if !errors.As(err, &gerr) || gerr.Code != http.StatusBadRequest { + t.Fatalf("Insert regional vpc-sc: err = %v, want 400", err) + } +} + +// TestSDKGCPForwardingRuleExternalKeepsNetwork: a non-PSC rule sent without a +// scheme still defaults to EXTERNAL, and now keeps the network it was sent. +func TestSDKGCPForwardingRuleExternalKeepsNetwork(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + op, err := client.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, + Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("plain-fr"), + Network: ptrStr(pscNetwork), + IPProtocol: ptrStr("TCP"), + PortRange: ptrStr("80"), + }, + }) + if err != nil { + t.Fatalf("Insert: %v", err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("Insert wait: %v", err) + } + + got, err := client.Get(ctx, &computepb.GetForwardingRuleRequest{Project: testProject, Region: testRegion, ForwardingRule: "plain-fr"}) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if got.GetLoadBalancingScheme() != "EXTERNAL" { + t.Errorf("loadBalancingScheme = %q, want EXTERNAL default", got.GetLoadBalancingScheme()) + } + + if got.GetNetwork() != pscNetwork { + t.Errorf("network = %q, want %q", got.GetNetwork(), pscNetwork) + } + + if got.PscConnectionStatus != nil || got.PscConnectionId != nil { + t.Errorf("non-PSC rule reports psc fields: status=%q id=%d", got.GetPscConnectionStatus(), got.GetPscConnectionId()) + } +} diff --git a/server/gcp/loadbalancer/operations.go b/server/gcp/loadbalancer/operations.go index 6b7637df5..48e9464e2 100644 --- a/server/gcp/loadbalancer/operations.go +++ b/server/gcp/loadbalancer/operations.go @@ -680,10 +680,14 @@ func (h *Handler) toForwardingRuleResponse(ctx context.Context, lb *lbdriver.LBI Target: lb.Tags[frTargetTag], Description: lb.Tags[frDescriptionTag], LoadBalancingScheme: forwardingRuleScheme(lb), + Network: lb.Tags[frNetworkTag], + Subnetwork: lb.Tags[frSubnetworkTag], CreationTimestamp: lb.Tags[frCreationTag], SelfLink: gcprest.SelfLink(host, rp.Project, rp.Scope, rp.ScopeName, resourceForwardingRules, name), } + applyPSCFields(&out, lb) + // A linked listener (a rule referencing a backend service) supersedes the // round-tripped protocol/portRange and adds the backendService self-link. if listeners, err := h.lb.DescribeListeners(ctx, lb.ARN); err == nil && len(listeners) > 0 { @@ -955,6 +959,8 @@ const ( frDescriptionTag = "cloudemu:gcpFrDescription" frCreationTag = "cloudemu:gcpFrCreationTimestamp" frTargetTag = "cloudemu:gcpFrTarget" + frNetworkTag = "cloudemu:gcpFrNetwork" + frSubnetworkTag = "cloudemu:gcpFrSubnetwork" // frNameTag/frScopeTag carry the client-facing name and scope key so a // scope-prefixed driver record re-emits its real name at its real scope. frNameTag = "cloudemu:gcpFrName" @@ -992,6 +998,14 @@ func forwardingRuleTags(req *forwardingRuleRequest) map[string]string { tags[frDescriptionTag] = req.Description } + if req.Network != "" { + tags[frNetworkTag] = req.Network + } + + if req.Subnetwork != "" { + tags[frSubnetworkTag] = req.Subnetwork + } + return tags } @@ -1017,12 +1031,17 @@ func forwardingRuleIP(lb *lbdriver.LBInfo) string { // forwardingRuleScheme returns the exact GCP loadBalancingScheme, preferring the // round-tripped value (EXTERNAL_MANAGED / INTERNAL_MANAGED / …) over the driver -// scheme's lossy EXTERNAL/INTERNAL collapse. +// scheme's lossy EXTERNAL/INTERNAL collapse. A Private Service Connect +// consumer rule sent without a scheme has none, so no default is synthesized. func forwardingRuleScheme(lb *lbdriver.LBInfo) string { if s := lb.Tags[frSchemeTag]; s != "" { return s } + if isPSCTarget(lb.Tags[frTargetTag]) { + return "" + } + return schemeToGCP(lb.Scheme) } diff --git a/server/gcp/loadbalancer/refvalidation.go b/server/gcp/loadbalancer/refvalidation.go index b32acec3a..730de9845 100644 --- a/server/gcp/loadbalancer/refvalidation.go +++ b/server/gcp/loadbalancer/refvalidation.go @@ -254,6 +254,13 @@ func (h *Handler) validateForwardingRuleTarget(ctx context.Context, rp gcprest.R return nil } + // A PSC service-attachment target names a producer resource the emulator + // does not model, so it is accepted unvalidated (targetCollectionFor does not + // recognize it); a Google APIs bundle is only valid on a global rule. + if err := validatePSCTarget(rp, target); err != nil { + return err + } + collection := targetCollectionFor(target) if collection == "" { return nil diff --git a/server/gcp/loadbalancer/types.go b/server/gcp/loadbalancer/types.go index 62514468c..bf50859c9 100644 --- a/server/gcp/loadbalancer/types.go +++ b/server/gcp/loadbalancer/types.go @@ -107,6 +107,8 @@ type forwardingRuleRequest struct { Target string `json:"target,omitempty"` BackendService string `json:"backendService,omitempty"` LoadBalancingScheme string `json:"loadBalancingScheme,omitempty"` + Network string `json:"network,omitempty"` + Subnetwork string `json:"subnetwork,omitempty"` } type forwardingRuleResponse struct { @@ -120,6 +122,10 @@ type forwardingRuleResponse struct { Target string `json:"target,omitempty"` BackendService string `json:"backendService,omitempty"` LoadBalancingScheme string `json:"loadBalancingScheme,omitempty"` + Network string `json:"network,omitempty"` + Subnetwork string `json:"subnetwork,omitempty"` + PscConnectionStatus string `json:"pscConnectionStatus,omitempty"` + PscConnectionID string `json:"pscConnectionId,omitempty"` CreationTimestamp string `json:"creationTimestamp,omitempty"` SelfLink string `json:"selfLink"` } From 4686c13bf4e23092457a4a43d2ce28f54e8599ae Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:31:29 +0530 Subject: [PATCH 09/24] fix(gcp-backupdr): address review of backup vaults (delete LRO, id format, retention lock, list filter) H1: a delete operation came back as {"done":true} with no response, inline, from the standalone poll and from the shared poller (the handler registered nil). The cloud.google.com/go/backupdr/apiv1 REST client's DeleteBackupVaultOperation.Wait rejects that with "unsupported result type ". Delete now returns and registers {"@type":"type.googleapis.com/google.protobuf.Empty"}, as managedkafka does; the standalone poll replays the vault for create/update ops and Empty otherwise. Covered by delete-and-Wait tests through the GAPIC REST client (added cloud.google.com/go/backupdr v1.16.0 as a test dependency). M1: the vault id was only length-checked, so "Bad_ID" was accepted and "a/bc" was stored under an unreachable name. It must now match the documented rule: lowercase letters, digits and hyphens, starting and ending with a letter or digit, 3-63 characters. M2: the retention lock is modelled. Once effectiveTime has been reached, backupMinimumEnforcedRetentionDuration may only increase and effectiveTime cannot change or be cleared (FAILED_PRECONDITION). The retention must be between 1 day and 99 years (previously 0s and 1s were accepted). M3 (documented only): the handler docs now name the unimplemented resources and note that /backupPlans is currently answered by gkebackup. M4: SetUsage moved out of the production API into export_test.go; the server tests seed usage through the snapshot/restore seam instead. L1: list honours filter (name/state/description/accessRestriction/ backupRetentionInheritance/labels. with = or !=, joined by AND) and orderBy (name/createTime/updateTime, asc/desc). An unsupported expression is 400 INVALID_ARGUMENT instead of silently returning every vault. L2: errors[].reason uses the camelCase tokens every other GCP handler uses ("aborted", "failedPrecondition"); gcprest now maps "aborted" to the canonical ABORTED status. The etag error keeps the vault name. L3: validateOnly create, update and delete no longer mint, record or register an operation; they return a done operation with the inline response and no name. --- go.mod | 5 +- go.sum | 10 +- providers/gcp/backupdr/backupdr.go | 49 ++-- providers/gcp/backupdr/backupdr_test.go | 159 ++++++++++- providers/gcp/backupdr/export_test.go | 23 ++ providers/gcp/backupdr/validate.go | 114 ++++++-- server/gcp/backupdr/filter.go | 204 ++++++++++++++ server/gcp/backupdr/filter_test.go | 60 ++++ server/gcp/backupdr/gapic_sdk_test.go | 173 ++++++++++++ server/gcp/backupdr/handler.go | 10 + server/gcp/backupdr/operations.go | 75 ++++- server/gcp/backupdr/review_fixes_test.go | 318 ++++++++++++++++++++++ server/gcp/backupdr/sdk_roundtrip_test.go | 59 +++- server/gcp/backupdr/wire.go | 58 ++-- server/wire/gcprest/gcprest.go | 6 +- services/backupdr/driver/driver.go | 6 +- 16 files changed, 1238 insertions(+), 91 deletions(-) create mode 100644 providers/gcp/backupdr/export_test.go create mode 100644 server/gcp/backupdr/filter.go create mode 100644 server/gcp/backupdr/filter_test.go create mode 100644 server/gcp/backupdr/gapic_sdk_test.go create mode 100644 server/gcp/backupdr/review_fixes_test.go diff --git a/go.mod b/go.mod index 9cdebd0b5..9c83160d1 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,13 @@ go 1.25.0 require ( cloud.google.com/go/artifactregistry v1.20.0 + cloud.google.com/go/backupdr v1.16.0 cloud.google.com/go/bigtable v1.42.0 cloud.google.com/go/compute v1.60.0 cloud.google.com/go/eventarc v1.18.0 cloud.google.com/go/firestore v1.22.0 - cloud.google.com/go/iam v1.7.0 - cloud.google.com/go/longrunning v0.9.0 + cloud.google.com/go/iam v1.11.0 + cloud.google.com/go/longrunning v1.2.0 cloud.google.com/go/secretmanager v1.16.0 cloud.google.com/go/storage v1.62.1 github.com/Azure/azure-kusto-go v0.16.1 diff --git a/go.sum b/go.sum index 444cf6647..8e5cabe6d 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/backupdr v1.16.0 h1:K2xLA4ZKu+jmX8V81r2M5X0mXCMXnNMoNwAP1egogLA= +cloud.google.com/go/backupdr v1.16.0/go.mod h1:VHHb3pxFlkJ28RvyZAEbECG6Sq5JH9JfZIjZqdFTP+U= cloud.google.com/go/bigtable v1.42.0 h1:SREvT4jLhJQZXUjsLmFs/1SMQJ+rKEj1cJuPE9liQs8= cloud.google.com/go/bigtable v1.42.0/go.mod h1:oZ30nofVB6/UYGg7lBwGLWSea7NZUvw/WvBBgLY07xU= cloud.google.com/go/compute v1.60.0 h1:CqGt23ysz990ZZe1vq/9aDPKKnmwM6kcC7Y1Q05H2kI= @@ -18,12 +20,12 @@ cloud.google.com/go/eventarc v1.18.0 h1:8WWG1/ogInYur1NQjML6EMHQ0ZBzAdMDGlUVpLD5 cloud.google.com/go/eventarc v1.18.0/go.mod h1:/6SDoqh5+9QNUqCX4/oQcJVK16fG/snHBSXu7lrJtO8= cloud.google.com/go/firestore v1.22.0 h1:avooeboIq37vKXobrbPUFhFBxS/c3FqmWoX0xs8dO6E= cloud.google.com/go/firestore v1.22.0/go.mod h1:PaM4i7i7ruALSKmlpHXXZaPObcZw0W7ie5UOPr72iTU= -cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= -cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= -cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= -cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= diff --git a/providers/gcp/backupdr/backupdr.go b/providers/gcp/backupdr/backupdr.go index 0cdab9c10..7d312af60 100644 --- a/providers/gcp/backupdr/backupdr.go +++ b/providers/gcp/backupdr/backupdr.go @@ -2,7 +2,8 @@ // control plane (backupdr.googleapis.com/v1). It models backup vaults and the // long-running operations their mutating RPCs return. It is control-plane only: // there are no data sources, backups, backup plans or restores, so every vault -// is empty unless a test seeds usage through SetUsage. +// is empty (backupCount 0) unless its state is restored from a snapshot that +// says otherwise. package backupdr import ( @@ -60,7 +61,15 @@ func resourceName(project, location, id string) string { // newOp records a completed operation scoped to the project+location it acted in // and returns it. The caller holds the write lock. -func (m *Mock) newOp(project, location, opType, target string) *bdrdriver.Operation { +// +// A validateOnly request performs no mutation, so it mints no operation id and +// records nothing: it returns a done operation with an empty name, which an LRO +// client resolves from the inline response without polling. +func (m *Mock) newOp(project, location, opType, target string, validateOnly bool) *bdrdriver.Operation { + if validateOnly { + return &bdrdriver.Operation{Done: true, TargetName: target, Type: opType} + } + scope := "projects/" + project + "/locations/" + location op := bdrdriver.Operation{ Name: fmt.Sprintf("%s/operations/operation-%d-%s", scope, m.opSeq.Add(1), idgen.UUID()), @@ -117,7 +126,7 @@ func (m *Mock) CreateBackupVault(_ context.Context, cfg *bdrdriver.BackupVaultCo m.vaults.Set(key, v) } - op := m.newOp(cfg.Project, cfg.Location, opCreate, key) + op := m.newOp(cfg.Project, cfg.Location, opCreate, key, cfg.ValidateOnly) out := cloneVault(&v) return &out, op, nil @@ -184,19 +193,21 @@ func (m *Mock) UpdateBackupVault(_ context.Context, cfg *bdrdriver.BackupVaultCo return nil, nil, fmt.Errorf("backup vault %q: %w", key, bdrdriver.ErrEtagMismatch) } - if err := applyMask(&v, cfg, fields); err != nil { + now := m.opts.Clock.Now().UTC() + + if err := applyMask(&v, cfg, fields, now); err != nil { return nil, nil, err } v.Revision++ - v.UpdateTime = m.opts.Clock.Now().UTC() + v.UpdateTime = now v.Etag = etagFor(&v) if !cfg.ValidateOnly { m.vaults.Set(key, v) } - op := m.newOp(cfg.Project, cfg.Location, opUpdate, key) + op := m.newOp(cfg.Project, cfg.Location, opUpdate, key, cfg.ValidateOnly) out := cloneVault(&v) return &out, op, nil @@ -216,7 +227,7 @@ func (m *Mock) DeleteBackupVault(_ context.Context, req *bdrdriver.DeleteBackupV v, ok := m.vaults.Get(key) if !ok { if req.AllowMissing { - return m.newOp(req.Project, req.Location, opDelete, key), nil + return m.newOp(req.Project, req.Location, opDelete, key, req.ValidateOnly), nil } return nil, notFoundErr(req.Project, req.Location, req.ID) @@ -235,7 +246,7 @@ func (m *Mock) DeleteBackupVault(_ context.Context, req *bdrdriver.DeleteBackupV m.vaults.Delete(key) } - return m.newOp(req.Project, req.Location, opDelete, key), nil + return m.newOp(req.Project, req.Location, opDelete, key, req.ValidateOnly), nil } // GetOperation returns a (done) long-running operation by name. An unknown name @@ -255,28 +266,6 @@ func (m *Mock) GetOperation(_ context.Context, name string) (*bdrdriver.Operatio return &out, nil } -// SetUsage seeds a vault's output-only backupCount and totalStoredBytes. The -// emulator has no data plane, so this is the only way a vault becomes -// non-empty (deletable=false); tests use it to exercise the force-delete guard. -// It is deliberately not part of the driver interface. -func (m *Mock) SetUsage(project, location, id string, backupCount, totalStoredBytes int64) error { - m.mu.Lock() - defer m.mu.Unlock() - - key := resourceName(project, location, id) - - v, ok := m.vaults.Get(key) - if !ok { - return notFoundErr(project, location, id) - } - - v.BackupCount = backupCount - v.TotalStoredBytes = totalStoredBytes - m.vaults.Set(key, v) - - return nil -} - // notFoundErr builds the NOT_FOUND error carrying the full resource name. func notFoundErr(project, location, id string) error { return cerrors.Newf(cerrors.NotFound, "backup vault %q not found", resourceName(project, location, id)) diff --git a/providers/gcp/backupdr/backupdr_test.go b/providers/gcp/backupdr/backupdr_test.go index f8cab270a..cb51e698b 100644 --- a/providers/gcp/backupdr/backupdr_test.go +++ b/providers/gcp/backupdr/backupdr_test.go @@ -86,6 +86,14 @@ func TestCreateValidation(t *testing.T) { "malformed retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "1d" }, "negative retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "-5s" }, "short id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "ab" }, + "uppercase id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "Bad_ID" }, + "slash id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "a/bc" }, + "trailing hyphen id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "abc-" }, + "leading hyphen id": func(c *bdrdriver.BackupVaultConfig) { c.ID = "-abc" }, + "long id": func(c *bdrdriver.BackupVaultConfig) { c.ID = strings.Repeat("a", 64) }, + "zero retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "0s" }, + "sub-day retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "86399s" }, + "over 99y retention": func(c *bdrdriver.BackupVaultConfig) { c.BackupMinimumEnforcedRetentionDuration = "3124202401s" }, "bad access": func(c *bdrdriver.BackupVaultConfig) { c.AccessRestriction = "NOPE" }, "bad inheritance": func(c *bdrdriver.BackupVaultConfig) { c.BackupRetentionInheritance = "NOPE" }, "bad effectiveTime": func(c *bdrdriver.BackupVaultConfig) { c.EffectiveTime = "yesterday" }, @@ -103,8 +111,10 @@ func TestCreateValidation(t *testing.T) { }) } - for _, ok := range []string{"0s", "1.5s", "3600s"} { + // The documented range is 1 day to 99 years, inclusive. + for _, ok := range []string{"86400s", "86400.5s", "3124202400s"} { cfg := vaultCfg("ok-" + strings.ReplaceAll(strings.TrimSuffix(ok, "s"), ".", "-")) + cfg.BackupMinimumEnforcedRetentionDuration = ok if _, _, err := m.CreateBackupVault(ctx, cfg); err != nil { @@ -277,3 +287,150 @@ func TestSnapshotRoundTrip(t *testing.T) { t.Fatalf("opSeq not restored: %v %+v", err, op) } } + +func TestCreateAcceptsDocumentedIDs(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + for _, id := range []string{"abc", "1ab", "vault-2", strings.Repeat("a", 63)} { + if _, _, err := m.CreateBackupVault(ctx, vaultCfg(id)); err != nil { + t.Fatalf("id %q rejected: %v", id, err) + } + } +} + +func TestRetentionLock(t *testing.T) { + m, clk := newMock(t) + ctx := context.Background() + + lockAt := clk.Now().Add(time.Hour).Format(time.RFC3339) + + cfg := vaultCfg(testVault) + cfg.BackupMinimumEnforcedRetentionDuration = "172800s" + cfg.EffectiveTime = lockAt + + if _, _, err := m.CreateBackupVault(ctx, cfg); err != nil { + t.Fatalf("create: %v", err) + } + + patch := func(retention, effective string, mask ...string) error { + _, _, err := m.UpdateBackupVault(ctx, &bdrdriver.BackupVaultConfig{ + Project: testProject, Location: testLocation, ID: testVault, + BackupMinimumEnforcedRetentionDuration: retention, EffectiveTime: effective, + }, mask) + + return err + } + + // Before the effective time the lock is not in force: both may change. + if err := patch("86400s", "", fieldRetention); err != nil { + t.Fatalf("pre-lock decrease: %v", err) + } + + if err := patch("", lockAt, fieldEffectiveTime); err != nil { + t.Fatalf("pre-lock effectiveTime change: %v", err) + } + + clk.Advance(2 * time.Hour) + + if err := patch("3600s", "", fieldRetention); !cerrors.IsInvalidArgument(err) { + t.Fatalf("sub-day retention err = %v, want InvalidArgument", err) + } + + if err := patch("86399.5s", "", fieldRetention); !cerrors.IsInvalidArgument(err) { + t.Fatalf("sub-day fractional retention err = %v, want InvalidArgument", err) + } + + if err := patch("86400s", "", fieldRetention); err != nil { + t.Fatalf("locked, unchanged retention: %v", err) + } + + if err := patch("259200s", "", fieldRetention); err != nil { + t.Fatalf("locked increase: %v", err) + } + + if err := patch("172800s", "", fieldRetention); !cerrors.IsFailedPrecondition(err) { + t.Fatalf("locked decrease err = %v, want FailedPrecondition", err) + } + + later := clk.Now().Add(time.Hour).Format(time.RFC3339) + if err := patch("", later, fieldEffectiveTime); !cerrors.IsFailedPrecondition(err) { + t.Fatalf("locked effectiveTime move err = %v, want FailedPrecondition", err) + } + + if err := patch("", "", fieldEffectiveTime); !cerrors.IsFailedPrecondition(err) { + t.Fatalf("locked effectiveTime clear err = %v, want FailedPrecondition", err) + } + + got, err := m.GetBackupVault(ctx, testProject, testLocation, testVault) + if err != nil { + t.Fatalf("get: %v", err) + } + + if got.BackupMinimumEnforcedRetentionDuration != "259200s" || got.EffectiveTime != lockAt { + t.Fatalf("locked vault changed: retention %q effectiveTime %q", got.BackupMinimumEnforcedRetentionDuration, got.EffectiveTime) + } +} + +func TestValidateOnlyRecordsNoOperation(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + if _, _, err := m.CreateBackupVault(ctx, vaultCfg(testVault)); err != nil { + t.Fatalf("create: %v", err) + } + + before := len(m.operations.All()) + + dry := vaultCfg("dry-run") + dry.ValidateOnly = true + + _, op, err := m.CreateBackupVault(ctx, dry) + if err != nil || !op.Done || op.Name != "" { + t.Fatalf("validateOnly create op = %+v, err %v; want done and unnamed", op, err) + } + + patch := vaultCfg(testVault) + patch.ValidateOnly = true + + if _, op, err = m.UpdateBackupVault(ctx, patch, []string{fieldDescription}); err != nil || op.Name != "" { + t.Fatalf("validateOnly update op = %+v, err %v", op, err) + } + + for _, req := range []*bdrdriver.DeleteBackupVaultRequest{ + {Project: testProject, Location: testLocation, ID: testVault, ValidateOnly: true}, + {Project: testProject, Location: testLocation, ID: "absent", ValidateOnly: true, AllowMissing: true}, + } { + if op, err = m.DeleteBackupVault(ctx, req); err != nil || op.Name != "" { + t.Fatalf("validateOnly delete %s op = %+v, err %v", req.ID, op, err) + } + } + + if after := len(m.operations.All()); after != before { + t.Fatalf("validateOnly recorded %d operations", after-before) + } + + if _, err := m.GetBackupVault(ctx, testProject, testLocation, testVault); err != nil { + t.Fatalf("validateOnly delete removed the vault: %v", err) + } +} + +func TestGetOperation(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + _, op, err := m.CreateBackupVault(ctx, vaultCfg(testVault)) + if err != nil { + t.Fatalf("create: %v", err) + } + + got, err := m.GetOperation(ctx, op.Name) + if err != nil || got.Type != opCreate || got.TargetName != resourceName(testProject, testLocation, testVault) { + t.Fatalf("GetOperation(%s) = %+v, %v", op.Name, got, err) + } + + unknown, err := m.GetOperation(ctx, "projects/p/locations/l/operations/nope") + if err != nil || !unknown.Done || unknown.Type != "" { + t.Fatalf("unknown op = %+v, %v", unknown, err) + } +} diff --git a/providers/gcp/backupdr/export_test.go b/providers/gcp/backupdr/export_test.go new file mode 100644 index 000000000..412c444be --- /dev/null +++ b/providers/gcp/backupdr/export_test.go @@ -0,0 +1,23 @@ +package backupdr + +// SetUsage seeds a vault's output-only backupCount and totalStoredBytes. The +// emulator has no data plane, so a test uses this to make a vault non-empty +// (deletable=false) and exercise the force-delete guard. It is a test seam +// only: it is compiled into this package's tests and is not production API. +func (m *Mock) SetUsage(project, location, id string, backupCount, totalStoredBytes int64) error { + m.mu.Lock() + defer m.mu.Unlock() + + key := resourceName(project, location, id) + + v, ok := m.vaults.Get(key) + if !ok { + return notFoundErr(project, location, id) + } + + v.BackupCount = backupCount + v.TotalStoredBytes = totalStoredBytes + m.vaults.Set(key, v) + + return nil +} diff --git a/providers/gcp/backupdr/validate.go b/providers/gcp/backupdr/validate.go index 85cedff45..9f42ea1e5 100644 --- a/providers/gcp/backupdr/validate.go +++ b/providers/gcp/backupdr/validate.go @@ -17,10 +17,12 @@ const ( accessWithinOrganization = "WITHIN_ORGANIZATION" accessUnspecified = "ACCESS_RESTRICTION_UNSPECIFIED" - // minVaultIDLen / maxVaultIDLen bound a vault id, per the BackupVault.name - // contract ("must be between 3-63 characters long"). - minVaultIDLen = 3 - maxVaultIDLen = 63 + // minRetention / maxRetention bound backupMinimumEnforcedRetentionDuration: + // "The minimum is 1 day and the maximum is 99 years" (Backup and DR "Create + // a backup vault"). 99 years is taken as 99 x 365.25 days so no calendar + // reading of "99 years" is rejected. + minRetention = 24 * time.Hour + maxRetention = 36159*24*time.Hour + 18*time.Hour // serviceAccountDomain is the Backup and DR service-agent domain. CloudEmu // synthesizes the vault serviceAccount as @@ -50,6 +52,12 @@ const ( decimalBase = 10 ) +// vaultIDPattern is the documented backup vault name rule: only lowercase +// letters, digits and hyphens, starting and ending with a letter or digit, 3-63 +// characters (Backup and DR "Backup vaults", name requirements). It also keeps +// '/' out of an id, which would otherwise mint an unreachable resource name. +var vaultIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$`) + // durationPattern matches the google.protobuf.Duration JSON form: an optionally // signed decimal number of seconds with up to nine fractional digits and an "s" // suffix (e.g. "86400s", "1.5s", "-3s"). @@ -100,9 +108,10 @@ func validateCreate(cfg *bdrdriver.BackupVaultConfig) error { return cerrors.New(cerrors.InvalidArgument, "a concrete location is required") } - if n := len(cfg.ID); n < minVaultIDLen || n > maxVaultIDLen { + if !vaultIDPattern.MatchString(cfg.ID) { return cerrors.Newf(cerrors.InvalidArgument, - "backupVaultId %q must be between %d and %d characters", cfg.ID, minVaultIDLen, maxVaultIDLen) + "backupVaultId %q must be 3-63 characters of lowercase letters, digits and hyphens, "+ + "starting and ending with a letter or digit", cfg.ID) } checks := []func(*bdrdriver.BackupVaultConfig) error{ @@ -119,23 +128,91 @@ func validateCreate(cfg *bdrdriver.BackupVaultConfig) error { } // checkRetention requires backupMinimumEnforcedRetentionDuration to be a -// well-formed, non-negative google.protobuf.Duration string. +// well-formed google.protobuf.Duration string between 1 day and 99 years. func checkRetention(cfg *bdrdriver.BackupVaultConfig) error { d := cfg.BackupMinimumEnforcedRetentionDuration if d == "" { return cerrors.New(cerrors.InvalidArgument, "backupMinimumEnforcedRetentionDuration is required") } - m := durationPattern.FindStringSubmatch(d) - if m == nil { + got, err := parseRetention(d) + if err != nil { + return err + } + + if got < minRetention || got > maxRetention { return cerrors.Newf(cerrors.InvalidArgument, + "backupMinimumEnforcedRetentionDuration %q must be between 1 day (86400s) and 99 years", d) + } + + return nil +} + +// parseRetention parses a google.protobuf.Duration JSON string ("86400s"). +func parseRetention(d string) (time.Duration, error) { + if !durationPattern.MatchString(d) { + return 0, cerrors.Newf(cerrors.InvalidArgument, "backupMinimumEnforcedRetentionDuration %q is not a valid duration (want e.g. \"86400s\")", d) } - if m[1] != "" { - if secs, err := strconv.ParseFloat(strings.TrimSuffix(d, "s"), 64); err != nil || secs != 0 { - return cerrors.Newf(cerrors.InvalidArgument, "backupMinimumEnforcedRetentionDuration %q must not be negative", d) - } + got, err := time.ParseDuration(d) + if err != nil { + return 0, cerrors.Newf(cerrors.InvalidArgument, + "backupMinimumEnforcedRetentionDuration %q is out of range", d) + } + + return got, nil +} + +// lockedAt reports whether a vault's retention lock is in effect at now: an +// effectiveTime is set and has been reached. The stored value was validated as +// RFC 3339 on write. +func lockedAt(v *bdrdriver.BackupVault, now time.Time) bool { + if v.EffectiveTime == "" { + return false + } + + t, err := time.Parse(time.RFC3339Nano, v.EffectiveTime) + + return err == nil && !now.Before(t) +} + +// checkLock enforces the retention lock on a masked update. Before the +// effective time a vault's retention and lock time may change freely; once it +// has passed, "no one (not even a Project Owner) can decrease the retention +// period. You are only permitted to increase it", and the lock "cannot be +// removed if the effective date has been reached", so effectiveTime is frozen. +// Both rejections are FAILED_PRECONDITION: the request is well-formed, the +// vault's state forbids it. +func checkLock(v *bdrdriver.BackupVault, cfg *bdrdriver.BackupVaultConfig, fields map[string]bool, now time.Time) error { + if !lockedAt(v, now) { + return nil + } + + name := resourceName(v.Project, v.Location, v.ID) + + if fields[fieldEffectiveTime] && cfg.EffectiveTime != v.EffectiveTime { + return cerrors.Newf(cerrors.FailedPrecondition, + "backup vault %q is locked since %s; effectiveTime cannot be changed", name, v.EffectiveTime) + } + + if !fields[fieldRetention] { + return nil + } + + // A stored value that does not parse (never written by this mock) compares + // as zero, so any valid new value is an increase. + cur, _ := parseRetention(v.BackupMinimumEnforcedRetentionDuration) + + next, err := parseRetention(cfg.BackupMinimumEnforcedRetentionDuration) + if err != nil { + return err + } + + if next < cur { + return cerrors.Newf(cerrors.FailedPrecondition, + "backup vault %q is locked since %s; backupMinimumEnforcedRetentionDuration can only be increased (currently %s)", + name, v.EffectiveTime, v.BackupMinimumEnforcedRetentionDuration) } return nil @@ -208,9 +285,9 @@ func normalizeMask(mask []string) (map[string]bool, error) { return out, nil } -// applyMask copies each masked field from cfg onto v, validating the new value. -// Fields outside the mask are left untouched. -func applyMask(v *bdrdriver.BackupVault, cfg *bdrdriver.BackupVaultConfig, fields map[string]bool) error { +// applyMask copies each masked field from cfg onto v, validating the new value +// and the retention lock as of now. Fields outside the mask are left untouched. +func applyMask(v *bdrdriver.BackupVault, cfg *bdrdriver.BackupVaultConfig, fields map[string]bool, now time.Time) error { checks := map[string]func(*bdrdriver.BackupVaultConfig) error{ fieldRetention: checkRetention, fieldInheritance: checkInheritance, @@ -226,7 +303,12 @@ func applyMask(v *bdrdriver.BackupVault, cfg *bdrdriver.BackupVaultConfig, field } } + if err := checkLock(v, cfg, fields, now); err != nil { + return err + } + setters := map[string]func(){ + fieldDescription: func() { v.Description = cfg.Description }, fieldLabels: func() { v.Labels = cloneStrMap(cfg.Labels) }, fieldAnnotations: func() { v.Annotations = cloneStrMap(cfg.Annotations) }, diff --git a/server/gcp/backupdr/filter.go b/server/gcp/backupdr/filter.go new file mode 100644 index 000000000..9960729e2 --- /dev/null +++ b/server/gcp/backupdr/filter.go @@ -0,0 +1,204 @@ +package backupdr + +import ( + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +const ( + labelsPrefix = "labels." + andSep = " AND " + descKeyword = "desc" + ascKeyword = "asc" + + filterName = "name" + filterState = "state" + filterDescription = "description" + filterAccess = "accessRestriction" + filterInheritance = "backupRetentionInheritance" + + orderName = filterName + orderCreateTime = "createTime" + orderUpdateTime = "updateTime" +) + +// clause is one `field (=|!=) value` term of an AIP-160 list filter. +type clause struct { + field string + value string + negate bool +} + +// parseFilter parses the AIP-160 subset ListBackupVaults honors: one or more +// `field = "value"` / `field != "value"` terms joined by AND, where field is +// name (full resource name or bare id), state, description, +// accessRestriction, backupRetentionInheritance or labels.. Anything else +// is INVALID_ARGUMENT rather than silently matching every vault, so a caller +// never mistakes an unfiltered list for a filtered one. +func parseFilter(filter string) ([]clause, error) { + filter = strings.TrimSpace(filter) + if filter == "" { + return nil, nil + } + + terms := strings.Split(filter, andSep) + out := make([]clause, 0, len(terms)) + + for _, term := range terms { + c, ok := parseClause(term) + if !ok { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "unsupported filter %q: supported terms are name, state, description, accessRestriction, "+ + "backupRetentionInheritance and labels. compared with = or !=, joined by AND", filter) + } + + out = append(out, c) + } + + return out, nil +} + +// parseClause splits one `field op value` term; ok is false for an unknown +// field, an unsupported operator, or an empty side. +func parseClause(term string) (clause, bool) { + op, negate := "=", false + + idx := strings.Index(term, "!=") + if idx >= 0 { + op, negate = "!=", true + } else if idx = strings.Index(term, "="); idx < 0 { + return clause{}, false + } + + field := strings.TrimSpace(term[:idx]) + value := strings.TrimSpace(term[idx+len(op):]) + + value, ok := unquote(value) + if !ok || field == "" || value == "" || !filterableField(field) { + return clause{}, false + } + + return clause{field: field, value: value, negate: negate}, true +} + +// unquote returns a single literal: a "…" or '…' string without its quotes +// (and without the quote character inside it), or a bare token with no +// whitespace. Anything else (e.g. `"a" OR b = "c"`) is not one literal. +func unquote(value string) (string, bool) { + if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] { + inner := value[1 : len(value)-1] + + return inner, !strings.ContainsRune(inner, rune(value[0])) + } + + return value, !strings.ContainsAny(value, " \t\"'") +} + +// filterableField reports whether field is one parseFilter can evaluate. +func filterableField(field string) bool { + switch field { + case filterName, filterState, filterDescription, filterAccess, filterInheritance: + return true + default: + return strings.HasPrefix(field, labelsPrefix) && len(field) > len(labelsPrefix) + } +} + +// matchesAll reports whether v satisfies every clause. +func matchesAll(v *bdrdriver.BackupVault, clauses []clause) bool { + for _, c := range clauses { + if matchClause(v, c) == c.negate { + return false + } + } + + return true +} + +// matchClause reports whether v's field equals the clause value (before +// negation). name matches the full resource name or the bare vault id. +func matchClause(v *bdrdriver.BackupVault, c clause) bool { + switch c.field { + case filterName: + return c.value == resourceName(v.Project, v.Location, v.ID) || c.value == v.ID + case filterState: + return c.value == v.State + case filterDescription: + return c.value == v.Description + case filterAccess: + return c.value == v.AccessRestriction + case filterInheritance: + return c.value == v.BackupRetentionInheritance + default: + got, ok := v.Labels[strings.TrimPrefix(c.field, labelsPrefix)] + + return ok && got == c.value + } +} + +// parseOrderBy parses the orderBy ListBackupVaults honors: a single field of +// name, createTime or updateTime, optionally followed by asc or desc. An empty +// orderBy is name ascending. Anything else is INVALID_ARGUMENT. +func parseOrderBy(orderBy string) (field string, desc bool, err error) { + parts := strings.Fields(orderBy) + + switch { + case len(parts) == 0: + return orderName, false, nil + case len(parts) > 2: //nolint:mnd // field plus direction + return "", false, unsupportedOrderBy(orderBy) + } + + field = parts[0] + if field != orderName && field != orderCreateTime && field != orderUpdateTime { + return "", false, unsupportedOrderBy(orderBy) + } + + if len(parts) == 2 { //nolint:mnd // field plus direction + switch strings.ToLower(parts[1]) { + case descKeyword: + desc = true + case ascKeyword: + default: + return "", false, unsupportedOrderBy(orderBy) + } + } + + return field, desc, nil +} + +func unsupportedOrderBy(orderBy string) error { + return cerrors.Newf(cerrors.InvalidArgument, + "unsupported orderBy %q: supported are name, createTime or updateTime, optionally followed by asc or desc", orderBy) +} + +// vaultLess orders vaults by field (ties broken by resource name), reversed +// when desc. +func vaultLess(field string, desc bool) func(a, b bdrdriver.BackupVault) bool { + return func(a, b bdrdriver.BackupVault) bool { + na, nb := resourceName(a.Project, a.Location, a.ID), resourceName(b.Project, b.Location, b.ID) + + var less, equal bool + + switch field { + case orderCreateTime: + less, equal = a.CreateTime.Before(b.CreateTime), a.CreateTime.Equal(b.CreateTime) + case orderUpdateTime: + less, equal = a.UpdateTime.Before(b.UpdateTime), a.UpdateTime.Equal(b.UpdateTime) + default: + less, equal = na < nb, na == nb + } + + if equal { + less = na < nb + } + + if desc { + return !less && na != nb + } + + return less + } +} diff --git a/server/gcp/backupdr/filter_test.go b/server/gcp/backupdr/filter_test.go new file mode 100644 index 000000000..73d017f55 --- /dev/null +++ b/server/gcp/backupdr/filter_test.go @@ -0,0 +1,60 @@ +package backupdr + +import ( + "testing" + "time" + + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" +) + +func TestFilterClauses(t *testing.T) { + v := &bdrdriver.BackupVault{ + Project: "p", Location: "l", ID: "v1", State: "ACTIVE", Description: "nightly", + AccessRestriction: "WITHIN_PROJECT", BackupRetentionInheritance: "INHERIT_VAULT_RETENTION", + Labels: map[string]string{"env": "prod"}, + } + + cases := map[string]bool{ + `name = "v1"`: true, + `name = 'projects/p/locations/l/backupVaults/v1'`: true, + `description = "nightly"`: true, + `accessRestriction = WITHIN_PROJECT`: true, + `backupRetentionInheritance != "MATCH_BACKUP_EXPIRE_TIME"`: true, + `labels.env = "dev"`: false, + `labels.missing = "x"`: false, + `state = "ACTIVE" AND description != "nightly"`: false, + } + + for filter, want := range cases { + clauses, err := parseFilter(filter) + if err != nil { + t.Fatalf("parseFilter(%q): %v", filter, err) + } + + if got := matchesAll(v, clauses); got != want { + t.Fatalf("filter %q matched %v, want %v", filter, got, want) + } + } + + for _, bad := range []string{ + `name = "a"b"`, `name = a b`, `labels. = "x"`, `= "x"`, `name = ""`, `name`, `name > "a"`, + } { + if _, err := parseFilter(bad); err == nil { + t.Fatalf("parseFilter(%q) accepted an unsupported filter", bad) + } + } +} + +func TestVaultLessTieBreak(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + a := bdrdriver.BackupVault{Project: "p", Location: "l", ID: "a", CreateTime: now} + b := bdrdriver.BackupVault{Project: "p", Location: "l", ID: "b", CreateTime: now} + + if !vaultLess(orderCreateTime, false)(a, b) || vaultLess(orderCreateTime, false)(b, a) { + t.Fatalf("equal createTime must break ties by name ascending") + } + + if !vaultLess(orderCreateTime, true)(b, a) || vaultLess(orderCreateTime, true)(a, a) { + t.Fatalf("desc must reverse the order and stay irreflexive") + } +} diff --git a/server/gcp/backupdr/gapic_sdk_test.go b/server/gcp/backupdr/gapic_sdk_test.go new file mode 100644 index 000000000..b8d7fe055 --- /dev/null +++ b/server/gcp/backupdr/gapic_sdk_test.go @@ -0,0 +1,173 @@ +package backupdr_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + gapic "cloud.google.com/go/backupdr/apiv1" + "cloud.google.com/go/backupdr/apiv1/backupdrpb" + "google.golang.org/api/option" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" + bdrserver "github.com/stackshy/cloudemu/v2/server/gcp/backupdr" +) + +// newGAPIC returns the idiomatic cloud.google.com/go/backupdr/apiv1 REST client +// pointed at srv. Unlike the google.golang.org/api discovery client, the GAPIC +// client decodes a completed operation's `response` Any and fails Wait when it +// is absent, so it is the client that proves the LRO wire shape. +func newGAPIC(t *testing.T, srv http.Handler) *gapic.Client { + t.Helper() + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + c, err := gapic.NewRESTClient(context.Background(), + option.WithEndpoint(ts.URL), + option.WithoutAuthentication(), + ) + if err != nil { + t.Fatalf("NewRESTClient: %v", err) + } + + t.Cleanup(func() { _ = c.Close() }) + + return c +} + +// gapicCreate creates a vault and waits for the returned operation. +func gapicCreate(t *testing.T, c *gapic.Client, id string) (*backupdrpb.BackupVault, string) { + t.Helper() + + ctx := context.Background() + + op, err := c.CreateBackupVault(ctx, &backupdrpb.CreateBackupVaultRequest{ + Parent: "projects/" + sdkProject + "/locations/" + sdkLocation, + BackupVaultId: id, + BackupVault: &backupdrpb.BackupVault{ + BackupMinimumEnforcedRetentionDuration: durationpb.New(86400e9), + }, + }) + if err != nil { + t.Fatalf("CreateBackupVault: %v", err) + } + + v, err := op.Wait(ctx) + if err != nil { + t.Fatalf("CreateBackupVault Wait: %v", err) + } + + return v, op.Name() +} + +// assertDeleteWaits deletes name through the GAPIC client, then Waits both on +// the returned operation (the inline response) and on a handle rebuilt from the +// operation name (which forces a poll), so both shapes must carry the Empty +// response the client requires. +func assertDeleteWaits(t *testing.T, c *gapic.Client, name string) { + t.Helper() + + ctx := context.Background() + + op, err := c.DeleteBackupVault(ctx, &backupdrpb.DeleteBackupVaultRequest{Name: name}) + if err != nil { + t.Fatalf("DeleteBackupVault: %v", err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("DeleteBackupVault Wait (inline response): %v", err) + } + + if err := c.DeleteBackupVaultOperation(op.Name()).Wait(ctx); err != nil { + t.Fatalf("DeleteBackupVault Wait (polled %s): %v", op.Name(), err) + } + + if _, err := c.GetBackupVault(ctx, &backupdrpb.GetBackupVaultRequest{Name: name}); err == nil { + t.Fatalf("GetBackupVault after delete succeeded, want NOT_FOUND") + } +} + +// TestGAPICDeleteWaitSharedPoller drives create+delete with Wait through the +// assembled GCP server, where operation polls go to the shared LRO poller. +func TestGAPICDeleteWaitSharedPoller(t *testing.T) { + cloud := cloudemu.NewGCP(config.WithClock(config.NewFakeClock(fixedNow))) + c := newGAPIC(t, gcpserver.NewFromProvider(cloud)) + + v, _ := gapicCreate(t, c, "gapic-vault") + assertDeleteWaits(t, c, v.GetName()) +} + +// TestGAPICDeleteWaitStandalone drives the same flow against a standalone +// package handler (no shared registry), which serves its own operation polls. +func TestGAPICDeleteWaitStandalone(t *testing.T) { + cloud := cloudemu.NewGCP(config.WithClock(config.NewFakeClock(fixedNow))) + c := newGAPIC(t, bdrserver.New(cloud.BackupDR)) + + v, opName := gapicCreate(t, c, "gapic-vault") + + // A create poll through the standalone handler must carry the vault too. + polled, err := c.CreateBackupVaultOperation(opName).Wait(context.Background()) + if err != nil { + t.Fatalf("CreateBackupVault Wait (polled %s): %v", opName, err) + } + + if polled.GetName() != v.GetName() { + t.Fatalf("polled create response name = %q, want %q", polled.GetName(), v.GetName()) + } + + assertDeleteWaits(t, c, v.GetName()) +} + +// fixedNow is the fake-clock start for the GAPIC tests. +// +//nolint:gochecknoglobals // immutable test fixture +var fixedNow = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +// TestStandalonePollShapes: a standalone poll of an unknown operation, or of a +// create whose vault has since been deleted, replays google.protobuf.Empty. +func TestStandalonePollShapes(t *testing.T) { + cloud := cloudemu.NewGCP(config.WithClock(config.NewFakeClock(fixedNow))) + ts := httptest.NewServer(bdrserver.New(cloud.BackupDR)) + t.Cleanup(ts.Close) + + c := newGAPIC(t, bdrserver.New(cloud.BackupDR)) + v, createOp := gapicCreate(t, c, "gone") + assertDeleteWaits(t, c, v.GetName()) + + parent := "projects/" + sdkProject + "/locations/" + sdkLocation + for _, name := range []string{createOp, parent + "/operations/never-created"} { + resp, err := http.Get(ts.URL + "/v1/" + name) //nolint:noctx // test poll + if err != nil { + t.Fatalf("poll %s: %v", name, err) + } + + var op struct { + Done bool `json:"done"` + Response map[string]any `json:"response"` + } + + err = json.NewDecoder(resp.Body).Decode(&op) + resp.Body.Close() + + if err != nil || !op.Done || op.Response["@type"] != "type.googleapis.com/google.protobuf.Empty" { + t.Fatalf("poll %s = %+v (%v), want done with an Empty response", name, op, err) + } + } + + resp, err := http.Post(ts.URL+"/v1/"+parent+"/operations/x", "application/json", nil) //nolint:noctx // test + if err != nil { + t.Fatalf("POST operation: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("POST operation = %d, want 405", resp.StatusCode) + } +} diff --git a/server/gcp/backupdr/handler.go b/server/gcp/backupdr/handler.go index 993fb09b4..4277b4970 100644 --- a/server/gcp/backupdr/handler.go +++ b/server/gcp/backupdr/handler.go @@ -23,6 +23,16 @@ // is wired, letting that poller win; a standalone package server (no registry) // serves its own polls. The backupVaults resource-type guard keeps this handler // disjoint from every other /v1/projects/ handler. +// +// Known gap (not implemented): backupPlans, backupPlanAssociations, +// dataSources, backups and managementServers. This handler does not claim +// those paths, so a vault is never protected by a backup-plan reference +// (ignoreBackupPlanReferences on delete has nothing to check). Note that +// /v1/projects/{p}/locations/{l}/backupPlans requests are currently answered +// by the GKE Backup handler (server/gcp/gkebackup), which shares that path +// shape and replies with a google.cloud.gkebackup.v1.BackupPlan; a Backup and +// DR backup plan (e.g. Terraform google_backup_dr_backup_plan) therefore lands +// in GKE Backup state rather than being rejected. package backupdr import ( diff --git a/server/gcp/backupdr/operations.go b/server/gcp/backupdr/operations.go index 767542e46..c985cd825 100644 --- a/server/gcp/backupdr/operations.go +++ b/server/gcp/backupdr/operations.go @@ -72,20 +72,40 @@ func (h *Handler) getVault(w http.ResponseWriter, r *http.Request, rt route) { } // listVaults handles GET .../backupVaults, scoped to the request's project and -// location ("-" for every location) and ordered by resource name. filter and -// orderBy are accepted and ignored. +// location ("-" for every location). filter supports the AIP-160 equality +// subset parseFilter documents and orderBy the fields parseOrderBy documents +// (default: resource name ascending); an unsupported expression in either is +// 400 INVALID_ARGUMENT rather than being ignored. func (h *Handler) listVaults(w http.ResponseWriter, r *http.Request, rt route) { + q := r.URL.Query() + + clauses, err := parseFilter(q.Get("filter")) + if err != nil { + writeErr(w, err) + return + } + + field, desc, err := parseOrderBy(q.Get("orderBy")) + if err != nil { + writeErr(w, err) + return + } + all, err := h.db.ListBackupVaults(r.Context(), rt.project, rt.location) if err != nil { writeErr(w, err) return } - page, err := pagination.PaginateSorted(all, - func(a, b bdrdriver.BackupVault) bool { - return resourceName(a.Project, a.Location, a.ID) < resourceName(b.Project, b.Location, b.ID) - }, - r.URL.Query().Get("pageToken"), pageSize(r)) + matched := make([]bdrdriver.BackupVault, 0, len(all)) + + for i := range all { + if matchesAll(&all[i], clauses) { + matched = append(matched, all[i]) + } + } + + page, err := pagination.PaginateSorted(matched, vaultLess(field, desc), q.Get("pageToken"), pageSize(r)) if err != nil { gcprest.WriteError(w, http.StatusBadRequest, "invalid", "invalid pageToken") return @@ -131,7 +151,8 @@ func (h *Handler) patchVault(w http.ResponseWriter, r *http.Request, rt route) { // deleteVault handles DELETE .../backupVaults/{id}. force, allowMissing, etag // and validateOnly are honored; ignoreBackupPlanReferences and requestId are // accepted and ignored (there are no backup plans). The operation completes -// inline with an empty response. +// inline with a google.protobuf.Empty response, which the GAPIC client's +// DeleteBackupVaultOperation.Wait requires. func (h *Handler) deleteVault(w http.ResponseWriter, r *http.Request, rt route) { req := &bdrdriver.DeleteBackupVaultRequest{ Project: rt.project, Location: rt.location, ID: rt.name, @@ -162,12 +183,16 @@ func (h *Handler) deleteVault(w http.ResponseWriter, r *http.Request, rt route) return } - gcprest.WriteJSON(w, http.StatusOK, h.doneOperation(op.Name, nil)) + h.writeEmptyOperation(w, op) } // serveOperation resolves a (done) long-running operation poll for a // standalone package server (no shared registry). The operation resource name -// is the request path without the /v1/ version prefix. +// is the request path without the /v1/ version prefix. The replayed operation +// carries the same `response` the mutating call returned: the target vault for +// a create or update (its current state, while it still exists) and +// google.protobuf.Empty otherwise, so a GAPIC Wait on a rebuilt operation +// handle decodes it. func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { writeMethodNotAllowed(w) @@ -182,7 +207,35 @@ func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request) { return } - gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true}) + if v := h.operationVault(r, op); v != nil { + h.writeVaultOperation(w, op, v) + return + } + + h.writeEmptyOperation(w, op) +} + +// operationVault returns the vault a create or update operation acted on, or +// nil for a delete, an unknown operation, or a vault that no longer exists. +func (h *Handler) operationVault(r *http.Request, op *bdrdriver.Operation) *bdrdriver.BackupVault { + if op.Type == "delete" || op.TargetName == "" { + return nil + } + + // projects/{p}/locations/{l}/backupVaults/{id} + const targetParts, collIdx, idIdx = 6, 4, 5 + + parts := strings.Split(op.TargetName, "/") + if len(parts) != targetParts || parts[collIdx] != vaultsColl { + return nil + } + + v, err := h.db.GetBackupVault(r.Context(), parts[1], parts[3], parts[idIdx]) + if err != nil { + return nil + } + + return v } // boolParam reads an optional boolean query parameter; a malformed value is a diff --git a/server/gcp/backupdr/review_fixes_test.go b/server/gcp/backupdr/review_fixes_test.go new file mode 100644 index 000000000..b515a4413 --- /dev/null +++ b/server/gcp/backupdr/review_fixes_test.go @@ -0,0 +1,318 @@ +package backupdr_test + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + backupdr "google.golang.org/api/backupdr/v1" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +// rawEnv drives the assembled server with plain HTTP, to assert on the exact +// wire bytes (error envelopes, operation shapes) an SDK would hide. +type rawEnv struct { + url string + clock *config.FakeClock + parent string +} + +func newRawEnv(t *testing.T) *rawEnv { + t.Helper() + + clk := config.NewFakeClock(fixedNow) + ts := httptest.NewServer(gcpserver.NewFromProvider(cloudemu.NewGCP(config.WithClock(clk)))) + t.Cleanup(ts.Close) + + return &rawEnv{url: ts.URL, clock: clk, parent: "projects/" + sdkProject + "/locations/" + sdkLocation} +} + +// do sends method to /v1/path with an optional JSON body and returns the +// status and decoded JSON body. +func (e *rawEnv) do(t *testing.T, method, path, body string) (int, map[string]any) { + t.Helper() + + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + + req, err := http.NewRequestWithContext(t.Context(), method, e.url+"/v1/"+path, rdr) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + + out := map[string]any{} + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decode %s %s: %v", method, path, err) + } + + return resp.StatusCode, out +} + +func (e *rawEnv) create(t *testing.T, id, body string) { + t.Helper() + + if code, out := e.do(t, http.MethodPost, e.parent+"/backupVaults?backupVaultId="+id, body); code != http.StatusOK { + t.Fatalf("create %s: %d %v", id, code, out) + } +} + +// errorOf returns the (message, status, errors[0].reason) of an error envelope. +func errorOf(t *testing.T, out map[string]any) (msg, status, reason string) { + t.Helper() + + e, ok := out["error"].(map[string]any) + if !ok { + t.Fatalf("no error envelope: %v", out) + } + + errs, _ := e["errors"].([]any) + if len(errs) == 0 { + t.Fatalf("no errors[]: %v", e) + } + + first, _ := errs[0].(map[string]any) + msg, _ = e["message"].(string) + status, _ = e["status"].(string) + reason, _ = first["reason"].(string) + + return msg, status, reason +} + +const vaultBody = `{"backupMinimumEnforcedRetentionDuration":"86400s"}` + +// TestVaultIDFormat: ids outside the documented format are 400, and a '/' in +// the id can no longer mint an unreachable vault. +func TestVaultIDFormat(t *testing.T) { + e := newRawEnv(t) + + for _, id := range []string{"Bad_ID", url.QueryEscape("a/bc"), "abc-", "-abc"} { + code, out := e.do(t, http.MethodPost, e.parent+"/backupVaults?backupVaultId="+id, vaultBody) + if code != http.StatusBadRequest { + t.Fatalf("create %q = %d %v, want 400", id, code, out) + } + } + + if _, out := e.do(t, http.MethodGet, e.parent+"/backupVaults", ""); out["backupVaults"] != nil && + len(out["backupVaults"].([]any)) != 0 { + t.Fatalf("rejected ids were stored: %v", out) + } +} + +// TestRetentionLockOverWire: once effectiveTime passes, lowering the retention +// or moving effectiveTime is a 400 FAILED_PRECONDITION; 1s retention is 400. +func TestRetentionLockOverWire(t *testing.T) { + e := newRawEnv(t) + name := e.parent + "/backupVaults/locked" + + e.create(t, "locked", `{"backupMinimumEnforcedRetentionDuration":"172800s","effectiveTime":"`+ + fixedNow.Add(time.Minute).Format(time.RFC3339)+`"}`) + + code, out := e.do(t, http.MethodPatch, name+"?updateMask=backupMinimumEnforcedRetentionDuration", + `{"backupMinimumEnforcedRetentionDuration":"1s"}`) + if code != http.StatusBadRequest { + t.Fatalf("1s retention = %d %v, want 400", code, out) + } + + e.clock.Advance(time.Hour) + + for _, tc := range []struct{ mask, body string }{ + {"backupMinimumEnforcedRetentionDuration", `{"backupMinimumEnforcedRetentionDuration":"86400s"}`}, + {"effectiveTime", `{"effectiveTime":"2030-01-01T00:00:00Z"}`}, + } { + code, out := e.do(t, http.MethodPatch, name+"?updateMask="+tc.mask, tc.body) + if code != http.StatusBadRequest { + t.Fatalf("locked patch %s = %d %v, want 400", tc.mask, code, out) + } + + if _, status, reason := errorOf(t, out); status != "FAILED_PRECONDITION" || reason != "failedPrecondition" { + t.Fatalf("locked patch %s status/reason = %q/%q", tc.mask, status, reason) + } + } + + code, out = e.do(t, http.MethodPatch, name+"?updateMask=backupMinimumEnforcedRetentionDuration", + `{"backupMinimumEnforcedRetentionDuration":"259200s"}`) + if code != http.StatusOK { + t.Fatalf("locked increase = %d %v, want 200", code, out) + } +} + +// TestErrorReasonsAndEtagMessage: errors[].reason is camelCase like every other +// GCP handler, the status is canonical, and the etag error names the vault. +func TestErrorReasonsAndEtagMessage(t *testing.T) { + e := newRawEnv(t) + name := e.parent + "/backupVaults/vault-etag" + + e.create(t, "vault-etag", vaultBody) + + code, out := e.do(t, http.MethodPatch, name+"?updateMask=description", `{"description":"x","etag":"stale"}`) + if code != http.StatusConflict { + t.Fatalf("stale etag patch = %d %v, want 409", code, out) + } + + msg, status, reason := errorOf(t, out) + if reason != "aborted" || status != "ABORTED" { + t.Fatalf("stale etag status/reason = %q/%q, want ABORTED/aborted", status, reason) + } + + if !strings.Contains(msg, name) || !strings.Contains(msg, "etag does not match") || strings.Contains(msg, "FailedPrecondition") { + t.Fatalf("stale etag message = %q, want the vault name and the etag text without the code prefix", msg) + } + + code, out = e.do(t, http.MethodDelete, name+"?etag=stale", "") + if code != http.StatusConflict { + t.Fatalf("stale etag delete = %d %v, want 409", code, out) + } + + if msg, _, _ := errorOf(t, out); !strings.Contains(msg, name) { + t.Fatalf("stale etag delete message = %q, want the vault name", msg) + } +} + +// TestValidateOnlyOperationShape: a validateOnly create/delete returns a done +// operation with its response inline and no name (nothing to poll), and the +// vault is untouched. +func TestValidateOnlyOperationShape(t *testing.T) { + e := newRawEnv(t) + name := e.parent + "/backupVaults/keep" + + e.create(t, "keep", vaultBody) + + code, out := e.do(t, http.MethodPost, e.parent+"/backupVaults?backupVaultId=dry&validateOnly=true", vaultBody) + if code != http.StatusOK || out["name"] != nil || out["done"] != true || out["response"] == nil { + t.Fatalf("validateOnly create = %d %v", code, out) + } + + code, out = e.do(t, http.MethodDelete, name+"?validateOnly=true", "") + if code != http.StatusOK || out["name"] != nil || out["done"] != true { + t.Fatalf("validateOnly delete = %d %v", code, out) + } + + if code, _ := e.do(t, http.MethodGet, name, ""); code != http.StatusOK { + t.Fatalf("validateOnly delete removed the vault: %d", code) + } + + // A real delete names its operation and carries google.protobuf.Empty, + // inline and when polled through the shared poller. + code, out = e.do(t, http.MethodDelete, name, "") + if code != http.StatusOK { + t.Fatalf("delete = %d %v", code, out) + } + + opName, _ := out["name"].(string) + resp, _ := out["response"].(map[string]any) + + if opName == "" || resp["@type"] != "type.googleapis.com/google.protobuf.Empty" { + t.Fatalf("delete op = %v, want a name and an Empty response", out) + } + + code, polled := e.do(t, http.MethodGet, opName, "") + if presp, _ := polled["response"].(map[string]any); code != http.StatusOK || + presp["@type"] != "type.googleapis.com/google.protobuf.Empty" { + t.Fatalf("polled delete op = %d %v, want an Empty response", code, polled) + } +} + +// TestListFilterAndOrderBy: filter narrows the list, orderBy orders it, and an +// unsupported expression in either is 400 instead of silently ignored. +func TestListFilterAndOrderBy(t *testing.T) { + e := newSDKEnv(t) + vaults := e.svc.Projects.Locations.BackupVaults + + for i, id := range []string{"vault-a", "vault-b", "vault-c"} { + e.create(t, id, &backupdr.BackupVault{ + BackupMinimumEnforcedRetentionDuration: retention, + Labels: map[string]string{"team": map[bool]string{true: "red", false: "blue"}[i != 1]}, + }) + e.clock.Advance(time.Minute) + } + + names := func(resp *backupdr.ListBackupVaultsResponse) string { + out := make([]string, 0, len(resp.BackupVaults)) + for _, v := range resp.BackupVaults { + out = append(out, v.Name[strings.LastIndex(v.Name, "/")+1:]) + } + + return strings.Join(out, ",") + } + + cases := []struct{ filter, orderBy, want string }{ + {`labels.team = "red"`, "", "vault-a,vault-c"}, + {`labels.team != "red"`, "", "vault-b"}, + {`name = "` + e.parent + `/backupVaults/vault-b"`, "", "vault-b"}, + {`state = "ACTIVE" AND labels.team = "red"`, "createTime desc", "vault-c,vault-a"}, + {"", "name desc", "vault-c,vault-b,vault-a"}, + {"", "updateTime", "vault-a,vault-b,vault-c"}, + } + + for _, tc := range cases { + resp, err := vaults.List(e.parent).Filter(tc.filter).OrderBy(tc.orderBy).Do() + if err != nil { + t.Fatalf("list filter=%q orderBy=%q: %v", tc.filter, tc.orderBy, err) + } + + if got := names(resp); got != tc.want { + t.Fatalf("list filter=%q orderBy=%q = %s, want %s", tc.filter, tc.orderBy, got, tc.want) + } + } + + for _, bad := range []struct{ filter, orderBy string }{ + {"createTime > \"2020-01-01T00:00:00Z\"", ""}, + {"bogus = \"x\"", ""}, + {"labels.team = \"red\" OR labels.team = \"blue\"", ""}, + {"", "totalStoredBytes"}, + {"", "name sideways"}, + } { + _, err := vaults.List(e.parent).Filter(bad.filter).OrderBy(bad.orderBy).Do() + wantCode(t, "list filter="+bad.filter+" orderBy="+bad.orderBy, err, http.StatusBadRequest) + } +} + +// TestRequestEdgeCases covers the request-shape rejections and the +// body-name id fallback. +func TestRequestEdgeCases(t *testing.T) { + e := newRawEnv(t) + coll := e.parent + "/backupVaults" + + if code, out := e.do(t, http.MethodPost, coll, + `{"name":"`+coll+`/from-body","backupMinimumEnforcedRetentionDuration":"86400s"}`); code != http.StatusOK { + t.Fatalf("create with id from body name = %d %v", code, out) + } + + cases := []struct { + method, path, body string + want int + }{ + {http.MethodPut, coll, "", http.StatusMethodNotAllowed}, + {http.MethodPut, coll + "/from-body", "", http.StatusMethodNotAllowed}, + {http.MethodPost, coll, vaultBody, http.StatusBadRequest}, + {http.MethodPost, coll + "?backupVaultId=x1y", "{not json", http.StatusBadRequest}, + {http.MethodPost, coll + "?backupVaultId=x1y&validateOnly=maybe", vaultBody, http.StatusBadRequest}, + {http.MethodPatch, coll + "/from-body?updateMask=description&validateOnly=maybe", "{}", http.StatusBadRequest}, + {http.MethodPatch, coll + "/from-body?updateMask=description", "{bad", http.StatusBadRequest}, + {http.MethodDelete, coll + "/from-body?force=maybe", "", http.StatusBadRequest}, + {http.MethodGet, coll + "?pageToken=garbage", "", http.StatusBadRequest}, + {http.MethodGet, coll + "?pageSize=100000", "", http.StatusOK}, + } + + for _, tc := range cases { + if code, out := e.do(t, tc.method, tc.path, tc.body); code != tc.want { + t.Fatalf("%s %s = %d %v, want %d", tc.method, tc.path, code, out, tc.want) + } + } +} diff --git a/server/gcp/backupdr/sdk_roundtrip_test.go b/server/gcp/backupdr/sdk_roundtrip_test.go index a2113e55c..b540df3cb 100644 --- a/server/gcp/backupdr/sdk_roundtrip_test.go +++ b/server/gcp/backupdr/sdk_roundtrip_test.go @@ -18,6 +18,7 @@ import ( "github.com/stackshy/cloudemu/v2/config" gcpprovider "github.com/stackshy/cloudemu/v2/providers/gcp" gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" + bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" ) const ( @@ -295,9 +296,14 @@ func TestSDKBackupVaultValidateOnly(t *testing.T) { t.Fatalf("validateOnly create: %v", err) } - e.wait(t, op) + // validateOnly mutates nothing, so it mints no pollable operation: the + // reply is done with the would-be vault inline and no name. + if !op.Done || op.Name != "" || len(op.Response) == 0 { + t.Fatalf("validateOnly create op = %+v, want done, unnamed, with a response", op) + } _, err = vaults.Get(e.parent + "/backupVaults/dry-run").Do() + wantCode(t, "get after validateOnly create", err, http.StatusNotFound) _, err = vaults.Create(e.parent, &backupdr.BackupVault{}).BackupVaultId("dry-run").ValidateOnly(true).Do() @@ -313,9 +319,7 @@ func TestSDKBackupVaultDeleteGuards(t *testing.T) { e.create(t, "vault-full", &backupdr.BackupVault{BackupMinimumEnforcedRetentionDuration: retention}) - if err := e.cloud.BackupDR.SetUsage(sdkProject, sdkLocation, "vault-full", 2, 4096); err != nil { - t.Fatalf("SetUsage: %v", err) - } + e.seedUsage(t, "vault-full", 2, 4096) full := e.get(t, name) if full.Deletable || full.BackupCount != 2 || full.TotalStoredBytes != 4096 { @@ -345,3 +349,50 @@ func TestSDKBackupVaultDeleteGuards(t *testing.T) { e.wait(t, op) } + +// seedUsage makes a vault non-empty through the provider's snapshot/restore +// seam (the emulator has no data plane that could create backups): it +// snapshots the Backup and DR state, sets the vault's backupCount and +// totalStoredBytes, and restores it. +func (e *sdkEnv) seedUsage(t *testing.T, id string, backupCount, totalStoredBytes int64) { + t.Helper() + + ctx := context.Background() + + raw, err := e.cloud.BackupDR.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + var snap map[string]json.RawMessage + if err := json.Unmarshal(raw, &snap); err != nil { + t.Fatalf("decode snapshot: %v", err) + } + + var vaults map[string]bdrdriver.BackupVault + if err := json.Unmarshal(snap["backupVaults"], &vaults); err != nil { + t.Fatalf("decode snapshot vaults: %v", err) + } + + key := e.parent + "/backupVaults/" + id + + v, ok := vaults[key] + if !ok { + t.Fatalf("seedUsage: vault %s not in snapshot", key) + } + + v.BackupCount, v.TotalStoredBytes = backupCount, totalStoredBytes + vaults[key] = v + + if snap["backupVaults"], err = json.Marshal(vaults); err != nil { + t.Fatalf("encode vaults: %v", err) + } + + if raw, err = json.Marshal(snap); err != nil { + t.Fatalf("encode snapshot: %v", err) + } + + if err := e.cloud.BackupDR.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } +} diff --git a/server/gcp/backupdr/wire.go b/server/gcp/backupdr/wire.go index dbea55e32..9a6983564 100644 --- a/server/gcp/backupdr/wire.go +++ b/server/gcp/backupdr/wire.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strconv" + "strings" "time" cerrors "github.com/stackshy/cloudemu/v2/errors" @@ -13,6 +14,16 @@ import ( bdrdriver "github.com/stackshy/cloudemu/v2/services/backupdr/driver" ) +// emptyTypeURL is the Any type of a delete operation's response. The GAPIC +// client's DeleteBackupVaultOperation.Wait fails on a done operation with no +// response ("unsupported result type "), so every delete replays it. +const emptyTypeURL = "type.googleapis.com/google.protobuf.Empty" + +// emptyResponse is google.protobuf.Empty wrapped as an Any. +// +//nolint:gochecknoglobals // immutable wire constant +var emptyResponse = json.RawMessage(`{"@type":"` + emptyTypeURL + `"}`) + // maxBodyBytes caps a decoded request body. const maxBodyBytes = 8 << 20 @@ -101,10 +112,11 @@ type listJSON struct { } // operationJSON mirrors google.longrunning.Operation. Mutating ops complete -// inline, so `done` is always true; `response` carries the resulting vault (an -// Any for create/patch, absent for delete). +// inline, so `done` is always true; `response` carries the result as an Any: +// the vault for create/patch, google.protobuf.Empty for delete. A validateOnly +// request mints no operation, so its name is empty. type operationJSON struct { - Name string `json:"name"` + Name string `json:"name,omitempty"` Done bool `json:"done"` Response json.RawMessage `json:"response,omitempty"` } @@ -170,35 +182,41 @@ func (h *Handler) writeVaultOperation(w http.ResponseWriter, op *bdrdriver.Opera gcprest.WriteJSON(w, http.StatusOK, h.doneOperation(op.Name, raw)) } +// writeEmptyOperation writes a completed operation whose response is +// google.protobuf.Empty (delete, and polls with no vault to return). +func (h *Handler) writeEmptyOperation(w http.ResponseWriter, op *bdrdriver.Operation) { + gcprest.WriteJSON(w, http.StatusOK, h.doneOperation(op.Name, emptyResponse)) +} + // doneOperation builds a completed google.longrunning.Operation and records it // with the shared LRO poller (a no-op on a nil registry) so a client polling -// the returned name resolves the same done operation (with its response). +// the returned name resolves the same done operation with the same response. +// A validateOnly operation has no name and is not registered: nothing was +// mutated, so there is nothing to poll. func (h *Handler) doneOperation(name string, resp json.RawMessage) operationJSON { - if h.ops != nil { - // A nil RawMessage stored in the registry's `any` would be a non-nil - // interface and replay as "response": null; register a true nil instead. - if resp == nil { - h.ops.Register(name, nil) - } else { - h.ops.Register(name, resp) - } + if h.ops != nil && name != "" { + h.ops.Register(name, resp) } return operationJSON{Name: name, Done: true, Response: resp} } -// writeErr maps a driver error onto the Google JSON error envelope. A stale -// etag is 409 ABORTED; a vault that still holds backups is FAILED_PRECONDITION, -// which the Google REST mapping renders as HTTP 400; everything else follows -// the shared cerrors mapping. +// writeErr maps a driver error onto the Google JSON error envelope, using the +// camelCase errors[].reason tokens every other GCP handler emits (the +// top-level status carries the canonical code). A stale etag is 409 ABORTED, +// with the vault name kept in the message; any other FAILED_PRECONDITION (a +// non-empty vault, a locked retention) is HTTP 400, the Google REST mapping of +// that code; everything else follows the shared cerrors mapping. func writeErr(w http.ResponseWriter, err error) { - msg := cerrors.Message(err) - switch { case errors.Is(err, bdrdriver.ErrEtagMismatch): - gcprest.WriteError(w, http.StatusConflict, "ABORTED", msg) + // The provider wraps the sentinel as `backup vault "": `; + // cerrors.Message would return only the sentinel's text and drop the + // name, so swap just the sentinel's code-prefixed text for its message. + msg := strings.Replace(err.Error(), bdrdriver.ErrEtagMismatch.Error(), bdrdriver.ErrEtagMismatch.Message, 1) + gcprest.WriteError(w, http.StatusConflict, "aborted", msg) case cerrors.IsFailedPrecondition(err): - gcprest.WriteError(w, http.StatusBadRequest, "FAILED_PRECONDITION", msg) + gcprest.WriteError(w, http.StatusBadRequest, "failedPrecondition", cerrors.Message(err)) default: gcprest.WriteCErr(w, err) } diff --git a/server/wire/gcprest/gcprest.go b/server/wire/gcprest/gcprest.go index 2c3615031..1517738a2 100644 --- a/server/wire/gcprest/gcprest.go +++ b/server/wire/gcprest/gcprest.go @@ -224,6 +224,7 @@ const ( codeNotFound = "NOT_FOUND" codeAlreadyExists = "ALREADY_EXISTS" codeFailedPrecondition = "FAILED_PRECONDITION" + codeAborted = "ABORTED" codePermissionDenied = "PERMISSION_DENIED" codeResourceExhausted = "RESOURCE_EXHAUSTED" codeUnimplemented = "UNIMPLEMENTED" @@ -263,6 +264,8 @@ func camelReasonToCode(reason string) string { case "conditionNotMet", "failedPrecondition", "resourceInUseByAnotherResource", "containerNotEmpty", "cnameResourceRecordSetConflict": return codeFailedPrecondition + case "aborted": + return codeAborted case "forbidden": return codePermissionDenied case "rateLimitExceeded": @@ -287,7 +290,8 @@ func isCanonicalCode(s string) bool { //nolint:misspell // google.rpc.Code enum name is CANCELLED (two Ls) case "OK", "CANCELLED", "UNKNOWN", codeInvalidArgument, "DEADLINE_EXCEEDED", codeNotFound, codeAlreadyExists, codePermissionDenied, codeResourceExhausted, - codeFailedPrecondition, "ABORTED", "OUT_OF_RANGE", codeUnimplemented, + codeFailedPrecondition, codeAborted, "OUT_OF_RANGE", codeUnimplemented, + codeInternal, codeUnavailable, "DATA_LOSS", "UNAUTHENTICATED": return true default: diff --git a/services/backupdr/driver/driver.go b/services/backupdr/driver/driver.go index 8af19f52b..d39e9da82 100644 --- a/services/backupdr/driver/driver.go +++ b/services/backupdr/driver/driver.go @@ -13,8 +13,10 @@ // // There is no data plane: no data sources, backups, backup plans, management // servers or restores. A vault therefore always reports backupCount and -// totalStoredBytes of 0 and deletable=true, unless a test seeds usage through -// the provider mock's SetUsage hook to exercise the non-empty delete guard. +// totalStoredBytes of 0 and deletable=true, unless restored from a snapshot +// that carries usage (the provider's tests seed it through a test-only hook to +// exercise the non-empty delete guard). + package driver import ( From 59962690029a0eecd3c153c70aac1576486d0da0 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:33:24 +0530 Subject: [PATCH 10/24] fix(aws-ecs): propagateTags, container-instance and capacity-provider tags, TagResource validation Follow-up to the ARN-keyed tag store becoming the single authority for Describe* include=TAGS. - Service tasks: serviceTaskSpec stamped the service's create-time tags on every task and ignored propagateTags. Tasks now get nothing for NONE (the default), the service's live tags for SERVICE, and the task definition's live tags for TASK_DEFINITION, read at launch so a TagResource/UntagResource before a forced redeploy shows up on the new tasks. The service's tags are now recorded before its first tasks launch. - Container instances: RegisterContainerInstance tags are stored in the tag store and returned by DescribeContainerInstances include=TAGS, reflecting TagResource/UntagResource. - Capacity providers: Create/Describe/Update/DeleteCapacityProvider were "unknown ECS operation". They are now implemented in the provider (driver interface + portable wrapper + wire handler): EC2_AUTOSCALING providers with the documented managedScaling defaults and ranges, MANAGED_INSTANCES providers (cluster-scoped, configuration echoed verbatim), the predefined FARGATE and FARGATE_SPOT in Describe, include=TAGS, the cluster filter, maxResults/nextToken (default page 10), and the documented name rules. FARGATE/FARGATE_SPOT cannot be updated, deleted or tagged; a provider still associated with a cluster or used by a service's strategy cannot be deleted. Capacity providers are included in snapshots. - DescribeServices, DescribeTaskDefinition and DescribeContainerInstances return tags only when include=TAGS is sent, as ECS does. - TagResource/UntagResource/ListTagsForResource resolve the ARN to the stored resource and reject, with InvalidParameterException: an unknown resource or non-ECS ARN, a short-format service ARN (the TagResource reference requires migrating to the long format before tagging), a predefined Fargate provider (write only), more than 50 tags on a resource, and aws:-prefixed keys or values (and removing aws: keys). --- docs/coverage/README.md | 2 +- docs/coverage/aws/README.md | 2 +- docs/coverage/aws/ecs.md | 6 +- docs/coverage/coverage.json | 12 + providers/aws/ecs/capacity_providers.go | 542 +++++++++++++++++ providers/aws/ecs/capacity_providers_test.go | 226 ++++++++ providers/aws/ecs/clone.go | 34 ++ providers/aws/ecs/container_instances.go | 7 +- providers/aws/ecs/ecs.go | 50 +- providers/aws/ecs/services.go | 34 +- providers/aws/ecs/snapshot.go | 23 +- providers/aws/ecs/tags.go | 253 ++++++-- server/aws/ecs/capacity_providers.go | 262 +++++++++ server/aws/ecs/container_instances.go | 14 +- server/aws/ecs/describe_live_tags_test.go | 70 +++ server/aws/ecs/handler.go | 2 +- server/aws/ecs/services.go | 12 +- server/aws/ecs/tag_semantics_test.go | 581 +++++++++++++++++++ server/aws/ecs/taskdefs.go | 12 +- server/aws/ecs/types.go | 2 + services/ecs/driver/driver.go | 63 ++ services/ecs/ecs.go | 54 ++ services/ecs/ecs_test.go | 40 ++ 23 files changed, 2214 insertions(+), 89 deletions(-) create mode 100644 providers/aws/ecs/capacity_providers.go create mode 100644 providers/aws/ecs/capacity_providers_test.go create mode 100644 server/aws/ecs/capacity_providers.go create mode 100644 server/aws/ecs/tag_semantics_test.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index ab223d9f8..674f7c36a 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -80,7 +80,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `digitaltwins` | - | [DigitalTwins](./azure/digitaltwins.md) | - | - | 8 | | `disks` | - | [Disks](./azure/disks.md) | - | - | 7 | | `dns` | [Route53](./aws/route53.md) | [DNS](./azure/dns.md) | [CloudDNS](./gcp/clouddns.md) | - | 16 | -| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 37 | +| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 41 | | `efs` | [EFS](./aws/efs.md) | - | - | - | 27 | | `eks` | [EKS](./aws/eks.md) | - | - | - | 40 | | `elasticsan` | - | [ElasticSan](./azure/elasticsan.md) | - | - | 8 | diff --git a/docs/coverage/aws/README.md b/docs/coverage/aws/README.md index bd47fac4b..73a23f1df 100644 --- a/docs/coverage/aws/README.md +++ b/docs/coverage/aws/README.md @@ -31,7 +31,7 @@ Services cloudemu emulates for AWS, by native name. Back to the [cross-provider | [DynamoDB](./dynamodb.md) | `database` | 24 | | [EC2](./ec2.md) | `compute` | 37 | | [ECR](./ecr.md) | `containerregistry` | 15 | -| [ECS](./ecs.md) | `ecs` | 37 | +| [ECS](./ecs.md) | `ecs` | 41 | | [EFS](./efs.md) | `efs` | 27 | | [EKS](./eks.md) | (provider-native) | 40 | | [ELB](./elb.md) | `loadbalancer` | 19 | diff --git a/docs/coverage/aws/ecs.md b/docs/coverage/aws/ecs.md index c5f96e842..d6713245a 100644 --- a/docs/coverage/aws/ecs.md +++ b/docs/coverage/aws/ecs.md @@ -3,18 +3,21 @@ AWS's `ecs` service · portable interface `driver.ECS` · [AWS index](./README.md) -## Operations (37) +## Operations (41) | Operation | Description | | --- | --- | +| `CreateCapacityProvider` | | | `CreateCluster` | | | `CreateService` | | | `DeleteAccountSetting` | | | `DeleteAttributes` | | +| `DeleteCapacityProvider` | | | `DeleteCluster` | | | `DeleteService` | | | `DeregisterContainerInstance` | | | `DeregisterTaskDefinition` | | +| `DescribeCapacityProviders` | | | `DescribeClusters` | | | `DescribeContainerInstances` | | | `DescribeServices` | | @@ -40,6 +43,7 @@ AWS's `ecs` service · portable interface `driver.ECS` · [AWS index](./README.m | `StopTask` | | | `TagResource` | | | `UntagResource` | | +| `UpdateCapacityProvider` | | | `UpdateCluster` | | | `UpdateClusterSettings` | | | `UpdateContainerInstancesState` | | diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index a6b684c70..af6948983 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -6023,6 +6023,9 @@ "service": "ecs", "interface": "ECS", "operations": [ + { + "name": "CreateCapacityProvider" + }, { "name": "CreateCluster" }, @@ -6035,6 +6038,9 @@ { "name": "DeleteAttributes" }, + { + "name": "DeleteCapacityProvider" + }, { "name": "DeleteCluster" }, @@ -6047,6 +6053,9 @@ { "name": "DeregisterTaskDefinition" }, + { + "name": "DescribeCapacityProviders" + }, { "name": "DescribeClusters" }, @@ -6122,6 +6131,9 @@ { "name": "UntagResource" }, + { + "name": "UpdateCapacityProvider" + }, { "name": "UpdateCluster" }, diff --git a/providers/aws/ecs/capacity_providers.go b/providers/aws/ecs/capacity_providers.go new file mode 100644 index 000000000..0977218aa --- /dev/null +++ b/providers/aws/ecs/capacity_providers.go @@ -0,0 +1,542 @@ +package ecs + +import ( + "context" + "encoding/json" + "regexp" + "sort" + "strings" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/regionctx" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// Capacity-provider types and update states, from the ECS CapacityProvider +// data type. +const ( + cpTypeASG = "EC2_AUTOSCALING" + cpTypeManagedInstances = "MANAGED_INSTANCES" + cpFargate = "FARGATE" + cpFargateSpot = "FARGATE_SPOT" + + // failureMissing is the Failure reason for an id that resolved to nothing. + failureMissing = "MISSING" + + cpUpdateComplete = "UPDATE_COMPLETE" + cpDeleteComplete = "DELETE_COMPLETE" + + msEnabledDisabledDefault = "DISABLED" + + // ManagedScaling documented defaults and bounds. + msDefaultTargetCapacity = 100 + msDefaultMinStep = 1 + msDefaultMaxStep = 10000 + msDefaultWarmup = 300 + msMaxStep = 10000 + msMaxWarmup = 10000 + msMaxTargetCapacity = 100 +) + +// capacityProviderName is the documented name shape: up to 255 letters, +// numbers, underscores and hyphens. +var capacityProviderName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,255}$`) + +// isBuiltinCapacityProvider reports whether name is one of the predefined +// Fargate capacity providers, which cannot be created, updated, deleted or +// tagged. +func isBuiltinCapacityProvider(name string) bool { + return name == cpFargate || name == cpFargateSpot +} + +// capacityProviderNameOf returns the bare name from a capacity-provider name or +// ARN (…:capacity-provider/name). +func capacityProviderNameOf(id string) string { + if i := strings.LastIndex(id, "capacity-provider/"); i >= 0 { + return id[i+len("capacity-provider/"):] + } + + return id +} + +// builtinCapacityProvider renders a predefined Fargate capacity provider. +func (m *Mock) builtinCapacityProvider(region, name string) driver.CapacityProvider { + return driver.CapacityProvider{ + ARN: m.arnIn(region, "capacity-provider/"+name), + Name: name, + Status: statusActive, + Type: name, + } +} + +// CreateCapacityProvider creates an Auto Scaling group (EC2_AUTOSCALING) or +// Managed Instances (MANAGED_INSTANCES) capacity provider. +// +//nolint:gocritic // in is passed by value to satisfy the driver.ECS interface; the copy is cheap for a mock. +func (m *Mock) CreateCapacityProvider( + ctx context.Context, in driver.CreateCapacityProviderInput, +) (*driver.CapacityProvider, error) { + if err := validateCapacityProviderName(in.Name); err != nil { + return nil, err + } + + if err := validateTagSet(in.Tags); err != nil { + return nil, err + } + + cp, err := m.newCapacityProvider(ctx, &in) + if err != nil { + return nil, err + } + + // A deleted (INACTIVE) provider's name may be reused; an ACTIVE one may not. + stored := false + + m.capacityProviders.Update(cp.Name, func(existing *driver.CapacityProvider) *driver.CapacityProvider { + if existing.Status == statusActive { + return existing + } + + stored = true + + return cp + }) + + if !stored && !m.capacityProviders.SetIfAbsent(cp.Name, cp) { + return nil, apiErrf(errors.AlreadyExists, excInvalidParameter, + "The specified capacity provider %q already exists.", in.Name) + } + + m.recordTags(cp.ARN, in.Tags) + + out := m.describeCapacityProvider(cp) + + return &out, nil +} + +// validateCapacityProviderName enforces the documented name rules. +func validateCapacityProviderName(name string) error { + if !capacityProviderName.MatchString(name) { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "The capacity provider name must be 1-255 letters, numbers, underscores or hyphens.") + } + + lower := strings.ToLower(name) + for _, reserved := range []string{"aws", arnServiceECS, "fargate"} { + if strings.HasPrefix(lower, reserved) { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "The capacity provider name can't be prefixed with %q.", reserved) + } + } + + return nil +} + +// newCapacityProvider validates the provider block and builds the stored record. +func (m *Mock) newCapacityProvider( + ctx context.Context, in *driver.CreateCapacityProviderInput, +) (*driver.CapacityProvider, error) { + hasASG := in.AutoScalingGroupProvider != nil + hasMI := len(in.ManagedInstancesProvider) > 0 + + if hasASG == hasMI { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "Exactly one of autoScalingGroupProvider or managedInstancesProvider must be specified.") + } + + cp := &driver.CapacityProvider{ + ARN: m.arnIn(regionctx.RegionOr(ctx, m.opts.Region), "capacity-provider/"+in.Name), + Name: in.Name, + Status: statusActive, + Tags: copyTags(in.Tags), + } + + if hasMI { + // Managed Instances capacity providers are cluster-scoped; the cluster + // is required for them. + cluster, err := m.capacityProviderCluster(in.Cluster, true) + if err != nil { + return nil, err + } + + cp.Type = cpTypeManagedInstances + cp.Cluster = cluster + + cp.ManagedInstancesProvider = append(json.RawMessage(nil), in.ManagedInstancesProvider...) + + return cp, nil + } + + cluster, err := m.capacityProviderCluster(in.Cluster, false) + if err != nil { + return nil, err + } + + asg, err := normalizeASGProvider(in.AutoScalingGroupProvider) + if err != nil { + return nil, err + } + + cp.Type = cpTypeASG + cp.Cluster = cluster + cp.AutoScalingGroupProvider = asg + + return cp, nil +} + +// capacityProviderCluster resolves the optional cluster a capacity provider is +// scoped to, returning its bare name. required makes an empty value an error. +func (m *Mock) capacityProviderCluster(id string, required bool) (string, error) { + if id == "" { + if required { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "cluster is required for a Managed Instances capacity provider.") + } + + return "", nil + } + + name := resolveClusterName(id) + if !m.clusterActive(name) { + return "", apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", name) + } + + return name, nil +} + +// normalizeASGProvider validates a create-time Auto Scaling group provider and +// applies the documented defaults: managed termination protection is off, and +// an omitted managedScaling field takes its documented default value. +func normalizeASGProvider(in *driver.AutoScalingGroupProvider) (*driver.AutoScalingGroupProvider, error) { + if in.AutoScalingGroupARN == "" { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "autoScalingGroupProvider.autoScalingGroupArn is required.") + } + + out := cloneASGProvider(in) + if out.ManagedTerminationProtection == "" { + out.ManagedTerminationProtection = msEnabledDisabledDefault + } + + if out.ManagedScaling != nil { + applyManagedScalingDefaults(out.ManagedScaling) + + if err := validateManagedScaling(out.ManagedScaling); err != nil { + return nil, err + } + } + + return out, nil +} + +func applyManagedScalingDefaults(ms *driver.ManagedScaling) { + defaults := []struct { + dst *(*int) + v int + }{ + {&ms.TargetCapacity, msDefaultTargetCapacity}, + {&ms.MinimumScalingStepSize, msDefaultMinStep}, + {&ms.MaximumScalingStepSize, msDefaultMaxStep}, + {&ms.InstanceWarmupPeriod, msDefaultWarmup}, + } + + for _, d := range defaults { + if *d.dst == nil { + *d.dst = ptrInt(d.v) + } + } +} + +// validateManagedScaling enforces the documented ManagedScaling ranges on the +// fields that are set. +func validateManagedScaling(ms *driver.ManagedScaling) error { + checks := []struct { + name string + v *int + min, max int + }{ + {"targetCapacity", ms.TargetCapacity, 1, msMaxTargetCapacity}, + {"minimumScalingStepSize", ms.MinimumScalingStepSize, 1, msMaxStep}, + {"maximumScalingStepSize", ms.MaximumScalingStepSize, 1, msMaxStep}, + {"instanceWarmupPeriod", ms.InstanceWarmupPeriod, 0, msMaxWarmup}, + } + + for _, c := range checks { + if c.v != nil && (*c.v < c.min || *c.v > c.max) { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "managedScaling.%s must be between %d and %d.", c.name, c.min, c.max) + } + } + + if ms.Status != "" && ms.Status != "ENABLED" && ms.Status != "DISABLED" { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "managedScaling.status must be ENABLED or DISABLED.") + } + + return nil +} + +// DescribeCapacityProviders resolves capacity providers by name or ARN. With no +// ids it returns every provider, starting with the predefined FARGATE and +// FARGATE_SPOT. A cluster narrows the result to the providers associated with +// that cluster (through PutClusterCapacityProviders or, for a cluster-scoped +// provider, its own cluster). +func (m *Mock) DescribeCapacityProviders(ctx context.Context, cluster string, ids []string) ( + []driver.CapacityProvider, []driver.Failure, error, +) { + region := regionctx.RegionOr(ctx, m.opts.Region) + + inCluster := func(string, string) bool { return true } + + if cluster != "" { + name := resolveClusterName(cluster) + + c, ok := m.clusters.Get(name) + if !ok { + return nil, nil, apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", name) + } + + associated := make(map[string]bool, len(c.CapacityProviders)) + for _, p := range c.CapacityProviders { + associated[p] = true + } + + inCluster = func(cpName, cpCluster string) bool { return associated[cpName] || cpCluster == name } + } + + if len(ids) == 0 { + return m.allCapacityProviders(region, inCluster), nil, nil + } + + found := make([]driver.CapacityProvider, 0, len(ids)) + failures := make([]driver.Failure, 0, len(ids)) + + for _, id := range ids { + cp, ok := m.resolveCapacityProvider(region, id) + if !ok || !inCluster(cp.Name, cp.Cluster) { + failures = append(failures, driver.Failure{ARN: id, Reason: failureMissing}) + continue + } + + found = append(found, cp) + } + + return found, failures, nil +} + +// allCapacityProviders lists the predefined Fargate providers followed by the +// stored ones in name order, filtered by keep. +func (m *Mock) allCapacityProviders(region string, keep func(name, cluster string) bool) []driver.CapacityProvider { + out := make([]driver.CapacityProvider, 0, m.capacityProviders.Len()+2) //nolint:mnd // the two Fargate providers + + for _, name := range []string{cpFargate, cpFargateSpot} { + if keep(name, "") { + out = append(out, m.builtinCapacityProvider(region, name)) + } + } + + stored := m.capacityProviders.SortedValues() + sort.SliceStable(stored, func(i, j int) bool { return stored[i].Name < stored[j].Name }) + + for _, cp := range stored { + if keep(cp.Name, cp.Cluster) { + out = append(out, m.describeCapacityProvider(cp)) + } + } + + return out +} + +// resolveCapacityProvider looks a provider up by name or ARN, including the +// predefined Fargate providers, and returns it described (live tags applied). +func (m *Mock) resolveCapacityProvider(region, id string) (driver.CapacityProvider, bool) { + name := capacityProviderNameOf(id) + if isBuiltinCapacityProvider(name) { + return m.builtinCapacityProvider(region, name), true + } + + cp, ok := m.capacityProviders.Get(name) + if !ok { + return driver.CapacityProvider{}, false + } + + return m.describeCapacityProvider(cp), true +} + +// describeCapacityProvider deep-copies a stored provider and overlays its live +// ARN-keyed tags. +func (m *Mock) describeCapacityProvider(cp *driver.CapacityProvider) driver.CapacityProvider { + out := cloneCapacityProvider(cp) + out.Tags = m.liveTags(cp.ARN, cp.Tags) + + return out +} + +// UpdateCapacityProvider modifies a provider's Auto Scaling group settings or +// replaces its Managed Instances configuration. The predefined Fargate +// providers cannot be updated. +func (m *Mock) UpdateCapacityProvider( + _ context.Context, in driver.UpdateCapacityProviderInput, +) (*driver.CapacityProvider, error) { + name := capacityProviderNameOf(in.Name) + if isBuiltinCapacityProvider(name) { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "The %s capacity provider is reserved and can't be updated.", name) + } + + var ( + updated driver.CapacityProvider + uerr error + ) + + ok := m.capacityProviders.Update(name, func(cp *driver.CapacityProvider) *driver.CapacityProvider { + if cp.Status != statusActive { + uerr = capacityProviderNotFound(in.Name) + return cp + } + + updated = cloneCapacityProvider(cp) + if uerr = applyCapacityProviderUpdate(&updated, &in); uerr != nil { + return cp + } + + updated.UpdateStatus = cpUpdateComplete + + return &updated + }) + if !ok { + return nil, capacityProviderNotFound(in.Name) + } + + if uerr != nil { + return nil, uerr + } + + out := m.describeCapacityProvider(&updated) + + return &out, nil +} + +// applyCapacityProviderUpdate merges an update into a cloned provider. An +// Auto Scaling group block only applies to an EC2_AUTOSCALING provider and a +// Managed Instances block only to a MANAGED_INSTANCES one. +func applyCapacityProviderUpdate(cp *driver.CapacityProvider, in *driver.UpdateCapacityProviderInput) error { + if in.AutoScalingGroupProvider != nil { + if cp.Type != cpTypeASG { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "autoScalingGroupProvider can only be updated on an %s capacity provider.", cpTypeASG) + } + + if err := mergeASGUpdate(cp.AutoScalingGroupProvider, in.AutoScalingGroupProvider); err != nil { + return err + } + } + + if len(in.ManagedInstancesProvider) > 0 { + if cp.Type != cpTypeManagedInstances { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "managedInstancesProvider can only be updated on a %s capacity provider.", cpTypeManagedInstances) + } + + cp.ManagedInstancesProvider = append(json.RawMessage(nil), in.ManagedInstancesProvider...) + } + + return nil +} + +// mergeASGUpdate applies the set fields of an AutoScalingGroupProviderUpdate +// onto the stored provider. The Auto Scaling group itself cannot be changed. +func mergeASGUpdate(dst, upd *driver.AutoScalingGroupProvider) error { + if upd.ManagedTerminationProtection != "" { + dst.ManagedTerminationProtection = upd.ManagedTerminationProtection + } + + if upd.ManagedDraining != "" { + dst.ManagedDraining = upd.ManagedDraining + } + + if upd.ManagedScaling == nil { + return nil + } + + ms := cloneManagedScaling(upd.ManagedScaling) + applyManagedScalingDefaults(ms) + + if err := validateManagedScaling(ms); err != nil { + return err + } + + dst.ManagedScaling = ms + + return nil +} + +// DeleteCapacityProvider deletes a capacity provider. The predefined Fargate +// providers are reserved, and a provider still associated with a cluster or +// referenced by a service's strategy must be disassociated first. The deletion +// is synchronous: the provider is returned INACTIVE / DELETE_COMPLETE, and its +// tags are deleted with it. +func (m *Mock) DeleteCapacityProvider(_ context.Context, _, capacityProvider string) (*driver.CapacityProvider, error) { + name := capacityProviderNameOf(capacityProvider) + if isBuiltinCapacityProvider(name) { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "The %s capacity provider is reserved and can't be deleted.", name) + } + + cp, ok := m.capacityProviders.Get(name) + if !ok || cp.Status != statusActive { + return nil, capacityProviderNotFound(capacityProvider) + } + + if err := m.checkCapacityProviderUnused(name); err != nil { + return nil, err + } + + deleted := cloneCapacityProvider(cp) + deleted.Status = statusInactive + deleted.UpdateStatus = cpDeleteComplete + deleted.Tags = m.liveTags(cp.ARN, cp.Tags) + + stored := cloneCapacityProvider(&deleted) + m.capacityProviders.Set(name, &stored) + m.tags.Delete(cp.ARN) + + return &deleted, nil +} + +// checkCapacityProviderUnused refuses a delete while an ACTIVE cluster lists +// the provider or an ACTIVE service's strategy references it. +func (m *Mock) checkCapacityProviderUnused(name string) error { + for _, c := range m.clusters.All() { + if c.Status != statusActive { + continue + } + + for _, p := range c.CapacityProviders { + if p == name { + return apiErrf(errors.FailedPrecondition, excInvalidParameter, + "The capacity provider %q is associated with cluster %q. "+ + "Remove it with PutClusterCapacityProviders before deleting it.", name, c.Name) + } + } + } + + for _, s := range m.services.All() { + if s.Status != statusActive { + continue + } + + for _, item := range s.CapacityProviderStrategy { + if item.CapacityProvider == name { + return apiErrf(errors.FailedPrecondition, excInvalidParameter, + "The capacity provider %q is in use by service %q.", name, s.Name) + } + } + } + + return nil +} + +func capacityProviderNotFound(id string) error { + return apiErrf(errors.NotFound, excClient, "The specified capacity provider %q does not exist.", id) +} diff --git a/providers/aws/ecs/capacity_providers_test.go b/providers/aws/ecs/capacity_providers_test.go new file mode 100644 index 000000000..5e3cf40bd --- /dev/null +++ b/providers/aws/ecs/capacity_providers_test.go @@ -0,0 +1,226 @@ +package ecs + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +func asgInput(name string) driver.CreateCapacityProviderInput { + return driver.CreateCapacityProviderInput{ + Name: name, + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{AutoScalingGroupARN: "asg-" + name}, + } +} + +func TestCreateCapacityProviderValidation(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + bad := []struct { + name string + in driver.CreateCapacityProviderInput + }{ + {"empty name", asgInput("")}, + {"bad characters", asgInput("has space")}, + {"aws prefix", asgInput("AWSthing")}, + {"ecs prefix", asgInput("ecs-x")}, + {"no provider block", driver.CreateCapacityProviderInput{Name: "none"}}, + {"both provider blocks", driver.CreateCapacityProviderInput{ + Name: "both", + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{AutoScalingGroupARN: "a"}, + ManagedInstancesProvider: json.RawMessage(`{}`), + }}, + {"asg without arn", driver.CreateCapacityProviderInput{ + Name: "noarn", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{}, + }}, + {"target capacity out of range", driver.CreateCapacityProviderInput{ + Name: "range", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ + AutoScalingGroupARN: "a", ManagedScaling: &driver.ManagedScaling{TargetCapacity: ptrInt(101)}, + }, + }}, + {"bad scaling status", driver.CreateCapacityProviderInput{ + Name: "status", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ + AutoScalingGroupARN: "a", ManagedScaling: &driver.ManagedScaling{Status: "ON"}, + }, + }}, + {"managed instances without cluster", driver.CreateCapacityProviderInput{ + Name: "mi", ManagedInstancesProvider: json.RawMessage(`{"infrastructureRoleArn":"r"}`), + }}, + {"reserved tag key", driver.CreateCapacityProviderInput{ + Name: "tagged", + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{AutoScalingGroupARN: "a"}, + Tags: []driver.Tag{{Key: "aws:x", Value: "1"}}, + }}, + } + + for _, tc := range bad { + _, err := m.CreateCapacityProvider(ctx, tc.in) + assert.True(t, errors.IsInvalidArgument(err), "%s: err = %v", tc.name, err) + } + + _, err = m.CreateCapacityProvider(ctx, driver.CreateCapacityProviderInput{ + Name: "mi", Cluster: "ghost", ManagedInstancesProvider: json.RawMessage(`{}`), + }) + assert.True(t, errors.IsNotFound(err), "unknown cluster: %v", err) + + _, err = m.CreateCapacityProvider(ctx, driver.CreateCapacityProviderInput{ + Name: "asg-ghost", Cluster: "ghost", + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{AutoScalingGroupARN: "a"}, + }) + assert.True(t, errors.IsNotFound(err), "unknown cluster (asg): %v", err) +} + +func TestManagedInstancesCapacityProvider(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + cp, err := m.CreateCapacityProvider(ctx, driver.CreateCapacityProviderInput{ + Name: "mi", Cluster: "prod", ManagedInstancesProvider: json.RawMessage(`{"infrastructureRoleArn":"r1"}`), + }) + require.NoError(t, err) + assert.Equal(t, cpTypeManagedInstances, cp.Type) + assert.Equal(t, "prod", cp.Cluster) + + // Cluster scoping: visible through its own cluster without association. + found, _, err := m.DescribeCapacityProviders(ctx, "prod", nil) + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "mi", found[0].Name) + + _, _, err = m.DescribeCapacityProviders(ctx, "ghost", nil) + assert.True(t, errors.IsNotFound(err)) + + // An ASG block does not apply to a Managed Instances provider. + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "mi", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ManagedDraining: "ENABLED"}, + }) + assert.True(t, errors.IsInvalidArgument(err)) + + updated, err := m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "mi", ManagedInstancesProvider: json.RawMessage(`{"infrastructureRoleArn":"r2"}`), + }) + require.NoError(t, err) + assert.JSONEq(t, `{"infrastructureRoleArn":"r2"}`, string(updated.ManagedInstancesProvider)) + assert.Equal(t, cpUpdateComplete, updated.UpdateStatus) +} + +func TestUpdateCapacityProviderErrors(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + _, err := m.CreateCapacityProvider(ctx, asgInput("asg")) + require.NoError(t, err) + + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{Name: "FARGATE"}) + assert.True(t, errors.IsInvalidArgument(err)) + + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{Name: "ghost"}) + assert.True(t, errors.IsNotFound(err)) + + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "asg", ManagedInstancesProvider: json.RawMessage(`{}`), + }) + assert.True(t, errors.IsInvalidArgument(err)) + + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "asg", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ + ManagedScaling: &driver.ManagedScaling{InstanceWarmupPeriod: ptrInt(-1)}, + }, + }) + assert.True(t, errors.IsInvalidArgument(err)) + + updated, err := m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "asg", AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ManagedTerminationProtection: "ENABLED"}, + }) + require.NoError(t, err) + assert.Equal(t, "ENABLED", updated.AutoScalingGroupProvider.ManagedTerminationProtection) + + // A deleted provider cannot be updated, and its name can be reused. + _, err = m.DeleteCapacityProvider(ctx, "", "asg") + require.NoError(t, err) + + _, err = m.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{Name: "asg"}) + assert.True(t, errors.IsNotFound(err)) + + _, err = m.DeleteCapacityProvider(ctx, "", "asg") + assert.True(t, errors.IsNotFound(err)) + + again, err := m.CreateCapacityProvider(ctx, asgInput("asg")) + require.NoError(t, err) + assert.Equal(t, statusActive, again.Status) +} + +func TestDeleteCapacityProviderInUseByService(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + _, err := m.CreateCapacityProvider(ctx, asgInput("asg")) + require.NoError(t, err) + + _, err = m.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ + Family: "web", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "app", Image: "nginx", Memory: 128}}, + }) + require.NoError(t, err) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "s", TaskDefinition: "web", + CapacityProviderStrategy: []driver.CapacityProviderStrategyItem{{CapacityProvider: "asg", Weight: 1}}, + }) + require.NoError(t, err) + + _, err = m.DeleteCapacityProvider(ctx, "", "asg") + assert.True(t, errors.IsFailedPrecondition(err), "err = %v", err) +} + +func TestCapacityProviderTagsAndSnapshot(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + in := asgInput("asg") + in.Tags = []driver.Tag{{Key: "team", Value: "a"}} + + cp, err := m.CreateCapacityProvider(ctx, in) + require.NoError(t, err) + + require.NoError(t, m.TagResource(ctx, cp.ARN, []driver.Tag{{Key: "env", Value: "dev"}})) + + // The predefined providers are readable but not taggable. + fargate := m.arn("capacity-provider/FARGATE") + tags, err := m.ListTagsForResource(ctx, fargate) + require.NoError(t, err) + assert.Empty(t, tags) + assert.True(t, errors.IsInvalidArgument(m.TagResource(ctx, fargate, []driver.Tag{{Key: "a", Value: "b"}}))) + + snap, err := m.Snapshot(ctx, false) + require.NoError(t, err) + + restored := newTestMock() + require.NoError(t, restored.Restore(ctx, snap)) + + found, failures, err := restored.DescribeCapacityProviders(ctx, "", []string{cp.ARN}) + require.NoError(t, err) + assert.Empty(t, failures) + require.Len(t, found, 1) + assert.Equal(t, []driver.Tag{{Key: "team", Value: "a"}, {Key: "env", Value: "dev"}}, found[0].Tags) + + // Deleting a provider deletes its tags. + _, err = restored.DeleteCapacityProvider(ctx, "", "asg") + require.NoError(t, err) + + _, err = restored.ListTagsForResource(ctx, cp.ARN) + assert.True(t, errors.IsNotFound(err)) +} diff --git a/providers/aws/ecs/clone.go b/providers/aws/ecs/clone.go index a18b77bbe..fe42c8d2e 100644 --- a/providers/aws/ecs/clone.go +++ b/providers/aws/ecs/clone.go @@ -297,3 +297,37 @@ func cloneIntPtr(in *int) *int { return &v } + +// cloneCapacityProvider deep-copies a capacity provider: its Auto Scaling group +// block (and nested managed scaling), raw Managed Instances configuration, and +// Tags. +func cloneCapacityProvider(cp *driver.CapacityProvider) driver.CapacityProvider { + out := *cp + out.Tags = copyTags(cp.Tags) + out.ManagedInstancesProvider = append(json.RawMessage(nil), cp.ManagedInstancesProvider...) + + if cp.AutoScalingGroupProvider != nil { + out.AutoScalingGroupProvider = cloneASGProvider(cp.AutoScalingGroupProvider) + } + + return out +} + +func cloneASGProvider(in *driver.AutoScalingGroupProvider) *driver.AutoScalingGroupProvider { + out := *in + if in.ManagedScaling != nil { + out.ManagedScaling = cloneManagedScaling(in.ManagedScaling) + } + + return &out +} + +func cloneManagedScaling(in *driver.ManagedScaling) *driver.ManagedScaling { + out := *in + out.TargetCapacity = cloneIntPtr(in.TargetCapacity) + out.MinimumScalingStepSize = cloneIntPtr(in.MinimumScalingStepSize) + out.MaximumScalingStepSize = cloneIntPtr(in.MaximumScalingStepSize) + out.InstanceWarmupPeriod = cloneIntPtr(in.InstanceWarmupPeriod) + + return &out +} diff --git a/providers/aws/ecs/container_instances.go b/providers/aws/ecs/container_instances.go index 5b5b292dc..d78a83a8f 100644 --- a/providers/aws/ecs/container_instances.go +++ b/providers/aws/ecs/container_instances.go @@ -117,9 +117,12 @@ func (m *Mock) RegisterContainerInstance( } ci := m.newInstance(in.Cluster, ec2ID, cpu, memory) + ci.Tags = copyTags(in.Tags) m.instances.Set(ci.ARN, ci) + m.recordTags(ci.ARN, in.Tags) out := *ci + out.Tags = copyTags(ci.Tags) return &out, nil } @@ -338,7 +341,9 @@ func (m *Mock) DescribeContainerInstances(_ context.Context, _ string, ids []str for _, id := range ids { if ci, ok := m.resolveInstance(id); ok { - found = append(found, *ci) + out := *ci + out.Tags = m.liveTags(ci.ARN, ci.Tags) + found = append(found, out) continue } diff --git a/providers/aws/ecs/ecs.go b/providers/aws/ecs/ecs.go index 50b21702b..42edda89e 100644 --- a/providers/aws/ecs/ecs.go +++ b/providers/aws/ecs/ecs.go @@ -41,18 +41,19 @@ const ( // Mock is an in-memory mock implementation of Amazon ECS. type Mock struct { - clusters *memstore.Store[*driver.Cluster] - taskDefs *memstore.Store[*driver.TaskDefinition] // keyed by "family:revision" - tasks *memstore.Store[*driver.Task] // keyed by task ARN - services *memstore.Store[*driver.Service] // keyed by "cluster/name" - instances *memstore.Store[*driver.ContainerInstance] - tags *memstore.Store[[]driver.Tag] // keyed by resource ARN - settings *memstore.Store[*driver.AccountSetting] // keyed by setting name - attributes *memstore.Store[*driver.Attribute] // keyed by targetId + "\x00" + name - opts *config.Options - regMu sync.Mutex // serializes task-definition revision allocation - placeMu sync.Mutex // serializes container-instance capacity reserve/release - clusterMu sync.Mutex // serializes CreateCluster name-reuse compare-and-set + clusters *memstore.Store[*driver.Cluster] + taskDefs *memstore.Store[*driver.TaskDefinition] // keyed by "family:revision" + tasks *memstore.Store[*driver.Task] // keyed by task ARN + services *memstore.Store[*driver.Service] // keyed by "cluster/name" + instances *memstore.Store[*driver.ContainerInstance] + tags *memstore.Store[[]driver.Tag] // keyed by resource ARN + settings *memstore.Store[*driver.AccountSetting] // keyed by setting name + attributes *memstore.Store[*driver.Attribute] // keyed by targetId + "\x00" + name + capacityProviders *memstore.Store[*driver.CapacityProvider] // keyed by name; excludes the predefined FARGATE/FARGATE_SPOT + opts *config.Options + regMu sync.Mutex // serializes task-definition revision allocation + placeMu sync.Mutex // serializes container-instance capacity reserve/release + clusterMu sync.Mutex // serializes CreateCluster name-reuse compare-and-set // reconcileLock serializes reconcileServiceAfterStop per service (see // service_reconcile_lock.go), closing the concurrent-StopTask over-launch @@ -108,18 +109,19 @@ func (m *Mock) SetManagedInstanceLauncher(l ManagedInstanceLauncher) { // New creates a new ECS mock with the given configuration options. func New(opts *config.Options) *Mock { return &Mock{ - clusters: memstore.New[*driver.Cluster](), - taskDefs: memstore.New[*driver.TaskDefinition](), - tasks: memstore.New[*driver.Task](), - services: memstore.New[*driver.Service](), - instances: memstore.New[*driver.ContainerInstance](), - tags: memstore.New[[]driver.Tag](), - settings: memstore.New[*driver.AccountSetting](), - attributes: memstore.New[*driver.Attribute](), - engineHandles: memstore.New[string](), - taskSettle: settle.NewSet(), - reconcileLock: newServiceReconcileLock(), - opts: opts, + clusters: memstore.New[*driver.Cluster](), + taskDefs: memstore.New[*driver.TaskDefinition](), + tasks: memstore.New[*driver.Task](), + services: memstore.New[*driver.Service](), + instances: memstore.New[*driver.ContainerInstance](), + tags: memstore.New[[]driver.Tag](), + settings: memstore.New[*driver.AccountSetting](), + attributes: memstore.New[*driver.Attribute](), + capacityProviders: memstore.New[*driver.CapacityProvider](), + engineHandles: memstore.New[string](), + taskSettle: settle.NewSet(), + reconcileLock: newServiceReconcileLock(), + opts: opts, } } diff --git a/providers/aws/ecs/services.go b/providers/aws/ecs/services.go index 6585d30e5..faf792b62 100644 --- a/providers/aws/ecs/services.go +++ b/providers/aws/ecs/services.go @@ -25,6 +25,11 @@ const ( deployControllerECS = "ECS" + // propagateTags values (NONE, the default, needs no constant: anything + // else propagates nothing). + propagateService = "SERVICE" + propagateTaskDefinition = "TASK_DEFINITION" + // Rolling-update deployment defaults ECS applies when the caller omits a // deploymentConfiguration on a service using the ECS (rolling update) // deployment controller. A REPLICA service defaults to 200/100, a DAEMON @@ -123,9 +128,12 @@ func (m *Mock) CreateService(ctx context.Context, in driver.CreateServiceInput) var events pendingTaskEvents + // Record the service's tags before its first tasks launch: propagateTags + // SERVICE reads them from the tag store, and a re-created service reuses + // its predecessor's ARN, whose stale entry must not leak into its tasks. + m.recordTags(svc.ARN, in.Tags) m.convergeNewService(ctx, svc, td, &events) m.services.Set(serviceKey(cluster, svc.Name), svc) - m.recordTags(svc.ARN, in.Tags) m.publish(ctx, &events) m.emitServiceSteadyState(ctx, svc) @@ -322,8 +330,9 @@ func (m *Mock) converge( } // serviceTaskSpec builds the placement spec for a service's tasks: group links -// the task to the service and startedBy carries the deployment id. -func (*Mock) serviceTaskSpec(svc *driver.Service, td *driver.TaskDefinition, deploymentID string) taskSpec { +// the task to the service and startedBy carries the deployment id. The task's +// tags are the ones the service propagates (see propagatedTaskTags). +func (m *Mock) serviceTaskSpec(svc *driver.Service, td *driver.TaskDefinition, deploymentID string) taskSpec { return taskSpec{ cluster: clusterNameFromARN(svc.ClusterARN), clusterARN: svc.ClusterARN, @@ -333,7 +342,24 @@ func (*Mock) serviceTaskSpec(svc *driver.Service, td *driver.TaskDefinition, dep startedBy: deploymentID, platformVersion: svc.PlatformVersion, netCfg: svc.NetworkConfiguration, - tags: svc.Tags, + tags: m.propagatedTaskTags(svc, td), + } +} + +// propagatedTaskTags returns the tags a service stamps on a task it launches, +// per its propagateTags setting, read when the task is launched: SERVICE copies +// the service's current tags, TASK_DEFINITION the task definition's current +// tags, and NONE (the default, also an empty value) copies nothing. Both reads +// go through the ARN-keyed tag store, so a TagResource/UntagResource on the +// source before a new deployment is reflected in the new tasks. +func (m *Mock) propagatedTaskTags(svc *driver.Service, td *driver.TaskDefinition) []driver.Tag { + switch svc.PropagateTags { + case propagateService: + return m.liveTags(svc.ARN, svc.Tags) + case propagateTaskDefinition: + return m.liveTags(td.ARN, td.Tags) + default: + return nil } } diff --git a/providers/aws/ecs/snapshot.go b/providers/aws/ecs/snapshot.go index 0b12364c0..9044f15a3 100644 --- a/providers/aws/ecs/snapshot.go +++ b/providers/aws/ecs/snapshot.go @@ -21,16 +21,17 @@ var _ snapshot.Snapshottable = (*Mock)(nil) // sub-few-second overlay, so a restored task is observed in its final // state, matching how EC2 excludes its own settle windows. type ecsSnapshot struct { - Clusters json.RawMessage `json:"clusters,omitempty"` - TaskDefs json.RawMessage `json:"taskDefs,omitempty"` - Tasks json.RawMessage `json:"tasks,omitempty"` - Services json.RawMessage `json:"services,omitempty"` - Instances json.RawMessage `json:"instances,omitempty"` - Tags json.RawMessage `json:"tags,omitempty"` - Settings json.RawMessage `json:"settings,omitempty"` - Attributes json.RawMessage `json:"attributes,omitempty"` - EngineHandles json.RawMessage `json:"engineHandles,omitempty"` - PortCounter uint32 `json:"portCounter,omitempty"` + Clusters json.RawMessage `json:"clusters,omitempty"` + TaskDefs json.RawMessage `json:"taskDefs,omitempty"` + Tasks json.RawMessage `json:"tasks,omitempty"` + Services json.RawMessage `json:"services,omitempty"` + Instances json.RawMessage `json:"instances,omitempty"` + Tags json.RawMessage `json:"tags,omitempty"` + Settings json.RawMessage `json:"settings,omitempty"` + Attributes json.RawMessage `json:"attributes,omitempty"` + EngineHandles json.RawMessage `json:"engineHandles,omitempty"` + CapacityProviders json.RawMessage `json:"capacityProviders,omitempty"` + PortCounter uint32 `json:"portCounter,omitempty"` } // Snapshot captures the mock's entire state as JSON. includeAssets is unused. ECS holds no bulk @@ -60,6 +61,7 @@ func (m *Mock) snapshotStores(snap *ecsSnapshot) error { {&snap.Settings, m.settings.Snapshot}, {&snap.Attributes, m.attributes.Snapshot}, {&snap.EngineHandles, m.engineHandles.Snapshot}, + {&snap.CapacityProviders, m.capacityProviders.Snapshot}, } for _, d := range dumps { @@ -105,6 +107,7 @@ func (m *Mock) restoreStores(snap *ecsSnapshot) error { {snap.Settings, m.settings.LoadSnapshot}, {snap.Attributes, m.attributes.LoadSnapshot}, {snap.EngineHandles, m.engineHandles.LoadSnapshot}, + {snap.CapacityProviders, m.capacityProviders.LoadSnapshot}, } for _, l := range loads { diff --git a/providers/aws/ecs/tags.go b/providers/aws/ecs/tags.go index 65a982c68..4cf5fd9e7 100644 --- a/providers/aws/ecs/tags.go +++ b/providers/aws/ecs/tags.go @@ -8,6 +8,14 @@ import ( "github.com/stackshy/cloudemu/v2/services/ecs/driver" ) +// maxUserTags is the ECS per-resource tag limit. Tags whose key starts with +// the reserved aws: prefix do not count against it. +const maxUserTags = 50 + +// reservedTagPrefix is the case-insensitive prefix ECS reserves for AWS use on +// both tag keys and values. +const reservedTagPrefix = "aws:" + // recordTags records a resource's creation-time tags under its ARN so that // ListTagsForResource can return them. A resource with no tags is still // recorded (as an empty slice) so its ARN is recognized as tag-managed. @@ -17,36 +25,65 @@ func (m *Mock) recordTags(arn string, tags []driver.Tag) { // TagResource merges tags onto a resource, replacing the value of any key that // already exists and appending new keys, mirroring AWS's upsert semantics. The +// ARN must name an existing taggable resource (a short-format service ARN, a +// predefined Fargate capacity provider, or an unknown resource is an +// InvalidParameterException), no key or value may carry the reserved aws: +// prefix, and the resource may not end up with more than 50 user tags. The // read-modify-write runs atomically under the store lock (SetIfAbsent seeds the // entry, then Update mutates it in place) so two concurrent tag writes on the // same ARN can't lose one another's changes. func (m *Mock) TagResource(_ context.Context, resourceARN string, tags []driver.Tag) error { - if resourceARN == "" { - return apiErrf(errors.InvalidArgument, excInvalidParameter, "resourceArn is required") + key, err := m.tagKey(resourceARN, true) + if err != nil { + return err } - m.tags.SetIfAbsent(resourceARN, nil) - m.tags.Update(resourceARN, func(existing []driver.Tag) []driver.Tag { - return mergeTags(existing, tags) + if err := validateTagSet(tags); err != nil { + return err + } + + overLimit := false + + m.tags.SetIfAbsent(key, nil) + m.tags.Update(key, func(existing []driver.Tag) []driver.Tag { + merged := mergeTags(existing, tags) + if userTagCount(merged) > maxUserTags { + overLimit = true + return existing + } + + return merged }) + if overLimit { + return tooManyTags() + } + return nil } -// UntagResource removes the given tag keys from a resource. The read-modify-write -// runs atomically under the store lock so it can't race a concurrent TagResource -// on the same ARN. An absent ARN is a no-op (Update returns false, ignored). +// UntagResource removes the given tag keys from a resource. The ARN is +// resolved exactly as TagResource resolves it, and reserved aws: keys cannot be +// removed. The read-modify-write runs atomically under the store lock so it +// can't race a concurrent TagResource on the same ARN. func (m *Mock) UntagResource(_ context.Context, resourceARN string, tagKeys []string) error { - if resourceARN == "" { - return apiErrf(errors.InvalidArgument, excInvalidParameter, "resourceArn is required") + key, err := m.tagKey(resourceARN, true) + if err != nil { + return err } drop := make(map[string]bool, len(tagKeys)) + for _, k := range tagKeys { + if hasReservedPrefix(k) { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "Tag keys with the %q prefix are reserved for AWS use and can't be removed.", reservedTagPrefix) + } + drop[k] = true } - m.tags.Update(resourceARN, func(existing []driver.Tag) []driver.Tag { + m.tags.Update(key, func(existing []driver.Tag) []driver.Tag { kept := make([]driver.Tag, 0, len(existing)) for _, t := range existing { @@ -61,18 +98,18 @@ func (m *Mock) UntagResource(_ context.Context, resourceARN string, tagKeys []st return nil } -// ListTagsForResource returns a resource's tags. An ARN that is neither -// tag-managed nor a resolvable ECS resource surfaces a NotFound error. +// ListTagsForResource returns a resource's tags. The ARN is resolved like +// TagResource resolves it, except that the predefined Fargate capacity +// providers are readable (they carry no tags). func (m *Mock) ListTagsForResource(_ context.Context, resourceARN string) ([]driver.Tag, error) { - if tags, ok := m.tags.Get(resourceARN); ok { - return copyTags(tags), nil + key, err := m.tagKey(resourceARN, false) + if err != nil { + return nil, err } - if m.resourceExists(resourceARN) { - return nil, nil - } + tags, _ := m.tags.Get(key) - return nil, apiErrf(errors.NotFound, excClient, "resource %q not found", resourceARN) + return copyTags(tags), nil } // liveTags returns a resource's current tags. The tag store keyed by ARN is @@ -115,30 +152,162 @@ func mergeTags(base, add []driver.Tag) []driver.Tag { return out } -// resourceExists reports whether an ARN resolves to a live cluster, service, -// task definition, task, or container instance. -func (m *Mock) resourceExists(arn string) bool { - switch { - case strings.Contains(arn, "cluster/"): - return m.clusterExists(resolveClusterName(arn)) - case strings.Contains(arn, "task-definition/"): - _, ok := m.resolveTaskDef(arn) - return ok - case strings.Contains(arn, "container-instance/"): - _, ok := m.resolveInstance(arn) - return ok - case strings.Contains(arn, "task/"): - _, ok := m.resolveTask(arn) - return ok - case strings.Contains(arn, "service/"): - for _, s := range m.services.All() { - if s.ARN == arn { - return true - } +// validateTagSet enforces the per-request tag rules: at most 50 tags, and no +// key or value carrying the reserved aws: prefix (in any letter case). +func validateTagSet(tags []driver.Tag) error { + if len(tags) > maxUserTags { + return tooManyTags() + } + + for _, t := range tags { + if hasReservedPrefix(t.Key) || hasReservedPrefix(t.Value) { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "Tag keys and values can't start with %q; the prefix is reserved for AWS use.", reservedTagPrefix) } + } - return false - default: - return false + return nil +} + +func tooManyTags() error { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "A resource can have at most %d tags.", maxUserTags) +} + +func hasReservedPrefix(s string) bool { + return len(s) >= len(reservedTagPrefix) && strings.EqualFold(s[:len(reservedTagPrefix)], reservedTagPrefix) +} + +// userTagCount counts the tags that count against the per-resource limit +// (reserved aws: keys are exempt). +func userTagCount(tags []driver.Tag) int { + n := 0 + + for _, t := range tags { + if !hasReservedPrefix(t.Key) { + n++ + } + } + + return n +} + +// tagKey resolves a resource ARN to the ARN its tags are stored under: the +// resource's own stored ARN, so a lookup by any accepted spelling reads and +// writes the same entry that Describe* reads. forWrite additionally rejects +// the predefined Fargate capacity providers, which cannot be tagged. +func (m *Mock) tagKey(resourceARN string, forWrite bool) (string, error) { + resourceType, rest, ok := splitECSARN(resourceARN) + if !ok { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "The ARN %q is not a valid Amazon ECS resource ARN.", resourceARN) + } + + if resourceType == arnTypeCapacityProvider && isBuiltinCapacityProvider(rest) && forWrite { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "The predefined %s capacity provider can't be tagged.", rest) } + + if resourceType == arnTypeService && !strings.Contains(rest, "/") { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "The service ARN %q uses the short ARN format. Migrate the service to the long ARN format "+ + "(service/cluster-name/service-name) to tag it.", resourceARN) + } + + if key, found := m.storedARN(resourceARN, resourceType, rest); found { + return key, nil + } + + return "", apiErrf(errors.NotFound, excInvalidParameter, + "The specified resource %q does not exist.", resourceARN) +} + +// ECS ARN pieces: the service namespace and the resource types TagResource +// accepts. +const ( + arnServiceECS = "ecs" + arnTypeCluster = "cluster" + arnTypeService = "service" + arnTypeTaskDefinition = "task-definition" + arnTypeTask = "task" + arnTypeContainerInst = "container-instance" + arnTypeCapacityProvider = "capacity-provider" +) + +// storedARN returns the stored ARN of the resource an ECS ARN names. +func (m *Mock) storedARN(resourceARN, resourceType, rest string) (string, bool) { + resolvers := map[string]func() (string, bool){ + arnTypeCluster: func() (string, bool) { + if c, ok := m.clusters.Get(rest); ok { + return c.ARN, true + } + + // The implicit default cluster exists even when never created. + return resourceARN, rest == defaultCluster + }, + arnTypeService: func() (string, bool) { + cluster, name, _ := strings.Cut(rest, "/") + s, ok := m.resolveService(cluster, name) + + return arnOf(s, ok, func(s *driver.Service) string { return s.ARN }) + }, + arnTypeTaskDefinition: func() (string, bool) { + td, ok := m.resolveTaskDef(resourceARN) + + return arnOf(td, ok, func(td *driver.TaskDefinition) string { return td.ARN }) + }, + arnTypeTask: func() (string, bool) { + t, ok := m.resolveTask(resourceARN) + + return arnOf(t, ok, func(t *driver.Task) string { return t.ARN }) + }, + arnTypeContainerInst: func() (string, bool) { + ci, ok := m.resolveInstance(resourceARN) + + return arnOf(ci, ok, func(ci *driver.ContainerInstance) string { return ci.ARN }) + }, + arnTypeCapacityProvider: func() (string, bool) { + if isBuiltinCapacityProvider(rest) { + return resourceARN, true + } + + cp, ok := m.capacityProviders.Get(rest) + + return arnOf(cp, ok && cp.Status == statusActive, func(cp *driver.CapacityProvider) string { return cp.ARN }) + }, + } + + resolve, known := resolvers[resourceType] + if !known { + return "", false + } + + return resolve() +} + +// arnOf returns arn(v) when ok, so each resolver reads as one lookup. +func arnOf[T any](v T, ok bool, arn func(T) string) (string, bool) { + if !ok { + return "", false + } + + return arn(v), true +} + +// splitECSARN splits arn::ecs:::/ into +// its resource type and the remainder after the first slash. +func splitECSARN(arn string) (resourceType, rest string, ok bool) { + const fields, serviceField, resourceField = 6, 2, 5 + + parts := strings.SplitN(arn, ":", fields) + if len(parts) != fields || parts[0] != "arn" || parts[serviceField] != arnServiceECS { + return "", "", false + } + + resourceType, rest, ok = strings.Cut(parts[resourceField], "/") + if !ok || rest == "" { + return "", "", false + } + + return resourceType, rest, true } diff --git a/server/aws/ecs/capacity_providers.go b/server/aws/ecs/capacity_providers.go new file mode 100644 index 000000000..691c9000f --- /dev/null +++ b/server/aws/ecs/capacity_providers.go @@ -0,0 +1,262 @@ +package ecs + +import ( + "encoding/json" + "net/http" + + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/server/wire" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// describeCapacityProvidersDefaultPage is the page size DescribeCapacityProviders +// uses when maxResults is omitted (the documented default and maximum is 10). +const describeCapacityProvidersDefaultPage = 10 + +// Response body keys shared by the capacity-provider operations. +const ( + keyCapacityProvider = "capacityProvider" + keyFailures = "failures" +) + +type wireManagedScaling struct { + Status string `json:"status,omitempty"` + TargetCapacity *int `json:"targetCapacity,omitempty"` + MinimumScalingStepSize *int `json:"minimumScalingStepSize,omitempty"` + MaximumScalingStepSize *int `json:"maximumScalingStepSize,omitempty"` + InstanceWarmupPeriod *int `json:"instanceWarmupPeriod,omitempty"` +} + +type wireAutoScalingGroupProvider struct { + AutoScalingGroupArn string `json:"autoScalingGroupArn,omitempty"` + ManagedScaling *wireManagedScaling `json:"managedScaling,omitempty"` + ManagedTerminationProtection string `json:"managedTerminationProtection,omitempty"` + ManagedDraining string `json:"managedDraining,omitempty"` +} + +type wireCapacityProvider struct { + CapacityProviderArn string `json:"capacityProviderArn"` + Name string `json:"name"` + Status string `json:"status"` + Type string `json:"type,omitempty"` + Cluster string `json:"cluster,omitempty"` + UpdateStatus string `json:"updateStatus,omitempty"` + UpdateStatusReason string `json:"updateStatusReason,omitempty"` + AutoScalingGroupProvider *wireAutoScalingGroupProvider `json:"autoScalingGroupProvider,omitempty"` + ManagedInstancesProvider json.RawMessage `json:"managedInstancesProvider,omitempty"` + Tags []wireTag `json:"tags,omitempty"` +} + +func (h *Handler) routeCapacityProviders(w http.ResponseWriter, r *http.Request, op string) bool { + switch op { + case "CreateCapacityProvider": + h.createCapacityProvider(w, r) + case "DescribeCapacityProviders": + h.describeCapacityProviders(w, r) + case "UpdateCapacityProvider": + h.updateCapacityProvider(w, r) + case "DeleteCapacityProvider": + h.deleteCapacityProvider(w, r) + default: + return false + } + + return true +} + +func (h *Handler) createCapacityProvider(w http.ResponseWriter, r *http.Request) { + var req struct { + Name string `json:"name"` + Cluster string `json:"cluster"` + AutoScalingGroupProvider *wireAutoScalingGroupProvider `json:"autoScalingGroupProvider"` + ManagedInstancesProvider json.RawMessage `json:"managedInstancesProvider"` + Tags []wireTag `json:"tags"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + cp, err := h.ecs.CreateCapacityProvider(r.Context(), driver.CreateCapacityProviderInput{ + Name: req.Name, + Cluster: req.Cluster, + AutoScalingGroupProvider: toASGProvider(req.AutoScalingGroupProvider), + ManagedInstancesProvider: nonNullRaw(req.ManagedInstancesProvider), + Tags: toTags(req.Tags), + }) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{keyCapacityProvider: capacityProviderToWire(cp)}) +} + +func (h *Handler) describeCapacityProviders(w http.ResponseWriter, r *http.Request) { + var req struct { + CapacityProviders []string `json:"capacityProviders"` + Cluster string `json:"cluster"` + Include []string `json:"include"` + MaxResults int `json:"maxResults"` + NextToken string `json:"nextToken"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + providers, failures, err := h.ecs.DescribeCapacityProviders(r.Context(), req.Cluster, req.CapacityProviders) + if err != nil { + writeErr(w, err) + + return + } + + maxResults := req.MaxResults + if maxResults <= 0 || maxResults > describeCapacityProvidersDefaultPage { + maxResults = describeCapacityProvidersDefaultPage + } + + page, err := pagination.Paginate(providers, req.NextToken, maxResults) + if err != nil { + wire.WriteJSONError(w, http.StatusBadRequest, "InvalidParameterException", "invalid nextToken: "+err.Error()) + + return + } + + // Tags are only returned when the caller opts in via include=TAGS. + wantTags := includes(req.Include, "TAGS") + + out := make([]wireCapacityProvider, 0, len(page.Items)) + + for i := range page.Items { + wcp := capacityProviderToWire(&page.Items[i]) + if !wantTags { + wcp.Tags = nil + } + + out = append(out, wcp) + } + + resp := map[string]any{"capacityProviders": out, keyFailures: fromFailures(failures)} + if page.NextPageToken != "" { + resp["nextToken"] = page.NextPageToken + } + + wire.WriteJSON(w, resp) +} + +func (h *Handler) updateCapacityProvider(w http.ResponseWriter, r *http.Request) { + var req struct { + Name string `json:"name"` + Cluster string `json:"cluster"` + AutoScalingGroupProvider *wireAutoScalingGroupProvider `json:"autoScalingGroupProvider"` + ManagedInstancesProvider json.RawMessage `json:"managedInstancesProvider"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + cp, err := h.ecs.UpdateCapacityProvider(r.Context(), driver.UpdateCapacityProviderInput{ + Name: req.Name, + Cluster: req.Cluster, + AutoScalingGroupProvider: toASGProvider(req.AutoScalingGroupProvider), + ManagedInstancesProvider: nonNullRaw(req.ManagedInstancesProvider), + }) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{keyCapacityProvider: capacityProviderToWire(cp)}) +} + +func (h *Handler) deleteCapacityProvider(w http.ResponseWriter, r *http.Request) { + var req struct { + CapacityProvider string `json:"capacityProvider"` + Cluster string `json:"cluster"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + cp, err := h.ecs.DeleteCapacityProvider(r.Context(), req.Cluster, req.CapacityProvider) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{keyCapacityProvider: capacityProviderToWire(cp)}) +} + +// nonNullRaw drops an absent or JSON-null raw block so the driver sees "not set". +func nonNullRaw(raw json.RawMessage) json.RawMessage { + if len(raw) == 0 || string(raw) == "null" { + return nil + } + + return raw +} + +func toASGProvider(in *wireAutoScalingGroupProvider) *driver.AutoScalingGroupProvider { + if in == nil { + return nil + } + + out := &driver.AutoScalingGroupProvider{ + AutoScalingGroupARN: in.AutoScalingGroupArn, + ManagedTerminationProtection: in.ManagedTerminationProtection, + ManagedDraining: in.ManagedDraining, + } + + if ms := in.ManagedScaling; ms != nil { + out.ManagedScaling = &driver.ManagedScaling{ + Status: ms.Status, + TargetCapacity: ms.TargetCapacity, + MinimumScalingStepSize: ms.MinimumScalingStepSize, + MaximumScalingStepSize: ms.MaximumScalingStepSize, + InstanceWarmupPeriod: ms.InstanceWarmupPeriod, + } + } + + return out +} + +func capacityProviderToWire(cp *driver.CapacityProvider) wireCapacityProvider { + out := wireCapacityProvider{ + CapacityProviderArn: cp.ARN, + Name: cp.Name, + Status: cp.Status, + Type: cp.Type, + Cluster: cp.Cluster, + UpdateStatus: cp.UpdateStatus, + UpdateStatusReason: cp.UpdateStatusReason, + ManagedInstancesProvider: cp.ManagedInstancesProvider, + Tags: fromTags(cp.Tags), + } + + if asg := cp.AutoScalingGroupProvider; asg != nil { + out.AutoScalingGroupProvider = &wireAutoScalingGroupProvider{ + AutoScalingGroupArn: asg.AutoScalingGroupARN, + ManagedTerminationProtection: asg.ManagedTerminationProtection, + ManagedDraining: asg.ManagedDraining, + } + + if ms := asg.ManagedScaling; ms != nil { + out.AutoScalingGroupProvider.ManagedScaling = &wireManagedScaling{ + Status: ms.Status, + TargetCapacity: ms.TargetCapacity, + MinimumScalingStepSize: ms.MinimumScalingStepSize, + MaximumScalingStepSize: ms.MaximumScalingStepSize, + InstanceWarmupPeriod: ms.InstanceWarmupPeriod, + } + } + } + + return out +} diff --git a/server/aws/ecs/container_instances.go b/server/aws/ecs/container_instances.go index f9a7c9e47..f9cf92695 100644 --- a/server/aws/ecs/container_instances.go +++ b/server/aws/ecs/container_instances.go @@ -32,6 +32,7 @@ func (h *Handler) registerContainerInstance(w http.ResponseWriter, r *http.Reque InstanceIdentityDocument string `json:"instanceIdentityDocument"` TotalResources []wireResource `json:"totalResources"` Attributes []wireAttribute `json:"attributes"` + Tags []wireTag `json:"tags"` } if !wire.DecodeJSON(w, r, &req) { @@ -43,6 +44,7 @@ func (h *Handler) registerContainerInstance(w http.ResponseWriter, r *http.Reque InstanceIdentityDocument: req.InstanceIdentityDocument, TotalResources: toResources(req.TotalResources), Attributes: toAttributes(req.Attributes), + Tags: toTags(req.Tags), }) if err != nil { writeErr(w, err) @@ -137,6 +139,7 @@ func (h *Handler) describeContainerInstances(w http.ResponseWriter, r *http.Requ var req struct { ContainerInstances []string `json:"containerInstances"` Cluster string `json:"cluster"` + Include []string `json:"include"` } if !wire.DecodeJSON(w, r, &req) { @@ -150,9 +153,18 @@ func (h *Handler) describeContainerInstances(w http.ResponseWriter, r *http.Requ return } + // Tags are only returned when the caller opts in via include=TAGS. + wantTags := includes(req.Include, "TAGS") + out := make([]wireContainerInstance, 0, len(instances)) + for i := range instances { - out = append(out, instanceToWire(&instances[i])) + wci := instanceToWire(&instances[i]) + if !wantTags { + wci.Tags = nil + } + + out = append(out, wci) } wire.WriteJSON(w, map[string]any{"containerInstances": out, "failures": fromFailures(failures)}) diff --git a/server/aws/ecs/describe_live_tags_test.go b/server/aws/ecs/describe_live_tags_test.go index e64c59cd7..6a8e4d65c 100644 --- a/server/aws/ecs/describe_live_tags_test.go +++ b/server/aws/ecs/describe_live_tags_test.go @@ -80,6 +80,8 @@ func liveTagsCases() []liveTagsCase { {name: "cluster", create: createTaggedCluster, describe: describeClusterTags}, {name: "task-definition", create: createTaggedTaskDef, describe: describeTaskDefTags}, {name: "task", create: createTaggedTask, describe: describeTaskTags}, + {name: "container-instance", create: createTaggedInstance, describe: describeInstanceTags}, + {name: "capacity-provider", create: createTaggedCapacityProvider, describe: describeCapacityProviderTags}, } } @@ -225,3 +227,71 @@ func tagMap(tags []ecstypes.Tag) map[string]string { return out } + +func createTaggedInstance(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.RegisterContainerInstance(ctx, &awsecs.RegisterContainerInstanceInput{ + Cluster: aws.String("prod"), + Tags: tags, + }) + if err != nil { + t.Fatalf("RegisterContainerInstance: %v", err) + } + + return aws.ToString(out.ContainerInstance.ContainerInstanceArn) +} + +func describeInstanceTags(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeContainerInstances(ctx, &awsecs.DescribeContainerInstancesInput{ + Cluster: aws.String("prod"), + ContainerInstances: []string{arn}, + Include: []ecstypes.ContainerInstanceField{ecstypes.ContainerInstanceFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeContainerInstances: %v", err) + } + + if len(out.ContainerInstances) != 1 { + t.Fatalf("DescribeContainerInstances = %d instances, want 1", len(out.ContainerInstances)) + } + + return out.ContainerInstances[0].Tags +} + +func createTaggedCapacityProvider(t *testing.T, client *awsecs.Client, ctx context.Context, tags []ecstypes.Tag) string { + t.Helper() + + out, err := client.CreateCapacityProvider(ctx, &awsecs.CreateCapacityProviderInput{ + Name: aws.String("asg-cp"), + AutoScalingGroupProvider: &ecstypes.AutoScalingGroupProvider{ + AutoScalingGroupArn: aws.String("arn:aws:autoscaling:us-east-1:000000000000:autoScalingGroup:x:autoScalingGroupName/asg"), + }, + Tags: tags, + }) + if err != nil { + t.Fatalf("CreateCapacityProvider: %v", err) + } + + return aws.ToString(out.CapacityProvider.CapacityProviderArn) +} + +func describeCapacityProviderTags(t *testing.T, client *awsecs.Client, ctx context.Context, arn string) []ecstypes.Tag { + t.Helper() + + out, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{ + CapacityProviders: []string{arn}, + Include: []ecstypes.CapacityProviderField{ecstypes.CapacityProviderFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeCapacityProviders: %v", err) + } + + if len(out.CapacityProviders) != 1 { + t.Fatalf("DescribeCapacityProviders = %d providers, want 1", len(out.CapacityProviders)) + } + + return out.CapacityProviders[0].Tags +} diff --git a/server/aws/ecs/handler.go b/server/aws/ecs/handler.go index d45f04469..ddb08330e 100644 --- a/server/aws/ecs/handler.go +++ b/server/aws/ecs/handler.go @@ -44,7 +44,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { routers := []func(http.ResponseWriter, *http.Request, string) bool{ h.routeClusters, h.routeTaskDefs, h.routeTasks, h.routeServices, h.routeContainerInstances, - h.routeTags, h.routeAccount, h.routeAttributes, + h.routeTags, h.routeAccount, h.routeAttributes, h.routeCapacityProviders, } for _, route := range routers { if route(w, r, op) { diff --git a/server/aws/ecs/services.go b/server/aws/ecs/services.go index 0e98853d3..0a18b4f24 100644 --- a/server/aws/ecs/services.go +++ b/server/aws/ecs/services.go @@ -188,6 +188,7 @@ func (h *Handler) describeServices(w http.ResponseWriter, r *http.Request) { var req struct { Services []string `json:"services"` Cluster string `json:"cluster"` + Include []string `json:"include"` } if !wire.DecodeJSON(w, r, &req) { @@ -201,9 +202,18 @@ func (h *Handler) describeServices(w http.ResponseWriter, r *http.Request) { return } + // Tags are only returned when the caller opts in via include=TAGS. + wantTags := includes(req.Include, "TAGS") + out := make([]wireService, 0, len(services)) + for i := range services { - out = append(out, serviceToWire(&services[i])) + ws := serviceToWire(&services[i]) + if !wantTags { + ws.Tags = nil + } + + out = append(out, ws) } wire.WriteJSON(w, map[string]any{"services": out, "failures": fromFailures(failures)}) diff --git a/server/aws/ecs/tag_semantics_test.go b/server/aws/ecs/tag_semantics_test.go new file mode 100644 index 000000000..b86e55f84 --- /dev/null +++ b/server/aws/ecs/tag_semantics_test.go @@ -0,0 +1,581 @@ +package ecs_test + +import ( + "context" + "fmt" + "maps" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" + ecstypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" +) + +const testASGArn = "arn:aws:autoscaling:us-east-1:000000000000:autoScalingGroup:x:autoScalingGroupName/asg" + +// serviceTaskTags returns the tags (include=TAGS) of the service's RUNNING +// tasks that belong to its current PRIMARY deployment. +func serviceTaskTags(t *testing.T, client *awsecs.Client, ctx context.Context, service string) []map[string]string { + t.Helper() + + svc, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{ + Cluster: aws.String("prod"), Services: []string{service}, + }) + if err != nil || len(svc.Services) != 1 { + t.Fatalf("DescribeServices: %v", err) + } + + var primary string + + for _, d := range svc.Services[0].Deployments { + if aws.ToString(d.Status) == "PRIMARY" { + primary = aws.ToString(d.Id) + } + } + + list, err := client.ListTasks(ctx, &awsecs.ListTasksInput{ + Cluster: aws.String("prod"), ServiceName: aws.String(service), DesiredStatus: ecstypes.DesiredStatusRunning, + }) + if err != nil { + t.Fatalf("ListTasks: %v", err) + } + + desc, err := client.DescribeTasks(ctx, &awsecs.DescribeTasksInput{ + Cluster: aws.String("prod"), Tasks: list.TaskArns, + Include: []ecstypes.TaskField{ecstypes.TaskFieldTags}, + }) + if err != nil { + t.Fatalf("DescribeTasks: %v", err) + } + + var out []map[string]string + + for i := range desc.Tasks { + if aws.ToString(desc.Tasks[i].StartedBy) == primary { + out = append(out, tagMap(desc.Tasks[i].Tags)) + } + } + + if len(out) == 0 { + t.Fatalf("service %s has no running tasks in deployment %s", service, primary) + } + + return out +} + +func sdkTags(kv ...string) []ecstypes.Tag { + out := make([]ecstypes.Tag, 0, len(kv)/2) + for i := 0; i+1 < len(kv); i += 2 { + out = append(out, ecstypes.Tag{Key: aws.String(kv[i]), Value: aws.String(kv[i+1])}) + } + + return out +} + +// TestSDKServicePropagateTags guards that a service's tasks carry exactly the +// tags its propagateTags setting selects, read at launch time from the live tag +// store: NONE (the default) propagates nothing, SERVICE the service's current +// tags, TASK_DEFINITION the task definition's current tags. The service used to +// stamp its create-time tags on every task regardless of the setting, so a +// forced redeploy after an untag still launched tasks with the removed tag. +func TestSDKServicePropagateTags(t *testing.T) { + cases := []struct { + name string + propagate ecstypes.PropagateTags + // mutate retags the propagation source between the two deployments. + mutate func(t *testing.T, client *awsecs.Client, ctx context.Context, svcARN, tdARN string) + first, next map[string]string + }{ + { + name: "default-none", + first: map[string]string{}, next: map[string]string{}, + }, + { + name: "explicit-none", propagate: ecstypes.PropagateTagsNone, + first: map[string]string{}, next: map[string]string{}, + }, + { + name: "service", propagate: ecstypes.PropagateTagsService, + mutate: func(t *testing.T, client *awsecs.Client, ctx context.Context, svcARN, _ string) { + t.Helper() + mustUntag(t, client, ctx, svcARN, "s") + mustTag(t, client, ctx, svcARN, sdkTags("s2", "1")) + }, + first: map[string]string{"s": "1"}, next: map[string]string{"s2": "1"}, + }, + { + name: "task-definition", propagate: ecstypes.PropagateTagsTaskDefinition, + mutate: func(t *testing.T, client *awsecs.Client, ctx context.Context, _, tdARN string) { + t.Helper() + mustTag(t, client, ctx, tdARN, sdkTags("td2", "1")) + }, + first: map[string]string{"td": "1"}, next: map[string]string{"td": "1", "td2": "1"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, cloud := newECSServer(t) + ctx := context.Background() + + if _, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + cloud.ECS.SeedContainerInstance("prod", "i-0prop") + + td, err := client.RegisterTaskDefinition(ctx, &awsecs.RegisterTaskDefinitionInput{ + Family: aws.String("web"), + ContainerDefinitions: []ecstypes.ContainerDefinition{{ + Name: aws.String("app"), Image: aws.String("nginx:latest"), Memory: aws.Int32(128), + }}, + Tags: sdkTags("td", "1"), + }) + if err != nil { + t.Fatalf("RegisterTaskDefinition: %v", err) + } + + svc, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("s"), TaskDefinition: aws.String("web"), + DesiredCount: aws.Int32(1), PropagateTags: tc.propagate, Tags: sdkTags("s", "1"), + }) + if err != nil { + t.Fatalf("CreateService: %v", err) + } + + assertAllTaskTags(t, serviceTaskTags(t, client, ctx, "s"), tc.first, "first deployment") + + if tc.mutate != nil { + tc.mutate(t, client, ctx, aws.ToString(svc.Service.ServiceArn), + aws.ToString(td.TaskDefinition.TaskDefinitionArn)) + } + + if _, err := client.UpdateService(ctx, &awsecs.UpdateServiceInput{ + Cluster: aws.String("prod"), Service: aws.String("s"), ForceNewDeployment: true, + }); err != nil { + t.Fatalf("UpdateService: %v", err) + } + + assertAllTaskTags(t, serviceTaskTags(t, client, ctx, "s"), tc.next, "after force-new-deployment") + }) + } +} + +func assertAllTaskTags(t *testing.T, got []map[string]string, want map[string]string, stage string) { + t.Helper() + + for _, tags := range got { + if !maps.Equal(tags, want) { + t.Fatalf("%s: task tags = %v, want %v", stage, tags, want) + } + } +} + +func mustTag(t *testing.T, client *awsecs.Client, ctx context.Context, arn string, tags []ecstypes.Tag) { + t.Helper() + + if _, err := client.TagResource(ctx, &awsecs.TagResourceInput{ResourceArn: aws.String(arn), Tags: tags}); err != nil { + t.Fatalf("TagResource(%s): %v", arn, err) + } +} + +func mustUntag(t *testing.T, client *awsecs.Client, ctx context.Context, arn string, keys ...string) { + t.Helper() + + if _, err := client.UntagResource(ctx, &awsecs.UntagResourceInput{ResourceArn: aws.String(arn), TagKeys: keys}); err != nil { + t.Fatalf("UntagResource(%s): %v", arn, err) + } +} + +// TestSDKDescribeOmitsTagsWithoutInclude guards that DescribeServices, +// DescribeTaskDefinition, DescribeContainerInstances and +// DescribeCapacityProviders return tags only when include=TAGS is sent, as +// real ECS does. +func TestSDKDescribeOmitsTagsWithoutInclude(t *testing.T) { + client, _ := newECSServer(t) + ctx := context.Background() + + if _, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + tdARN := createTaggedTaskDef(t, client, ctx, sdkTags("k", "v")) + registerNginx(t, client, ctx) + createTaggedService(t, client, ctx, sdkTags("k", "v")) + svc, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{ + Cluster: aws.String("prod"), Services: []string{"web-svc"}, + }) + if err != nil || len(svc.Services) != 1 { + t.Fatalf("DescribeServices: %v", err) + } + + if svc.Services[0].Tags != nil { + t.Errorf("DescribeServices without include tags = %v, want none", svc.Services[0].Tags) + } + + td, err := client.DescribeTaskDefinition(ctx, &awsecs.DescribeTaskDefinitionInput{TaskDefinition: aws.String(tdARN)}) + if err != nil { + t.Fatalf("DescribeTaskDefinition: %v", err) + } + + if td.Tags != nil { + t.Errorf("DescribeTaskDefinition without include tags = %v, want none", td.Tags) + } + + ciARN := createTaggedInstance(t, client, ctx, sdkTags("k", "v")) + cpARN := createTaggedCapacityProvider(t, client, ctx, sdkTags("k", "v")) + + ci, err := client.DescribeContainerInstances(ctx, &awsecs.DescribeContainerInstancesInput{ + Cluster: aws.String("prod"), ContainerInstances: []string{ciARN}, + }) + if err != nil || len(ci.ContainerInstances) != 1 { + t.Fatalf("DescribeContainerInstances: %v", err) + } + + if ci.ContainerInstances[0].Tags != nil { + t.Errorf("DescribeContainerInstances without include tags = %v, want none", ci.ContainerInstances[0].Tags) + } + + cp, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{ + CapacityProviders: []string{cpARN}, + }) + if err != nil || len(cp.CapacityProviders) != 1 { + t.Fatalf("DescribeCapacityProviders: %v", err) + } + + if cp.CapacityProviders[0].Tags != nil { + t.Errorf("DescribeCapacityProviders without include tags = %v, want none", cp.CapacityProviders[0].Tags) + } +} + +// TestSDKTagResourceRejectsInvalidRequests guards the TagResource validation +// real ECS applies: an unknown resource, a short-format service ARN (the +// TagResource reference requires migrating it to the long format first), a +// predefined Fargate capacity provider, more than 50 tags on a resource, and a +// key or value with the reserved aws: prefix are all InvalidParameterException. +func TestSDKTagResourceRejectsInvalidRequests(t *testing.T) { + client, _ := newECSServer(t) + ctx := context.Background() + + c, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + clusterARN := aws.ToString(c.Cluster.ClusterArn) + + registerNginx(t, client, ctx) + svcARN := createTaggedService(t, client, ctx, nil) + shortSvcARN := "arn:aws:ecs:us-east-1:000000000000:service/web-svc" + + fifty := make([]ecstypes.Tag, 0, 50) + for i := range 50 { + fifty = append(fifty, ecstypes.Tag{Key: aws.String(fmt.Sprintf("k%02d", i)), Value: aws.String("v")}) + } + + cases := []struct { + name string + arn string + tags []ecstypes.Tag + }{ + {"nonexistent-cluster", "arn:aws:ecs:us-east-1:000000000000:cluster/ghost", sdkTags("a", "1")}, + {"nonexistent-service", "arn:aws:ecs:us-east-1:000000000000:service/prod/ghost", sdkTags("a", "1")}, + {"not-an-ecs-arn", "arn:aws:s3:::bucket", sdkTags("a", "1")}, + {"short-service-arn", shortSvcARN, sdkTags("a", "1")}, + {"fargate-capacity-provider", "arn:aws:ecs:us-east-1:000000000000:capacity-provider/FARGATE", sdkTags("a", "1")}, + {"fifty-one-in-one-call", clusterARN, append(append([]ecstypes.Tag{}, fifty...), sdkTags("extra", "1")...)}, + {"aws-prefixed-key", clusterARN, sdkTags("aws:owner", "x")}, + {"AWS-prefixed-key", clusterARN, sdkTags("AWS:owner", "x")}, + {"aws-prefixed-value", clusterARN, sdkTags("owner", "aws:x")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := client.TagResource(ctx, &awsecs.TagResourceInput{ResourceArn: aws.String(tc.arn), Tags: tc.tags}) + + var ipe *ecstypes.InvalidParameterException + if !errorsAs(err, &ipe) { + t.Fatalf("TagResource err = %v, want InvalidParameterException", err) + } + }) + } + + // 50 tags fit; one more on a later call pushes the resource past the limit + // and leaves its tags unchanged. + mustTag(t, client, ctx, svcARN, fifty) + + _, err = client.TagResource(ctx, &awsecs.TagResourceInput{ResourceArn: aws.String(svcARN), Tags: sdkTags("k50", "v")}) + + var ipe *ecstypes.InvalidParameterException + if !errorsAs(err, &ipe) { + t.Fatalf("TagResource(51st tag) err = %v, want InvalidParameterException", err) + } + + list, err := client.ListTagsForResource(ctx, &awsecs.ListTagsForResourceInput{ResourceArn: aws.String(svcARN)}) + if err != nil || len(list.Tags) != 50 { + t.Fatalf("ListTagsForResource after rejected tag = %d tags, %v; want 50", len(list.Tags), err) + } + + // Overwriting an existing key does not count as a new tag. + mustTag(t, client, ctx, svcARN, sdkTags("k00", "changed")) + + // Reserved keys cannot be removed, and an unknown resource cannot be untagged. + _, err = client.UntagResource(ctx, &awsecs.UntagResourceInput{ResourceArn: aws.String(svcARN), TagKeys: []string{"aws:x"}}) + if !errorsAs(err, &ipe) { + t.Fatalf("UntagResource(aws: key) err = %v, want InvalidParameterException", err) + } + + _, err = client.UntagResource(ctx, &awsecs.UntagResourceInput{ + ResourceArn: aws.String("arn:aws:ecs:us-east-1:000000000000:cluster/ghost"), TagKeys: []string{"a"}, + }) + if !errorsAs(err, &ipe) { + t.Fatalf("UntagResource(unknown) err = %v, want InvalidParameterException", err) + } +} + +// TestSDKShortServiceARNResolves guards that a short-format service ARN +// (service/, no cluster segment) resolves on the service APIs that take a +// service name or ARN, while tagging it is refused (see +// TestSDKTagResourceRejectsInvalidRequests). +func TestSDKShortServiceARNResolves(t *testing.T) { + client, _ := newECSServer(t) + ctx := context.Background() + + if _, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + registerNginx(t, client, ctx) + longARN := createTaggedService(t, client, ctx, nil) + shortARN := "arn:aws:ecs:us-east-1:000000000000:service/web-svc" + + out, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{ + Cluster: aws.String("prod"), Services: []string{shortARN}, + }) + if err != nil { + t.Fatalf("DescribeServices: %v", err) + } + + if len(out.Services) != 1 || aws.ToString(out.Services[0].ServiceArn) != longARN { + t.Fatalf("DescribeServices(short ARN) = %+v, failures %+v; want %s", out.Services, out.Failures, longARN) + } + + upd, err := client.UpdateService(ctx, &awsecs.UpdateServiceInput{ + Cluster: aws.String("prod"), Service: aws.String(shortARN), DesiredCount: aws.Int32(0), + }) + if err != nil || aws.ToString(upd.Service.ServiceArn) != longARN { + t.Fatalf("UpdateService(short ARN) = %v, %v", upd, err) + } +} + +// TestSDKCapacityProviderLifecycle drives Create/Describe/Update/Delete +// CapacityProvider through the SDK, including the predefined FARGATE and +// FARGATE_SPOT providers. +func TestSDKCapacityProviderLifecycle(t *testing.T) { + client, _ := newECSServer(t) + ctx := context.Background() + + all, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{}) + if err != nil { + t.Fatalf("DescribeCapacityProviders: %v", err) + } + + if names := capacityProviderNames(all.CapacityProviders); !maps.Equal(names, map[string]string{ + "FARGATE": "FARGATE", "FARGATE_SPOT": "FARGATE_SPOT", + }) { + t.Fatalf("predefined providers = %v", names) + } + + created, err := client.CreateCapacityProvider(ctx, &awsecs.CreateCapacityProviderInput{ + Name: aws.String("asg-cp"), + AutoScalingGroupProvider: &ecstypes.AutoScalingGroupProvider{ + AutoScalingGroupArn: aws.String(testASGArn), + ManagedScaling: &ecstypes.ManagedScaling{Status: ecstypes.ManagedScalingStatusEnabled}, + }, + Tags: sdkTags("team", "a"), + }) + if err != nil { + t.Fatalf("CreateCapacityProvider: %v", err) + } + + cp := created.CapacityProvider + if cp.Status != ecstypes.CapacityProviderStatusActive || cp.Type != ecstypes.CapacityProviderTypeEc2Autoscaling || + aws.ToString(cp.AutoScalingGroupProvider.AutoScalingGroupArn) != testASGArn { + t.Fatalf("created provider = %+v", cp) + } + + ms := cp.AutoScalingGroupProvider.ManagedScaling + if aws.ToInt32(ms.TargetCapacity) != 100 || aws.ToInt32(ms.MinimumScalingStepSize) != 1 || + aws.ToInt32(ms.MaximumScalingStepSize) != 10000 || aws.ToInt32(ms.InstanceWarmupPeriod) != 300 { + t.Fatalf("managedScaling defaults = %+v", ms) + } + + if cp.AutoScalingGroupProvider.ManagedTerminationProtection != ecstypes.ManagedTerminationProtectionDisabled { + t.Fatalf("managedTerminationProtection = %q, want DISABLED", cp.AutoScalingGroupProvider.ManagedTerminationProtection) + } + + expectCapacityProviderError(t, "duplicate name", func() error { + _, err := client.CreateCapacityProvider(ctx, &awsecs.CreateCapacityProviderInput{ + Name: aws.String("asg-cp"), + AutoScalingGroupProvider: &ecstypes.AutoScalingGroupProvider{AutoScalingGroupArn: aws.String(testASGArn)}, + }) + + return err + }) + + expectCapacityProviderError(t, "reserved name prefix", func() error { + _, err := client.CreateCapacityProvider(ctx, &awsecs.CreateCapacityProviderInput{ + Name: aws.String("fargate-mine"), + AutoScalingGroupProvider: &ecstypes.AutoScalingGroupProvider{AutoScalingGroupArn: aws.String(testASGArn)}, + }) + + return err + }) + + upd, err := client.UpdateCapacityProvider(ctx, &awsecs.UpdateCapacityProviderInput{ + Name: aws.String("asg-cp"), + AutoScalingGroupProvider: &ecstypes.AutoScalingGroupProviderUpdate{ + ManagedScaling: &ecstypes.ManagedScaling{Status: ecstypes.ManagedScalingStatusEnabled, TargetCapacity: aws.Int32(80)}, + }, + }) + if err != nil { + t.Fatalf("UpdateCapacityProvider: %v", err) + } + + if upd.CapacityProvider.UpdateStatus != ecstypes.CapacityProviderUpdateStatusUpdateComplete || + aws.ToInt32(upd.CapacityProvider.AutoScalingGroupProvider.ManagedScaling.TargetCapacity) != 80 { + t.Fatalf("updated provider = %+v", upd.CapacityProvider) + } + + // A provider associated with a cluster can't be deleted until it is removed. + if _, err := client.CreateCluster(ctx, &awsecs.CreateClusterInput{ClusterName: aws.String("prod")}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + if _, err := client.PutClusterCapacityProviders(ctx, &awsecs.PutClusterCapacityProvidersInput{ + Cluster: aws.String("prod"), CapacityProviders: []string{"asg-cp"}, + DefaultCapacityProviderStrategy: []ecstypes.CapacityProviderStrategyItem{}, + }); err != nil { + t.Fatalf("PutClusterCapacityProviders: %v", err) + } + + scoped, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{Cluster: aws.String("prod")}) + if err != nil || len(scoped.CapacityProviders) != 1 || aws.ToString(scoped.CapacityProviders[0].Name) != "asg-cp" { + t.Fatalf("DescribeCapacityProviders(cluster) = %+v, %v", scoped, err) + } + + expectCapacityProviderError(t, "delete associated", func() error { + _, err := client.DeleteCapacityProvider(ctx, &awsecs.DeleteCapacityProviderInput{CapacityProvider: aws.String("asg-cp")}) + return err + }) + + expectCapacityProviderError(t, "delete FARGATE", func() error { + _, err := client.DeleteCapacityProvider(ctx, &awsecs.DeleteCapacityProviderInput{CapacityProvider: aws.String("FARGATE")}) + return err + }) + + if _, err := client.PutClusterCapacityProviders(ctx, &awsecs.PutClusterCapacityProvidersInput{ + Cluster: aws.String("prod"), CapacityProviders: []string{}, + DefaultCapacityProviderStrategy: []ecstypes.CapacityProviderStrategyItem{}, + }); err != nil { + t.Fatalf("PutClusterCapacityProviders(clear): %v", err) + } + + del, err := client.DeleteCapacityProvider(ctx, &awsecs.DeleteCapacityProviderInput{ + CapacityProvider: cp.CapacityProviderArn, + }) + if err != nil { + t.Fatalf("DeleteCapacityProvider: %v", err) + } + + if del.CapacityProvider.Status != ecstypes.CapacityProviderStatusInactive || + del.CapacityProvider.UpdateStatus != ecstypes.CapacityProviderUpdateStatusDeleteComplete { + t.Fatalf("deleted provider = %+v", del.CapacityProvider) + } + + // A deleted provider can't be tagged any more. + _, err = client.TagResource(ctx, &awsecs.TagResourceInput{ResourceArn: cp.CapacityProviderArn, Tags: sdkTags("a", "1")}) + + var ipe *ecstypes.InvalidParameterException + if !errorsAs(err, &ipe) { + t.Fatalf("TagResource(deleted provider) err = %v, want InvalidParameterException", err) + } +} + +func capacityProviderNames(in []ecstypes.CapacityProvider) map[string]string { + out := make(map[string]string, len(in)) + for i := range in { + out[aws.ToString(in[i].Name)] = string(in[i].Type) + } + + return out +} + +// expectCapacityProviderError asserts call fails with a typed ECS client error +// (InvalidParameterException or ClientException). +func expectCapacityProviderError(t *testing.T, what string, call func() error) { + t.Helper() + + err := call() + + var ( + ipe *ecstypes.InvalidParameterException + ce *ecstypes.ClientException + ) + + if !errorsAs(err, &ipe) && !errorsAs(err, &ce) { + t.Fatalf("%s: err = %v, want InvalidParameterException or ClientException", what, err) + } +} + +// TestSDKDescribeCapacityProvidersPagination guards maxResults/nextToken paging +// and the error paths of the capacity-provider wire handlers. +func TestSDKDescribeCapacityProvidersPagination(t *testing.T) { + client, _ := newECSServer(t) + ctx := context.Background() + + first, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{MaxResults: aws.Int32(1)}) + if err != nil { + t.Fatalf("DescribeCapacityProviders page 1: %v", err) + } + + if len(first.CapacityProviders) != 1 || first.NextToken == nil { + t.Fatalf("page 1 = %d providers, nextToken %v; want 1 and a token", len(first.CapacityProviders), first.NextToken) + } + + second, err := client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{ + MaxResults: aws.Int32(1), NextToken: first.NextToken, + }) + if err != nil { + t.Fatalf("DescribeCapacityProviders page 2: %v", err) + } + + if len(second.CapacityProviders) != 1 || second.NextToken != nil || + aws.ToString(second.CapacityProviders[0].Name) == aws.ToString(first.CapacityProviders[0].Name) { + t.Fatalf("page 2 = %+v, nextToken %v", second.CapacityProviders, second.NextToken) + } + + var ipe *ecstypes.InvalidParameterException + + _, err = client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{NextToken: aws.String("!!bad")}) + if !errorsAs(err, &ipe) { + t.Fatalf("DescribeCapacityProviders(bad token) err = %v, want InvalidParameterException", err) + } + + _, err = client.DescribeCapacityProviders(ctx, &awsecs.DescribeCapacityProvidersInput{Cluster: aws.String("ghost")}) + + var cnf *ecstypes.ClusterNotFoundException + if !errorsAs(err, &cnf) { + t.Fatalf("DescribeCapacityProviders(unknown cluster) err = %v, want ClusterNotFoundException", err) + } + + expectCapacityProviderError(t, "update unknown", func() error { + _, err := client.UpdateCapacityProvider(ctx, &awsecs.UpdateCapacityProviderInput{Name: aws.String("ghost")}) + return err + }) + + expectCapacityProviderError(t, "create without provider block", func() error { + _, err := client.CreateCapacityProvider(ctx, &awsecs.CreateCapacityProviderInput{Name: aws.String("bare")}) + return err + }) +} diff --git a/server/aws/ecs/taskdefs.go b/server/aws/ecs/taskdefs.go index 00348f703..608ac73f4 100644 --- a/server/aws/ecs/taskdefs.go +++ b/server/aws/ecs/taskdefs.go @@ -136,7 +136,8 @@ func (h *Handler) listTaskDefinitions(w http.ResponseWriter, r *http.Request) { func (h *Handler) describeTaskDefinition(w http.ResponseWriter, r *http.Request) { var req struct { - TaskDefinition string `json:"taskDefinition"` + TaskDefinition string `json:"taskDefinition"` + Include []string `json:"include"` } if !wire.DecodeJSON(w, r, &req) { @@ -150,7 +151,14 @@ func (h *Handler) describeTaskDefinition(w http.ResponseWriter, r *http.Request) return } - wire.WriteJSON(w, map[string]any{"taskDefinition": taskDefToWire(td), "tags": fromTags(td.Tags)}) + resp := map[string]any{"taskDefinition": taskDefToWire(td)} + + // Tags are only returned when the caller opts in via include=TAGS. + if includes(req.Include, "TAGS") { + resp["tags"] = fromTags(td.Tags) + } + + wire.WriteJSON(w, resp) } func (h *Handler) deregisterTaskDefinition(w http.ResponseWriter, r *http.Request) { diff --git a/server/aws/ecs/types.go b/server/aws/ecs/types.go index ba56dae6c..fe2790023 100644 --- a/server/aws/ecs/types.go +++ b/server/aws/ecs/types.go @@ -399,6 +399,7 @@ type wireContainerInstance struct { AgentConnected bool `json:"agentConnected"` RegisteredResources []wireResource `json:"registeredResources,omitempty"` RemainingResources []wireResource `json:"remainingResources,omitempty"` + Tags []wireTag `json:"tags,omitempty"` } // --- request -> driver converters --- @@ -1478,5 +1479,6 @@ func instanceToWire(ci *driver.ContainerInstance) wireContainerInstance { AgentConnected: ci.AgentConnected, RegisteredResources: resourcesFromCapacity(ci.RegisteredCPU, ci.RegisteredMemory), RemainingResources: resourcesFromCapacity(ci.RemainingCPU, ci.RemainingMemory), + Tags: fromTags(ci.Tags), } } diff --git a/services/ecs/driver/driver.go b/services/ecs/driver/driver.go index b5d771f2f..f1a1599e9 100644 --- a/services/ecs/driver/driver.go +++ b/services/ecs/driver/driver.go @@ -321,6 +321,62 @@ type CapacityProviderStrategyItem struct { Weight int } +// ManagedScaling is the managed-scaling block of an Auto Scaling group +// capacity provider. Nil integer pointers mean "not set". +type ManagedScaling struct { + Status string + TargetCapacity *int + MinimumScalingStepSize *int + MaximumScalingStepSize *int + InstanceWarmupPeriod *int +} + +// AutoScalingGroupProvider is the Auto Scaling group backing an +// EC2_AUTOSCALING capacity provider. On update AutoScalingGroupARN is ignored +// (the group cannot be changed) and empty fields leave the stored value alone. +type AutoScalingGroupProvider struct { + AutoScalingGroupARN string + ManagedScaling *ManagedScaling + ManagedTerminationProtection string + ManagedDraining string +} + +// CapacityProvider is an ECS capacity provider. The predefined FARGATE and +// FARGATE_SPOT providers are reported with Type FARGATE / FARGATE_SPOT and +// cannot be tagged. ManagedInstancesProvider is stored and echoed verbatim as +// raw JSON; the emulator does not launch managed instances from it. +type CapacityProvider struct { + ARN string + Name string + Status string + Type string + Cluster string + UpdateStatus string + UpdateStatusReason string + AutoScalingGroupProvider *AutoScalingGroupProvider + ManagedInstancesProvider json.RawMessage + Tags []Tag +} + +// CreateCapacityProviderInput describes a capacity provider to create. Exactly +// one of AutoScalingGroupProvider and ManagedInstancesProvider must be set. +type CreateCapacityProviderInput struct { + Name string + Cluster string + AutoScalingGroupProvider *AutoScalingGroupProvider + ManagedInstancesProvider json.RawMessage + Tags []Tag +} + +// UpdateCapacityProviderInput describes changes to an existing capacity +// provider. Nil blocks leave the stored configuration unchanged. +type UpdateCapacityProviderInput struct { + Name string + Cluster string + AutoScalingGroupProvider *AutoScalingGroupProvider + ManagedInstancesProvider json.RawMessage +} + // Task is a running or stopped ECS task. type Task struct { ARN string @@ -467,6 +523,7 @@ type ContainerInstance struct { RegisteredMemory int RemainingCPU int RemainingMemory int + Tags []Tag } // Failure describes a resource that could not be resolved in a batch @@ -579,6 +636,7 @@ type RegisterContainerInstanceInput struct { InstanceIdentityDocument string TotalResources []Resource Attributes []Attribute + Tags []Tag } // UpdateClusterInput describes mutations to a cluster's settings and @@ -650,6 +708,11 @@ type ECS interface { UntagResource(ctx context.Context, resourceARN string, tagKeys []string) error ListTagsForResource(ctx context.Context, resourceARN string) ([]Tag, error) + CreateCapacityProvider(ctx context.Context, in CreateCapacityProviderInput) (*CapacityProvider, error) + DescribeCapacityProviders(ctx context.Context, cluster string, ids []string) ([]CapacityProvider, []Failure, error) + UpdateCapacityProvider(ctx context.Context, in UpdateCapacityProviderInput) (*CapacityProvider, error) + DeleteCapacityProvider(ctx context.Context, cluster, capacityProvider string) (*CapacityProvider, error) + PutAccountSetting(ctx context.Context, name, value string) (*AccountSetting, error) PutAccountSettingDefault(ctx context.Context, name, value string) (*AccountSetting, error) ListAccountSettings(ctx context.Context) ([]AccountSetting, error) diff --git a/services/ecs/ecs.go b/services/ecs/ecs.go index 9d650e801..e0e6db646 100644 --- a/services/ecs/ecs.go +++ b/services/ecs/ecs.go @@ -518,3 +518,57 @@ func (e *ECS) ExecuteCommand(ctx context.Context, in driver.ExecuteCommandInput) return out.(*driver.ExecuteCommandResult), nil } + +// CreateCapacityProvider creates an Auto Scaling group or Managed Instances capacity provider. +// +//nolint:gocritic // in is passed by value to mirror the driver.ECS interface; the copy is cheap for a mock. +func (e *ECS) CreateCapacityProvider(ctx context.Context, in driver.CreateCapacityProviderInput) ( + *driver.CapacityProvider, error, +) { + out, err := e.do(ctx, "CreateCapacityProvider", in, func() (any, error) { + return e.driver.CreateCapacityProvider(ctx, in) + }) + if err != nil { + return nil, err + } + + return out.(*driver.CapacityProvider), nil +} + +// DescribeCapacityProviders resolves capacity providers by name or ARN (all of +// them, including FARGATE and FARGATE_SPOT, when ids is empty). +func (e *ECS) DescribeCapacityProviders(ctx context.Context, cluster string, ids []string) ( + []driver.CapacityProvider, []driver.Failure, error, +) { + return doBatch(ctx, e, "DescribeCapacityProviders", ids, func() ([]driver.CapacityProvider, []driver.Failure, error) { + return e.driver.DescribeCapacityProviders(ctx, cluster, ids) + }) +} + +// UpdateCapacityProvider modifies a capacity provider's scaling configuration. +func (e *ECS) UpdateCapacityProvider(ctx context.Context, in driver.UpdateCapacityProviderInput) ( + *driver.CapacityProvider, error, +) { + out, err := e.do(ctx, "UpdateCapacityProvider", in, func() (any, error) { + return e.driver.UpdateCapacityProvider(ctx, in) + }) + if err != nil { + return nil, err + } + + return out.(*driver.CapacityProvider), nil +} + +// DeleteCapacityProvider deletes a capacity provider that no cluster uses. +func (e *ECS) DeleteCapacityProvider(ctx context.Context, cluster, capacityProvider string) ( + *driver.CapacityProvider, error, +) { + out, err := e.do(ctx, "DeleteCapacityProvider", capacityProvider, func() (any, error) { + return e.driver.DeleteCapacityProvider(ctx, cluster, capacityProvider) + }) + if err != nil { + return nil, err + } + + return out.(*driver.CapacityProvider), nil +} diff --git a/services/ecs/ecs_test.go b/services/ecs/ecs_test.go index 9c4ceef93..9688d1454 100644 --- a/services/ecs/ecs_test.go +++ b/services/ecs/ecs_test.go @@ -188,3 +188,43 @@ func TestWrapperWave4bFlow(t *testing.T) { // The wrapper records every proxied call. assert.Equal(t, 1, rec.CallCountFor("ecs", "TagResource")) } + +func TestWrapperCapacityProviderFlow(t *testing.T) { + e, _, rec := newWrapper(t) + ctx := context.Background() + + cp, err := e.CreateCapacityProvider(ctx, driver.CreateCapacityProviderInput{ + Name: "asg", + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{AutoScalingGroupARN: "asg-name"}, + }) + require.NoError(t, err) + assert.Equal(t, "EC2_AUTOSCALING", cp.Type) + + found, failures, err := e.DescribeCapacityProviders(ctx, "", []string{"asg", "ghost"}) + require.NoError(t, err) + assert.Len(t, found, 1) + assert.Len(t, failures, 1) + + updated, err := e.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{ + Name: "asg", + AutoScalingGroupProvider: &driver.AutoScalingGroupProvider{ManagedDraining: "ENABLED"}, + }) + require.NoError(t, err) + assert.Equal(t, "ENABLED", updated.AutoScalingGroupProvider.ManagedDraining) + + deleted, err := e.DeleteCapacityProvider(ctx, "", "asg") + require.NoError(t, err) + assert.Equal(t, "INACTIVE", deleted.Status) + + // Errors propagate through every wrapper. + _, err = e.CreateCapacityProvider(ctx, driver.CreateCapacityProviderInput{Name: "bad"}) + require.Error(t, err) + _, err = e.UpdateCapacityProvider(ctx, driver.UpdateCapacityProviderInput{Name: "ghost"}) + require.Error(t, err) + _, err = e.DeleteCapacityProvider(ctx, "", "ghost") + require.Error(t, err) + + for _, op := range []string{"CreateCapacityProvider", "DescribeCapacityProviders", "UpdateCapacityProvider", "DeleteCapacityProvider"} { + assert.Positive(t, rec.CallCountFor("ecs", op), op) + } +} From 676e0000c2e72b211d3831350343a5cd6177d992 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:35:41 +0530 Subject: [PATCH 11/24] fix(aws-ecs): DescribeTasks returns tags only with include=TAGS DescribeTasks always returned a task's tags. ECS returns them only when the caller sends include=TAGS, as DescribeServices, DescribeClusters and DescribeTaskDefinition now do. RunTask's response is unchanged. The roundtrip test now asserts both halves: no tags without include, the tags with it. --- server/aws/ecs/sdk_roundtrip_test.go | 14 ++++++++++++++ server/aws/ecs/tasks.go | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/server/aws/ecs/sdk_roundtrip_test.go b/server/aws/ecs/sdk_roundtrip_test.go index 7dcaa42fe..8fd284492 100644 --- a/server/aws/ecs/sdk_roundtrip_test.go +++ b/server/aws/ecs/sdk_roundtrip_test.go @@ -890,9 +890,23 @@ func TestSDKRunTaskTagsRoundtrip(t *testing.T) { t.Fatalf("RunTask response tags = %+v, want [env=prod]", run.Tasks[0].Tags) } + // ECS returns a task's tags only when the caller opts in via include=TAGS. + bare, err := client.DescribeTasks(ctx, &awsecs.DescribeTasksInput{ + Cluster: aws.String("prod"), + Tasks: []string{taskArn}, + }) + if err != nil { + t.Fatalf("DescribeTasks without include: %v", err) + } + + if len(bare.Tasks) != 1 || len(bare.Tasks[0].Tags) != 0 { + t.Fatalf("DescribeTasks without include tags = %+v, want none", bare.Tasks) + } + desc, err := client.DescribeTasks(ctx, &awsecs.DescribeTasksInput{ Cluster: aws.String("prod"), Tasks: []string{taskArn}, + Include: []ecstypes.TaskField{ecstypes.TaskFieldTags}, }) if err != nil { t.Fatalf("DescribeTasks: %v", err) diff --git a/server/aws/ecs/tasks.go b/server/aws/ecs/tasks.go index 90f36f1ed..c9c27e878 100644 --- a/server/aws/ecs/tasks.go +++ b/server/aws/ecs/tasks.go @@ -164,6 +164,7 @@ func (h *Handler) describeTasks(w http.ResponseWriter, r *http.Request) { var req struct { Tasks []string `json:"tasks"` Cluster string `json:"cluster"` + Include []string `json:"include"` } if !wire.DecodeJSON(w, r, &req) { @@ -177,6 +178,13 @@ func (h *Handler) describeTasks(w http.ResponseWriter, r *http.Request) { return } + // Tags are only returned when the caller opts in via include=TAGS. + if !includes(req.Include, "TAGS") { + for i := range tasks { + tasks[i].Tags = nil + } + } + h.writeTasks(w, tasks, failures) } From 956451e721d7ef615ee1cdb39d1dcdd4bc72ce3e Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:46:36 +0530 Subject: [PATCH 12/24] feat(gcp): shared TypedAny/FormatTime/DecodeOptionalJSON helpers and LRO metadata replay Location-scoped GCP services each carried a private copy of the same three helpers (responseAny/anyWithType, formatTime, a decodeBody that tolerates an empty body). Add one shared copy of each to server/wire/gcprest so new services stop re-deriving them: - TypedAny renders a value as a proto3-JSON google.protobuf.Any ("@type" added to its object), the shape a done Operation's response and metadata carry. - FormatTime renders a google.protobuf.Timestamp (RFC 3339 UTC, ns) and a zero time as "". - DecodeOptionalJSON is DecodeJSON for requests whose body may be empty. The shared lro.Registry only replayed an operation's response, so a service whose operations carry OperationMetadata lost it on every poll. Add RegisterWithMetadata; a done poll now replays metadata beside the response. Register is unchanged for every existing caller. --- server/gcp/lro/handler.go | 29 ++++++++++-- server/gcp/lro/handler_test.go | 24 ++++++++++ server/wire/gcprest/anyjson.go | 65 ++++++++++++++++++++++++++ server/wire/gcprest/anyjson_test.go | 71 +++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 server/wire/gcprest/anyjson.go create mode 100644 server/wire/gcprest/anyjson_test.go diff --git a/server/gcp/lro/handler.go b/server/gcp/lro/handler.go index dc5ad3330..fad190e48 100644 --- a/server/gcp/lro/handler.go +++ b/server/gcp/lro/handler.go @@ -48,6 +48,7 @@ const ( // poll replays, and whether Cancel has since been called on it. type entry struct { response any + metadata any canceled bool } @@ -81,6 +82,20 @@ func (r *Registry) Register(name string, response any) { r.ops[name] = entry{response: response} } +// RegisterWithMetadata is Register for a service whose operations also carry a +// typed metadata message (an OperationMetadata google.protobuf.Any), which a +// done poll then replays alongside the response. A nil registry is a no-op. +func (r *Registry) RegisterWithMetadata(name string, response, metadata any) { + if r == nil { + return + } + + r.mu.Lock() + defer r.mu.Unlock() + + r.ops[name] = entry{response: response, metadata: metadata} +} + // lookup returns the recorded entry for name and whether it was registered. func (r *Registry) lookup(name string) (e entry, found bool) { r.mu.RLock() @@ -197,7 +212,7 @@ func (h *Handler) serveGet(w http.ResponseWriter, name string) { return } - writeDone(w, name, e.response, e.canceled) + writeDone(w, name, e, e.canceled) } // serveCancel implements Operations.Cancel. Real GCP makes a best-effort @@ -238,25 +253,29 @@ func writeLegacy(w http.ResponseWriter, name string, cancel bool, method string) return } - writeDone(w, name, nil, false) + writeDone(w, name, entry{}, false) } // writeDone writes a completed operation. It returns a superset that satisfies // both operation schemas served here: google.longrunning.Operation reads `done` // (artifactregistry, eventarc, memorystore, alloydb) while GKE's // container.Operation reads `status`. -func writeDone(w http.ResponseWriter, name string, response any, canceled bool) { +func writeDone(w http.ResponseWriter, name string, e entry, canceled bool) { body := map[string]any{ "name": name, "done": true, "status": "DONE", } + if e.metadata != nil { + body["metadata"] = e.metadata + } + switch { case canceled: body["error"] = map[string]any{"code": canceledCode, "message": "Operation was canceled"} - case response != nil: - body["response"] = response + case e.response != nil: + body["response"] = e.response } gcprest.WriteJSON(w, http.StatusOK, body) diff --git a/server/gcp/lro/handler_test.go b/server/gcp/lro/handler_test.go index 11677f86f..e2db61259 100644 --- a/server/gcp/lro/handler_test.go +++ b/server/gcp/lro/handler_test.go @@ -181,3 +181,27 @@ func TestRegistryConcurrentAccess(t *testing.T) { wg.Wait() } + +// TestRegisteredOperationReplaysMetadata: an operation registered with +// metadata replays it beside the response; one registered without has none. +func TestRegisteredOperationReplaysMetadata(t *testing.T) { + reg := lro.NewRegistry() + reg.RegisterWithMetadata("projects/p/locations/us/operations/op-1", + map[string]any{"@type": "t/Resource"}, map[string]any{"@type": "t/OperationMetadata", "verb": "create"}) + reg.Register("projects/p/locations/us/operations/op-2", nil) + + h := lro.New(reg) + + code, body := get(t, h, opPath) + if code != http.StatusOK || !strings.Contains(body, `"metadata":{"@type":"t/OperationMetadata","verb":"create"}`) || + !strings.Contains(body, `"response":{"@type":"t/Resource"}`) { + t.Fatalf("op-1: %d %s", code, body) + } + + if _, body = get(t, h, "/v1/projects/p/locations/us/operations/op-2"); strings.Contains(body, "metadata") { + t.Fatalf("op-2 has metadata: %s", body) + } + + var nilReg *lro.Registry + nilReg.RegisterWithMetadata("x", nil, nil) // a nil registry is a no-op +} diff --git a/server/wire/gcprest/anyjson.go b/server/wire/gcprest/anyjson.go new file mode 100644 index 000000000..47fedf882 --- /dev/null +++ b/server/wire/gcprest/anyjson.go @@ -0,0 +1,65 @@ +package gcprest + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "time" +) + +// TypedAny renders v as a google.protobuf.Any in proto3 JSON: v's own JSON +// object with the "@type" discriminator added. It is the shape a done +// google.longrunning.Operation carries in `response` and `metadata`; a wrong or +// missing type URL makes a GAPIC or Terraform LRO wait fail, so callers pass the +// exact proto type (type.googleapis.com/google.cloud..v1.). v must +// marshal to a JSON object (an empty struct yields {"@type": …}). +func TypedAny(v any, typeURL string) (json.RawMessage, error) { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + + var fields map[string]json.RawMessage + if uErr := json.Unmarshal(raw, &fields); uErr != nil { + return nil, uErr + } + + if fields == nil { + fields = map[string]json.RawMessage{} + } + + typ, err := json.Marshal(typeURL) + if err != nil { + return nil, err + } + + fields["@type"] = typ + + return json.Marshal(fields) +} + +// FormatTime renders t as a proto3-JSON google.protobuf.Timestamp (RFC 3339, +// UTC, nanosecond precision); a zero time renders as "" so an omitempty field +// drops it. +func FormatTime(t time.Time) string { + if t.IsZero() { + return "" + } + + return t.UTC().Format(time.RFC3339Nano) +} + +// DecodeOptionalJSON is DecodeJSON for a request whose body the API allows to +// be empty: an empty body leaves v at its zero value instead of failing. A +// present but malformed body is still 400 INVALID_ARGUMENT. +func DecodeOptionalJSON(w http.ResponseWriter, r *http.Request, v any) bool { + r.Body = http.MaxBytesReader(w, r.Body, MaxBodyBytes) + + if err := json.NewDecoder(r.Body).Decode(v); err != nil && !errors.Is(err, io.EOF) { + WriteError(w, http.StatusBadRequest, "invalid", "malformed JSON body: "+err.Error()) + return false + } + + return true +} diff --git a/server/wire/gcprest/anyjson_test.go b/server/wire/gcprest/anyjson_test.go new file mode 100644 index 000000000..89243aac5 --- /dev/null +++ b/server/wire/gcprest/anyjson_test.go @@ -0,0 +1,71 @@ +package gcprest_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" +) + +func TestTypedAny(t *testing.T) { + got, err := gcprest.TypedAny(struct { + Name string `json:"name"` + }{"n"}, "type.googleapis.com/x.Y") + if err != nil || string(got) != `{"@type":"type.googleapis.com/x.Y","name":"n"}` { + t.Fatalf("TypedAny = %s, %v", got, err) + } + + if got, err = gcprest.TypedAny(struct{}{}, "type.googleapis.com/google.protobuf.Empty"); err != nil || + string(got) != `{"@type":"type.googleapis.com/google.protobuf.Empty"}` { + t.Fatalf("gcprest.TypedAny(empty) = %s, %v", got, err) + } + + if got, err = gcprest.TypedAny(nil, "t"); err != nil || string(got) != `{"@type":"t"}` { + t.Fatalf("gcprest.TypedAny(nil) = %s, %v", got, err) + } + + if _, err = gcprest.TypedAny([]int{1}, "t"); err == nil { + t.Fatal("TypedAny of a non-object must fail") + } + + if _, err = gcprest.TypedAny(make(chan int), "t"); err == nil { + t.Fatal("TypedAny of an unmarshalable value must fail") + } +} + +func TestFormatTime(t *testing.T) { + if gcprest.FormatTime(time.Time{}) != "" { + t.Fatal("zero time must render empty") + } + + ts := time.Date(2026, 1, 2, 3, 4, 5, 6, time.FixedZone("x", 3600)) + if got := gcprest.FormatTime(ts); got != "2026-01-02T02:04:05.000000006Z" { + t.Fatalf("FormatTime = %q", got) + } +} + +func TestDecodeOptionalJSON(t *testing.T) { + var v struct { + A int `json:"a"` + } + + for _, tc := range []struct { + body string + ok bool + code int + }{ + {"", true, http.StatusOK}, + {`{"a":1}`, true, http.StatusOK}, + {`{`, false, http.StatusBadRequest}, + } { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.body)) + + if ok := gcprest.DecodeOptionalJSON(w, r, &v); ok != tc.ok || w.Code != tc.code { + t.Fatalf("body %q: ok=%v code=%d", tc.body, ok, w.Code) + } + } +} From aca0460d859ccfe212fffeecf2310f5d549e12d5 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:46:51 +0530 Subject: [PATCH 13/24] fix(gcp-kafka): route the shared /clusters path by ownership, accept numeric enums, fill missing outputs Managed Kafka, GKE and AlloyDB all serve /v1/projects/{p}/locations/{l}/clusters. Real GCP separates them by hostname; the emulator cannot, and the Kafka handler claimed the whole list as soon as it owned one cluster in a location, so `gcloud container clusters list` and AlloyDB lists lost every cluster once a Kafka cluster existed there. Routing (server layer, since the conflict exists only on the shared path): - The assembled server now hands the Kafka handler a read-only ClusterSibling probe of whichever of GKE / AlloyDB is enabled, wired the way the load balancer's BucketLister is. Kafka claims the list only where it owns a cluster and the sibling owns none; where both do, the sibling keeps its list (the shapes differ and the request says nothing about which API it is for). - A Kafka-shaped PATCH of a cluster nobody owns is claimed, so it is Kafka's 404 instead of GKE's 405. - A create reusing an id the other service already holds in that location is 409 ALREADY_EXISTS, in both directions, instead of making one cluster unreachable. Wire: - rebalanceConfig.mode and the echoed output-only state decode as a name or a proto3-JSON number. The official cloud.google.com/go/managedkafka REST client sends numbers (UseEnumNumbers) and was rejected with 400 on any RebalanceConfig. - kafkaVersion, tlsConfig, updateOptions and brokerCapacityConfig are read, stored and returned instead of dropped; LROs carry google.cloud.managedkafka.v1.OperationMetadata. - Uses the shared gcprest TypedAny/FormatTime/DecodeOptionalJSON helpers. - A standalone package server answers operation polls from a private lro.Registry: the typed response and metadata, and 404 for a name it never issued (it answered done with no response for any name). Provider: - kafkaVersion defaults to 3.7.x and is an optional, updatable input (current discovery doc); rebalanceConfig.mode defaults to NO_REBALANCE, and MODE_UNSPECIFIED means unset, as proto3 has it. - Subnets must be in the cluster's region (the project may differ); brokerCapacityConfig.diskSizeGib >= 100; at most 10 CA pools, each a CA Service pool name. - --async-settle reports a new cluster CREATING before ACTIVE. - GetOperation 404s an unknown name; the operation store is capped at 1000, evicting the oldest. Adds cloud.google.com/go/managedkafka v1.0.0 as a test dependency: it needs go 1.25 (what CI runs) and bumps no existing module. v1.1.0 would bump cloud.google.com/go/longrunning v0.9.0 -> v1.2.0 and v1.2.0 needs go 1.26. Its REST transport encodes enums as numbers, like v1.2.0. --- docs/coverage/coverage.json | 2 +- docs/coverage/gcp/managedkafka.md | 2 +- docs/sdk-server.md | 2 +- go.mod | 1 + go.sum | 2 + providers/gcp/managedkafka/clone.go | 13 + providers/gcp/managedkafka/managedkafka.go | 121 +++++- .../gcp/managedkafka/managedkafka_test.go | 4 +- .../gcp/managedkafka/review_fixes_test.go | 389 ++++++++++++++++++ providers/gcp/managedkafka/validate.go | 127 +++++- server/gcp/cluster_siblings.go | 50 +++ server/gcp/gcp.go | 20 +- server/gcp/managedkafka/handler.go | 220 ++++++++-- server/gcp/managedkafka/operations.go | 53 ++- server/gcp/managedkafka/sdk_gapic_test.go | 161 ++++++++ server/gcp/managedkafka/wire.go | 273 ++++++++---- server/gcp/managedkafka/wire_test.go | 312 ++++++++++++++ server/gcp/operations_ownership_test.go | 95 +++++ services/managedkafka/driver/driver.go | 40 +- 19 files changed, 1705 insertions(+), 182 deletions(-) create mode 100644 providers/gcp/managedkafka/review_fixes_test.go create mode 100644 server/gcp/cluster_siblings.go create mode 100644 server/gcp/managedkafka/sdk_gapic_test.go create mode 100644 server/gcp/managedkafka/wire_test.go diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index af6948983..b45ba2c30 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -10202,7 +10202,7 @@ }, { "name": "GetOperation", - "doc": "GetOperation resolves a (done) long-running operation by name, for a" + "doc": "GetOperation returns an operation this driver created; unknown is NOT_FOUND." }, { "name": "GetTopic" diff --git a/docs/coverage/gcp/managedkafka.md b/docs/coverage/gcp/managedkafka.md index 39ede2ddd..ea8d1105c 100644 --- a/docs/coverage/gcp/managedkafka.md +++ b/docs/coverage/gcp/managedkafka.md @@ -12,7 +12,7 @@ GCP's `managedkafka` service · portable interface `driver.ManagedKafka` · [GCP | `DeleteCluster` | DeleteCluster removes the cluster and every topic under it. | | `DeleteTopic` | | | `GetCluster` | | -| `GetOperation` | GetOperation resolves a (done) long-running operation by name, for a | +| `GetOperation` | GetOperation returns an operation this driver created; unknown is NOT_FOUND. | | `GetTopic` | | | `ListClusters` | | | `ListTopics` | | diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 8b9ca4e60..b8a8153e7 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -229,7 +229,7 @@ All handlers speak REST + JSON. | **GKE** | Clusters (Create/Get/List/Update/Delete + `:setLogging`/`:setMonitoring`/`:setMasterAuth`/`:setLegacyAbac`/`:setNetworkPolicy`/`:setMaintenancePolicy`/`:setResourceLabels`/`:startIpRotation`/`:completeIpRotation`), NodePools (Create/Get/List/Update/Delete + `:setSize`/`:setAutoscaling`/`:setManagement`/`:rollback`), Operations (Get/List/`:cancel`). The cluster endpoint points at the shared [Kubernetes data plane](#kubernetes). | | **Cloud Asset Inventory** | `assets.list` (filter by `assetTypes[]`), `searchAllResources` (query + asset-type filter), `searchAllIamPolicies` (returns empty; not implemented), `exportAssets` (sync; inline results in the returned Operation), `batchGetAssetsHistory`, Feeds (create/list/get/patch/delete), `operations.get`. Resource names returned as GCP-shaped `//service/path` URNs. | | **IAM (iam.googleapis.com v1)** | ServiceAccounts (Create/Get/List/Delete/Patch), custom Roles (Create/Get/List/Delete/Patch), ServiceAccountKeys (Create/Get/List/Delete). Real `google.golang.org/api/iam/v1` clients round-trip end-to-end; errors surface as typed `*googleapi.Error`. Resource-level `getIamPolicy`/`setIamPolicy` bindings on individual GCP resources are out of scope. | -| **Managed Service for Apache Kafka (managedkafka.googleapis.com v1)** | Clusters (Create/Update/Delete as LROs polled through the shared location-operations route, Get, List with `pageToken`/`pageSize`; `updateMask` required, only masked fields change, unknown/immutable/output-only paths are 400) and Topics (Create/Get/List/Patch/Delete, synchronous; `partitionCount` can only increase, `replicationFactor` immutable). Validates `vcpuCount >= 3`, 1-8 GiB memory per vCPU, 1-10 `networkConfigs[].subnet`, and the RFC 1035 `clusterId`. Shares the `/clusters` path with GKE/AlloyDB: it claims only Kafka-shaped creates and clusters it owns. Real `google.golang.org/api/managedkafka/v1` clients round-trip end-to-end. Consumer groups, ACLs, Connect and Schema Registry are out of scope. | +| **Managed Service for Apache Kafka (managedkafka.googleapis.com v1)** | Clusters (Create/Update/Delete as LROs polled through the shared location-operations route, Get, List with `pageToken`/`pageSize`; `updateMask` required, only masked fields change, unknown/immutable/output-only paths are 400) and Topics (Create/Get/List/Patch/Delete, synchronous; `partitionCount` can only increase, `replicationFactor` immutable). Validates `vcpuCount >= 3`, 1-8 GiB memory per vCPU, 1-10 `networkConfigs[].subnet` in the cluster's region, `brokerCapacityConfig.diskSizeGib >= 100`, up to 10 `tlsConfig.trustConfig.casConfigs`, and the RFC 1035 `clusterId`; defaults `kafkaVersion` to `3.7.x` and `rebalanceConfig.mode` to `NO_REBALANCE`; accepts enums as names or proto3-JSON numbers; LROs carry `OperationMetadata`; `--async-settle` reports a new cluster `CREATING` before `ACTIVE`. Shares the `/clusters` path with GKE/AlloyDB and routes it by ownership: it claims Kafka-shaped creates, clusters it owns, Kafka-shaped PATCHes of a missing cluster (404), and a list only where the sibling owns no cluster; an id already used by the other service in that location is 409 `ALREADY_EXISTS`. Real `google.golang.org/api/managedkafka/v1` and `cloud.google.com/go/managedkafka` (GAPIC REST) clients round-trip end-to-end. Consumer groups, ACLs, Connect and Schema Registry are out of scope. | | **Backup and DR (backupdr.googleapis.com v1)** | Backup vaults: create (LRO; `backupVaultId`, required `backupMinimumEnforcedRetentionDuration`, `validateOnly`), get, list (`pageSize`/`pageToken`, `locations/-` wildcard; `filter`/`orderBy` accepted and ignored), patch (LRO; required `updateMask`, output-only/unknown paths 400, stale body `etag` 409 ABORTED), delete (LRO; `force`, `allowMissing`, `etag`, `validateOnly`; a vault holding backups without `force` is 400 FAILED_PRECONDITION). Output-only `state` ACTIVE, `deletable`, `etag` (rotates per update), `uid`, deterministic `serviceAccount` (`service-{12-digit number derived from the project id}@gcp-sa-backupdr-pr.iam.gserviceaccount.com`), `backupCount`/`totalStoredBytes` "0". Operations poll through the shared location-scoped LRO poller. Data sources, backups, backup plans, management servers and restores are out of scope. Real `google.golang.org/api/backupdr/v1` clients round-trip end-to-end. | | **Cloud CDN backend buckets** *(`compute/v1` `global/backendBuckets`)* | Insert, Get, List (`maxResults`/`pageToken`, `name` filter), Patch (JSON merge patch: `cdnPolicy` merges member-by-member), Update (full replace), Delete, `setEdgeSecurityPolicy`; every mutation returns a global `compute#operation` the SDK polls to DONE. Validates the RFC 1035 name, a required `bucketName` that must name an existing GCS bucket when the GCS backend is wired, `compressionMode`, `cdnPolicy.cacheMode`, TTL ranges (0-31,622,400s, `defaultTtl` <= `maxTtl`), `serveWhileStale` <= 604,800s, <= 5 bypass headers, and `negativeCachingPolicy` requiring `negativeCaching`. Duplicate -> 409, missing -> 404. Url-map `defaultService` / `pathMatchers[].defaultService` / `pathRules[].service` references to a missing backend bucket are rejected (400), and deleting a backend bucket a url-map routes to returns 400 `resourceInUseByAnotherResource`. Signed URL keys, IAM policy, and the edge security policy reference itself (no `securityPolicies` resource) are not modeled. | diff --git a/go.mod b/go.mod index 9c83160d1..e590178f4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( cloud.google.com/go/firestore v1.22.0 cloud.google.com/go/iam v1.11.0 cloud.google.com/go/longrunning v1.2.0 + cloud.google.com/go/managedkafka v1.0.0 cloud.google.com/go/secretmanager v1.16.0 cloud.google.com/go/storage v1.62.1 github.com/Azure/azure-kusto-go v0.16.1 diff --git a/go.sum b/go.sum index 8e5cabe6d..cbfbd03e0 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHA cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/managedkafka v1.0.0 h1:wKZdHCPUr3mQDo/9sbPI2+vhXXw9AgC9EfSsZnxbUvY= +cloud.google.com/go/managedkafka v1.0.0/go.mod h1:Phh9R6RQhGCNq36Uifsi+HHFIOlRFUE8Z3Xl2fYelBo= cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= diff --git a/providers/gcp/managedkafka/clone.go b/providers/gcp/managedkafka/clone.go index ba9b4901d..55f7b9979 100644 --- a/providers/gcp/managedkafka/clone.go +++ b/providers/gcp/managedkafka/clone.go @@ -10,10 +10,23 @@ func cloneCluster(c *mkdriver.Cluster) mkdriver.Cluster { out := *c out.Subnets = append([]string(nil), c.Subnets...) out.Labels = cloneStringMap(c.Labels) + out.TLS = cloneTLS(c.TLS) return out } +// cloneTLS deep-copies a TLS config; nil stays nil. +func cloneTLS(in *mkdriver.TLSConfig) *mkdriver.TLSConfig { + if in == nil { + return nil + } + + out := *in + out.CAPools = append([]string(nil), in.CAPools...) + + return &out +} + // cloneTopic returns a deep copy of t. func cloneTopic(t *mkdriver.Topic) mkdriver.Topic { out := *t diff --git a/providers/gcp/managedkafka/managedkafka.go b/providers/gcp/managedkafka/managedkafka.go index 37c952a37..2e73bb32e 100644 --- a/providers/gcp/managedkafka/managedkafka.go +++ b/providers/gcp/managedkafka/managedkafka.go @@ -8,6 +8,7 @@ package managedkafka import ( "context" "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -16,6 +17,7 @@ import ( cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/internal/memstore" + "github.com/stackshy/cloudemu/v2/internal/settle" mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" ) @@ -25,13 +27,32 @@ const ( clustersColl = "clusters" topicsColl = "topics" - // stateActive is the steady state a created cluster reports. Real Managed - // Kafka passes through CREATING first; CloudEmu completes synchronously. - stateActive = "ACTIVE" + // stateActive is the steady state a cluster reports. stateCreating is the + // transient state a new cluster reports for a settle window under + // config.Options.AsyncSettle (real Managed Kafka passes through CREATING); + // with AsyncSettle off (the default) a new cluster is ACTIVE at once. + stateActive = "ACTIVE" + stateCreating = "CREATING" + + // defaultKafkaVersion is the version the real API assigns when a create + // leaves kafkaVersion unset. + defaultKafkaVersion = "3.7.x" + + // apiVersion is the OperationMetadata.apiVersion every operation reports. + apiVersion = "v1" + + // maxOperations bounds the operation store: the oldest operation is evicted + // once more than this many exist, so a long-lived emulator does not grow + // without bound. Every operation is done when returned, so an evicted name + // only matters to a caller that polls it far later (it is then NOT_FOUND, + // as a garbage-collected real operation is). + maxOperations = 1000 opCreate = "create" opUpdate = "update" opDelete = "delete" + + opNameMarker = "/operations/operation-" ) // Mock is the in-memory Managed Kafka control-plane implementation. Clusters and @@ -43,6 +64,10 @@ type Mock struct { topics *memstore.Store[mkdriver.Topic] operations *memstore.Store[mkdriver.Operation] + // creating overlays a transient CREATING window (keyed by cluster name) on + // the stored ACTIVE state; inert unless config.Options.AsyncSettle is set. + creating *settle.Set + opSeq atomic.Uint64 opts *config.Options } @@ -53,6 +78,7 @@ func New(opts *config.Options) *Mock { clusters: memstore.New[mkdriver.Cluster](), topics: memstore.New[mkdriver.Topic](), operations: memstore.New[mkdriver.Operation](), + creating: settle.NewSet(), opts: opts, } } @@ -68,28 +94,80 @@ func topicName(project, location, clusterID, id string) string { } // newOp records a completed operation scoped to the project+location it acted in -// and returns it. The caller holds the write lock. +// and returns it, evicting the oldest operation past maxOperations. The caller +// holds the write lock. func (m *Mock) newOp(project, location, opType, target string) *mkdriver.Operation { + now := m.opts.Clock.Now().UTC() scope := "projects/" + project + "/locations/" + location op := mkdriver.Operation{ - Name: fmt.Sprintf("%s/operations/operation-%d-%s", scope, m.opSeq.Add(1), idgen.UUID()), + Name: fmt.Sprintf("%s%s%d-%s", scope, opNameMarker, m.opSeq.Add(1), idgen.UUID()), Done: true, TargetName: target, Type: opType, + APIVersion: apiVersion, + CreateTime: now, + EndTime: now, } m.operations.Set(op.Name, op) + m.evictOldestOps() return &op } -// CreateCluster validates and stores a new cluster, reporting it ACTIVE, and -// returns the completed LRO. +// evictOldestOps drops the lowest-sequence operations until at most +// maxOperations remain. The caller holds the write lock. +func (m *Mock) evictOldestOps() { + for m.operations.Len() > maxOperations { + oldest, oldestSeq := "", uint64(0) + + for _, k := range m.operations.Keys() { + if seq := opSeqOf(k); oldest == "" || seq < oldestSeq { + oldest, oldestSeq = k, seq + } + } + + m.operations.Delete(oldest) + } +} + +// opSeqOf parses the sequence number out of an operation name +// ".../operations/operation-{seq}-{uuid}"; an unparseable name sorts first. +func opSeqOf(name string) uint64 { + _, rest, ok := strings.Cut(name, opNameMarker) + if !ok { + return 0 + } + + digits, _, _ := strings.Cut(rest, "-") + + seq, err := strconv.ParseUint(digits, 10, 64) + if err != nil { + return 0 + } + + return seq +} + +// observe returns a clone of stored with its settle window overlaid on State. +func (m *Mock) observe(key string, stored *mkdriver.Cluster) mkdriver.Cluster { + out := cloneCluster(stored) + out.State = m.creating.State(key, m.opts.Clock.Now(), out.State) + + return out +} + +// CreateCluster validates and stores a new cluster (defaulting kafkaVersion and +// rebalanceConfig.mode as the real API does) and returns the completed LRO. The +// cluster reports ACTIVE, or CREATING for a settle window under AsyncSettle. func (m *Mock) CreateCluster(_ context.Context, c *mkdriver.Cluster) (*mkdriver.Cluster, *mkdriver.Operation, error) { if err := validateClusterID(c.ID); err != nil { return nil, nil, err } - if err := validateCluster(c); err != nil { + stored := cloneCluster(c) + applyClusterDefaults(&stored) + + if err := validateCluster(&stored); err != nil { return nil, nil, err } @@ -102,14 +180,14 @@ func (m *Mock) CreateCluster(_ context.Context, c *mkdriver.Cluster) (*mkdriver. } now := m.opts.Clock.Now().UTC() - stored := cloneCluster(c) stored.State = stateActive stored.CreateTime = now stored.UpdateTime = now m.clusters.Set(key, stored) + m.creating.Begin(key, stateCreating, now, m.opts.SettleDuration(settle.DefaultClusterSettle)) op := m.newOp(c.Project, c.Location, opCreate, key) - out := cloneCluster(&stored) + out := m.observe(key, &stored) return &out, op, nil } @@ -119,12 +197,14 @@ func (m *Mock) GetCluster(_ context.Context, project, location, id string) (*mkd m.mu.RLock() defer m.mu.RUnlock() - c, ok := m.clusters.Get(clusterName(project, location, id)) + key := clusterName(project, location, id) + + c, ok := m.clusters.Get(key) if !ok { return nil, clusterNotFound(project, location, id) } - out := cloneCluster(&c) + out := m.observe(key, &c) return &out, nil } @@ -139,8 +219,8 @@ func (m *Mock) ListClusters(_ context.Context, project, location string) ([]mkdr out := make([]mkdriver.Cluster, 0, len(all)) for i := range all { - if strings.HasPrefix(clusterName(all[i].Project, all[i].Location, all[i].ID), prefix) { - out = append(out, cloneCluster(&all[i])) + if key := clusterName(all[i].Project, all[i].Location, all[i].ID); strings.HasPrefix(key, prefix) { + out = append(out, m.observe(key, &all[i])) } } @@ -169,6 +249,8 @@ func (m *Mock) UpdateCluster(_ context.Context, c *mkdriver.Cluster, mask []stri return nil, nil, err } + applyClusterDefaults(&next) + if err := validateCluster(&next); err != nil { return nil, nil, err } @@ -177,7 +259,7 @@ func (m *Mock) UpdateCluster(_ context.Context, c *mkdriver.Cluster, mask []stri m.clusters.Set(key, next) op := m.newOp(c.Project, c.Location, opUpdate, key) - out := cloneCluster(&next) + out := m.observe(key, &next) return &out, op, nil } @@ -194,6 +276,7 @@ func (m *Mock) DeleteCluster(_ context.Context, project, location, id string) (* } m.clusters.Delete(key) + m.creating.Clear(key) prefix := key + "/" + topicsColl + "/" for _, k := range m.topics.Keys() { @@ -205,17 +288,15 @@ func (m *Mock) DeleteCluster(_ context.Context, project, location, id string) (* return m.newOp(project, location, opDelete, key), nil } -// GetOperation returns a (done) long-running operation by name. An unknown name -// is reported as a done operation: the mock completes synchronously, so any op -// id a standalone poll asks for has already finished. In an assembled server -// the shared LRO poller answers polls instead, and 404s unknown names. +// GetOperation returns a long-running operation this mock created, by name. An +// unknown (never created, or evicted) name is NOT_FOUND, as in the real API. func (m *Mock) GetOperation(_ context.Context, name string) (*mkdriver.Operation, error) { m.mu.RLock() defer m.mu.RUnlock() op, ok := m.operations.Get(name) if !ok { - return &mkdriver.Operation{Name: name, Done: true}, nil + return nil, cerrors.Newf(cerrors.NotFound, "operation %q not found", name) } return &op, nil diff --git a/providers/gcp/managedkafka/managedkafka_test.go b/providers/gcp/managedkafka/managedkafka_test.go index 75ed783d0..ba3197c24 100644 --- a/providers/gcp/managedkafka/managedkafka_test.go +++ b/providers/gcp/managedkafka/managedkafka_test.go @@ -93,7 +93,7 @@ func TestClusterValidation(t *testing.T) { "bad subnet shape": func(c *mkdriver.Cluster) { c.Subnets = []string{"projects/p/zones/z/subnetworks/s"} }, "bad id": func(c *mkdriver.Cluster) { c.ID = "9starts-with-digit" }, "empty id": func(c *mkdriver.Cluster) { c.ID = "" }, - "bad mode": func(c *mkdriver.Cluster) { c.RebalanceMode = "MODE_UNSPECIFIED" }, + "bad mode": func(c *mkdriver.Cluster) { c.RebalanceMode = "REBALANCE_SOMETIMES" }, } for name, mutate := range cases { @@ -126,7 +126,7 @@ func TestClusterMask(t *testing.T) { } if op.Type != opUpdate || got.VcpuCount != 4 || got.MemoryBytes != 32*gib || len(got.Subnets) != 2 || - got.RebalanceMode != "" || got.Labels["a"] != "b" || got.KmsKey != "k1" { + got.RebalanceMode != rebalanceNone || got.Labels["a"] != "b" || got.KmsKey != "k1" { t.Fatalf("masked update = %+v", got) } diff --git a/providers/gcp/managedkafka/review_fixes_test.go b/providers/gcp/managedkafka/review_fixes_test.go new file mode 100644 index 000000000..a356cad32 --- /dev/null +++ b/providers/gcp/managedkafka/review_fixes_test.go @@ -0,0 +1,389 @@ +package managedkafka + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/settle" + mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" +) + +// TestCreateDefaultsAndOptionalFields: an unset rebalanceConfig.mode defaults to +// NO_REBALANCE and an unset kafkaVersion to 3.7.x (as the real API does), while +// tlsConfig, updateOptions and brokerCapacityConfig are stored and returned. +func TestCreateDefaultsAndOptionalFields(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + got, _, err := m.CreateCluster(ctx, cluster("d1")) + if err != nil { + t.Fatalf("create: %v", err) + } + + if got.RebalanceMode != rebalanceNone || got.KafkaVersion != defaultKafkaVersion { + t.Fatalf("defaults: mode=%q version=%q, want %q/%q", got.RebalanceMode, got.KafkaVersion, + rebalanceNone, defaultKafkaVersion) + } + + unspecified := cluster("d2") + unspecified.RebalanceMode = rebalanceUnspecified + + if got, _, err = m.CreateCluster(ctx, unspecified); err != nil || got.RebalanceMode != rebalanceNone { + t.Fatalf("MODE_UNSPECIFIED create = %+v, %v; want NO_REBALANCE", got, err) + } + + full := cluster("f1") + full.KafkaVersion = "4.3.x" + full.TLS = &mkdriver.TLSConfig{ + SSLPrincipalMappingRules: "RULE:^CN=(.*?),OU=S.*$/$1/,DEFAULT", + CAPools: []string{"projects/other/locations/europe-west1/caPools/pool"}, + } + full.AllowBrokerDownscaleOnClusterUpscale = true + full.BrokerDiskSizeGib = 150 + + if _, _, err = m.CreateCluster(ctx, full); err != nil { + t.Fatalf("create full: %v", err) + } + + full.TLS.CAPools[0] = "mutated-after-create" + + got, err = m.GetCluster(ctx, proj, region, "f1") + if err != nil { + t.Fatalf("get: %v", err) + } + + if got.KafkaVersion != "4.3.x" || got.TLS == nil || + got.TLS.SSLPrincipalMappingRules != "RULE:^CN=(.*?),OU=S.*$/$1/,DEFAULT" || + len(got.TLS.CAPools) != 1 || got.TLS.CAPools[0] != "projects/other/locations/europe-west1/caPools/pool" || + !got.AllowBrokerDownscaleOnClusterUpscale || got.BrokerDiskSizeGib != 150 { + t.Fatalf("stored optional fields = %+v tls=%+v", got, got.TLS) + } +} + +// TestOptionalFieldValidation covers the region, TLS and broker-disk rules. +func TestOptionalFieldValidation(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + pools := make([]string, maxCAPools+1) + for i := range pools { + pools[i] = fmt.Sprintf("projects/p/locations/l/caPools/p%d", i) + } + + cases := map[string]func(c *mkdriver.Cluster){ + "subnet in another region": func(c *mkdriver.Cluster) { + c.Subnets = []string{"projects/p/regions/europe-west1/subnetworks/s"} + }, + "second subnet in another region": func(c *mkdriver.Cluster) { + c.Subnets = []string{subnet, "projects/p/regions/us-east1/subnetworks/s"} + }, + "broker disk under 100": func(c *mkdriver.Cluster) { c.BrokerDiskSizeGib = 99 }, + "bad ca pool": func(c *mkdriver.Cluster) { c.TLS = &mkdriver.TLSConfig{CAPools: []string{"pool"}} }, + "11 ca pools": func(c *mkdriver.Cluster) { c.TLS = &mkdriver.TLSConfig{CAPools: pools} }, + } + + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + c := cluster("v") + mutate(c) + + if _, _, err := m.CreateCluster(ctx, c); !cerrors.IsInvalidArgument(err) { + t.Fatalf("want INVALID_ARGUMENT, got %v", err) + } + }) + } + + // A subnet in another PROJECT but the same region is allowed. + c := cluster("xproj") + c.Subnets = []string{"projects/host-project/regions/" + region + "/subnetworks/shared"} + + if _, _, err := m.CreateCluster(ctx, c); err != nil { + t.Fatalf("cross-project same-region subnet: %v", err) + } +} + +// TestMaskOptionalFields: kafkaVersion is updatable (the API marks it an +// optional input, not immutable), and the tlsConfig, updateOptions and +// brokerCapacityConfig paths apply. +func TestMaskOptionalFields(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + mustCreate(t, m, "c1") + + want := cluster("c1") + want.KafkaVersion = "4.3.x" + want.TLS = &mkdriver.TLSConfig{SSLPrincipalMappingRules: "DEFAULT", CAPools: []string{"projects/a/locations/b/caPools/c"}} + want.AllowBrokerDownscaleOnClusterUpscale = true + want.BrokerDiskSizeGib = 200 + + steps := []struct { + mask []string + check func(c *mkdriver.Cluster) bool + }{ + {[]string{"kafka_version"}, func(c *mkdriver.Cluster) bool { return c.KafkaVersion == "4.3.x" && c.TLS == nil }}, + {[]string{"tlsConfig.sslPrincipalMappingRules"}, func(c *mkdriver.Cluster) bool { + return c.TLS != nil && c.TLS.SSLPrincipalMappingRules == "DEFAULT" && len(c.TLS.CAPools) == 0 + }}, + {[]string{"tls_config.trust_config.cas_configs"}, func(c *mkdriver.Cluster) bool { + return len(c.TLS.CAPools) == 1 && c.TLS.SSLPrincipalMappingRules == "DEFAULT" + }}, + {[]string{"updateOptions"}, func(c *mkdriver.Cluster) bool { return c.AllowBrokerDownscaleOnClusterUpscale }}, + {[]string{"brokerCapacityConfig.diskSizeGib"}, func(c *mkdriver.Cluster) bool { return c.BrokerDiskSizeGib == 200 }}, + } + + for _, s := range steps { + got, _, err := m.UpdateCluster(ctx, want, s.mask) + if err != nil || !s.check(got) { + t.Fatalf("mask %v = %+v tls=%+v, %v", s.mask, got, got.TLS, err) + } + } + + // Clearing tlsConfig with an empty block, and resetting the version with "*", + // which re-applies the 3.7.x default. + clear := cluster("c1") + + got, _, err := m.UpdateCluster(ctx, clear, []string{"tlsConfig", "tlsConfig.trustConfig", "kafkaVersion"}) + if err != nil || got.TLS == nil || len(got.TLS.CAPools) != 0 || got.KafkaVersion != defaultKafkaVersion { + t.Fatalf("clear = %+v tls=%+v, %v", got, got.TLS, err) + } + + bad := cluster("c1") + bad.BrokerDiskSizeGib = 10 + + if _, _, err := m.UpdateCluster(ctx, bad, []string{"brokerCapacityConfig"}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("broker disk 10 via mask: want INVALID_ARGUMENT, got %v", err) + } +} + +// TestGetOperationUnknownIsNotFound: the real API 404s an operation name it +// never issued; a created operation carries its metadata. +func TestGetOperationUnknownIsNotFound(t *testing.T) { + m, clock := newMock(t) + ctx := context.Background() + + if _, err := m.GetOperation(ctx, "projects/p/locations/us-central1/operations/nope"); !cerrors.IsNotFound(err) { + t.Fatalf("unknown op: want NOT_FOUND, got %v", err) + } + + _, op, err := m.CreateCluster(ctx, cluster("c1")) + if err != nil { + t.Fatalf("create: %v", err) + } + + got, err := m.GetOperation(ctx, op.Name) + if err != nil || !got.Done || got.Type != opCreate || got.APIVersion != apiVersion || + got.TargetName != clusterName(proj, region, "c1") || !got.CreateTime.Equal(clock.Now()) || + !got.EndTime.Equal(clock.Now()) { + t.Fatalf("created op = %+v, %v", got, err) + } +} + +// TestOperationStoreIsBounded: the store keeps at most maxOperations, evicting +// the oldest first. +func TestOperationStoreIsBounded(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + mustCreate(t, m, "c1") + + first := "" + + for i := range maxOperations + 5 { + c := cluster("c1") + c.Labels = map[string]string{"i": fmt.Sprint(i)} + + _, op, err := m.UpdateCluster(ctx, c, []string{"labels"}) + if err != nil { + t.Fatalf("update %d: %v", i, err) + } + + if i == 0 { + first = op.Name + } + } + + if n := m.operations.Len(); n != maxOperations { + t.Fatalf("operation store = %d, want %d", n, maxOperations) + } + + if _, err := m.GetOperation(ctx, first); !cerrors.IsNotFound(err) { + t.Fatalf("oldest op should be evicted, got %v", err) + } + + if opSeqOf("no-marker") != 0 || opSeqOf("x/operations/operation-zz-u") != 0 { + t.Fatal("unparseable op names must sort first") + } +} + +// TestAsyncSettleCreatingThenActive: under --async-settle a new cluster reports +// CREATING for the settle window, then ACTIVE; without it, ACTIVE at once. +func TestAsyncSettleCreatingThenActive(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := New(config.NewOptions(config.WithProjectID(proj), config.WithClock(clock), config.WithAsyncSettle())) + ctx := context.Background() + + created, _, err := m.CreateCluster(ctx, cluster("s1")) + if err != nil || created.State != stateCreating { + t.Fatalf("create under async settle = %+v, %v; want CREATING", created, err) + } + + if got, _ := m.GetCluster(ctx, proj, region, "s1"); got.State != stateCreating { + t.Fatalf("get in window = %q, want CREATING", got.State) + } + + if all, _ := m.ListClusters(ctx, proj, region); len(all) != 1 || all[0].State != stateCreating { + t.Fatalf("list in window = %+v", all) + } + + clock.Advance(settle.DefaultClusterSettle) + + if got, _ := m.GetCluster(ctx, proj, region, "s1"); got.State != stateActive { + t.Fatalf("get after window = %q, want ACTIVE", got.State) + } + + if _, err := m.DeleteCluster(ctx, proj, region, "s1"); err != nil { + t.Fatalf("delete: %v", err) + } + + sync, _ := newMock(t) + if got, _, _ := sync.CreateCluster(ctx, cluster("s2")); got.State != stateActive { + t.Fatalf("default create = %q, want ACTIVE", got.State) + } +} + +// TestTopicSurface covers list, get, update masks and delete, including the +// NOT_FOUND and INVALID_ARGUMENT paths. +func TestTopicSurface(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + if _, err := m.ListTopics(ctx, proj, region, "c1"); !cerrors.IsNotFound(err) { + t.Fatalf("list under missing cluster: %v", err) + } + + if err := m.DeleteTopic(ctx, proj, region, "c1", "t"); !cerrors.IsNotFound(err) { + t.Fatalf("delete under missing cluster: %v", err) + } + + if _, err := m.UpdateTopic(ctx, &mkdriver.Topic{Project: proj, Location: region, ClusterID: "c1", ID: "t"}, + []string{"configs"}); !cerrors.IsNotFound(err) { + t.Fatalf("update under missing cluster: %v", err) + } + + if _, err := m.GetTopic(ctx, proj, region, "c1", "t"); !cerrors.IsNotFound(err) { + t.Fatalf("get under missing cluster: %v", err) + } + + mustCreate(t, m, "c1") + mustCreate(t, m, "c2") + + mk := func(cluster, id string) *mkdriver.Topic { + return &mkdriver.Topic{Project: proj, Location: region, ClusterID: cluster, ID: id, PartitionCount: 2, ReplicationFactor: 3} + } + + for _, tp := range []*mkdriver.Topic{mk("c1", "b"), mk("c1", "a"), mk("c2", "z")} { + if _, err := m.CreateTopic(ctx, tp); err != nil { + t.Fatalf("create %s: %v", tp.ID, err) + } + } + + if _, err := m.CreateTopic(ctx, mk("c1", "a")); !cerrors.IsAlreadyExists(err) { + t.Fatalf("duplicate topic: %v", err) + } + + for name, mutate := range map[string]func(*mkdriver.Topic){ + "zero partitions": func(tp *mkdriver.Topic) { tp.PartitionCount = 0 }, + "zero rf": func(tp *mkdriver.Topic) { tp.ReplicationFactor = 0 }, + } { + tp := mk("c1", "v") + mutate(tp) + + if _, err := m.CreateTopic(ctx, tp); !cerrors.IsInvalidArgument(err) { + t.Fatalf("%s: %v", name, err) + } + } + + list, err := m.ListTopics(ctx, proj, region, "c1") + if err != nil || len(list) != 2 || list[0].ID != "a" || list[1].ID != "b" { + t.Fatalf("list = %+v, %v", list, err) + } + + if _, err := m.GetTopic(ctx, proj, region, "c1", "nope"); !cerrors.IsNotFound(err) { + t.Fatalf("get missing topic: %v", err) + } + + withCfg := mk("c1", "a") + withCfg.Configs = map[string]string{"cleanup.policy": "compact"} + + got, err := m.UpdateTopic(ctx, withCfg, []string{"configs"}) + if err != nil || got.Configs["cleanup.policy"] != "compact" { + t.Fatalf("configs update = %+v, %v", got, err) + } + + all := mk("c1", "a") + all.PartitionCount = 5 + + if got, err = m.UpdateTopic(ctx, all, []string{"*"}); err != nil || got.PartitionCount != 5 || got.Configs != nil { + t.Fatalf("* update = %+v, %v", got, err) + } + + for _, bad := range [][]string{nil, {"replication_factor"}, {"name"}, {"nope"}} { + if _, err := m.UpdateTopic(ctx, mk("c1", "a"), bad); !cerrors.IsInvalidArgument(err) { + t.Fatalf("topic mask %v: %v", bad, err) + } + } + + if _, err := m.UpdateTopic(ctx, mk("c1", "ghost"), []string{"configs"}); !cerrors.IsNotFound(err) { + t.Fatalf("update missing topic: %v", err) + } + + zero := mk("c1", "b") + zero.PartitionCount = 0 + + if _, err := m.UpdateTopic(ctx, zero, []string{"partitionCount"}); !cerrors.IsInvalidArgument(err) { + t.Fatalf("shrink to zero: %v", err) + } + + if err := m.DeleteTopic(ctx, proj, region, "c1", "a"); err != nil { + t.Fatalf("delete: %v", err) + } + + if err := m.DeleteTopic(ctx, proj, region, "c1", "a"); !cerrors.IsNotFound(err) { + t.Fatalf("delete twice: %v", err) + } + + if _, err := m.DeleteCluster(ctx, proj, region, "ghost"); !cerrors.IsNotFound(err) { + t.Fatalf("delete missing cluster: %v", err) + } + + if _, err := m.GetCluster(ctx, proj, region, "ghost"); !cerrors.IsNotFound(err) { + t.Fatalf("get missing cluster: %v", err) + } + + if _, _, err := m.CreateCluster(ctx, cluster("c1")); !cerrors.IsAlreadyExists(err) { + t.Fatalf("duplicate cluster: %v", err) + } +} + +// TestRestoreRejectsCorruptSnapshots covers the snapshot error paths. +func TestRestoreRejectsCorruptSnapshots(t *testing.T) { + m, _ := newMock(t) + ctx := context.Background() + + if err := m.Restore(ctx, []byte(`not json`)); err == nil { + t.Fatal("restore of non-JSON must fail") + } + + if err := m.Restore(ctx, []byte(`{"clusters":"not-a-map"}`)); err == nil { + t.Fatal("restore of a corrupt store must fail") + } + + if err := m.Restore(ctx, []byte(`{}`)); err != nil { + t.Fatalf("restore of an empty snapshot: %v", err) + } +} diff --git a/providers/gcp/managedkafka/validate.go b/providers/gcp/managedkafka/validate.go index ff87de6f9..9b1ba78e4 100644 --- a/providers/gcp/managedkafka/validate.go +++ b/providers/gcp/managedkafka/validate.go @@ -24,6 +24,12 @@ const ( minNetworkConfigs = 1 maxNetworkConfigs = 10 + // maxCAPools is the trustConfig.casConfigs limit. + maxCAPools = 10 + + // minBrokerDiskGib is brokerCapacityConfig.diskSizeGib's documented minimum. + minBrokerDiskGib = 100 + // maxTopicIDLen is Apache Kafka's own topic-name length limit. maxTopicIDLen = 249 @@ -33,7 +39,12 @@ const ( pathRebalance = "rebalanceConfig" pathLabels = "labels" pathName = "name" + pathVersion = "kafkaVersion" + pathTLS = "tlsConfig" + pathUpdateOps = "updateOptions" + pathBroker = "brokerCapacityConfig" + rebalanceUnspecified = "MODE_UNSPECIFIED" rebalanceNone = "NO_REBALANCE" rebalanceOnScaleUp = "AUTO_REBALANCE_ON_SCALE_UP" subnetPathSegments = 6 // projects/{p}/regions/{r}/subnetworks/{s} @@ -48,8 +59,25 @@ var ( // topicIDPattern is Apache Kafka's legal topic-name alphabet. topicIDPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + + // caPoolPattern is a CA Service pool name; it need not share the cluster's + // project or location. + caPoolPattern = regexp.MustCompile(`^projects/[^/]+/locations/[^/]+/caPools/[^/]+$`) ) +// applyClusterDefaults fills the fields the real API defaults when a create (or +// a masked update) leaves them unset: rebalanceConfig.mode NO_REBALANCE and +// kafkaVersion 3.7.x. +func applyClusterDefaults(c *mkdriver.Cluster) { + if c.RebalanceMode == "" || c.RebalanceMode == rebalanceUnspecified { + c.RebalanceMode = rebalanceNone + } + + if c.KafkaVersion == "" { + c.KafkaVersion = defaultKafkaVersion + } +} + // validateClusterID enforces the clusterId format. func validateClusterID(id string) error { if id == "" { @@ -77,25 +105,57 @@ func validateTopicID(id string) error { return nil } -// validateCluster checks the required cluster configuration: capacity, network -// access, and the rebalance mode enum. +// validateCluster checks the cluster configuration: capacity, network access +// (subnets in the cluster's region), the rebalance mode enum, TLS trust config +// and per-broker disk. func validateCluster(c *mkdriver.Cluster) error { if err := validateCapacity(c.VcpuCount, c.MemoryBytes); err != nil { return err } - if err := validateSubnets(c.Subnets); err != nil { + if err := validateSubnets(c.Subnets, c.Location); err != nil { + return err + } + + if err := validateTLS(c.TLS); err != nil { return err } + if c.BrokerDiskSizeGib != 0 && c.BrokerDiskSizeGib < minBrokerDiskGib { + return cerrors.Newf(cerrors.InvalidArgument, + "broker_capacity_config.disk_size_gib must be at least %d, got %d", minBrokerDiskGib, c.BrokerDiskSizeGib) + } + switch c.RebalanceMode { - case "", rebalanceNone, rebalanceOnScaleUp: + case rebalanceNone, rebalanceOnScaleUp: return nil default: return cerrors.Newf(cerrors.InvalidArgument, "rebalance_config.mode %q is not a valid mode", c.RebalanceMode) } } +// validateTLS requires at most maxCAPools trust-config CA pools, each a CA +// Service pool name. +func validateTLS(tls *mkdriver.TLSConfig) error { + if tls == nil { + return nil + } + + if len(tls.CAPools) > maxCAPools { + return cerrors.Newf(cerrors.InvalidArgument, + "tls_config.trust_config.cas_configs must contain at most %d entries, got %d", maxCAPools, len(tls.CAPools)) + } + + for _, p := range tls.CAPools { + if !caPoolPattern.MatchString(p) { + return cerrors.Newf(cerrors.InvalidArgument, + "cas_configs.ca_pool %q must be projects/{project}/locations/{location}/caPools/{ca_pool}", p) + } + } + + return nil +} + // validateCapacity enforces vcpuCount >= 3 and 1 GiB..8 GiB of memory per vCPU // (inclusive). func validateCapacity(vcpu, memory int64) error { @@ -118,8 +178,9 @@ func validateCapacity(vcpu, memory int64) error { } // validateSubnets requires 1..10 network configs, each naming a subnet as -// projects/{project}/regions/{region}/subnetworks/{subnet}. -func validateSubnets(subnets []string) error { +// projects/{project}/regions/{region}/subnetworks/{subnet} in the cluster's +// region (the project may differ), as the real API requires. +func validateSubnets(subnets []string, location string) error { if len(subnets) < minNetworkConfigs || len(subnets) > maxNetworkConfigs { return cerrors.Newf(cerrors.InvalidArgument, "gcp_config.access_config.network_configs must contain %d to %d entries, got %d", @@ -131,6 +192,11 @@ func validateSubnets(subnets []string) error { return cerrors.Newf(cerrors.InvalidArgument, "network_configs.subnet %q must be projects/{project}/regions/{region}/subnetworks/{subnet}", s) } + + if region := strings.Split(s, "/")[subnetRegionsIdx+1]; region != location { + return cerrors.Newf(cerrors.InvalidArgument, + "network_configs.subnet %q is in region %q; it must be in the cluster's region %q", s, region, location) + } } return nil @@ -181,6 +247,51 @@ var clusterMaskAppliers = map[string]func(dst, src *mkdriver.Cluster){ pathRebalance: func(dst, src *mkdriver.Cluster) { dst.RebalanceMode = src.RebalanceMode }, "rebalanceConfig.mode": func(dst, src *mkdriver.Cluster) { dst.RebalanceMode = src.RebalanceMode }, pathLabels: func(dst, src *mkdriver.Cluster) { dst.Labels = cloneStringMap(src.Labels) }, + pathVersion: func(dst, src *mkdriver.Cluster) { dst.KafkaVersion = src.KafkaVersion }, + pathTLS: func(dst, src *mkdriver.Cluster) { dst.TLS = cloneTLS(src.TLS) }, + "tlsConfig.sslPrincipalMappingRules": copyPrincipalRules, + "tlsConfig.trustConfig": copyCAPools, + "tlsConfig.trustConfig.casConfigs": copyCAPools, + pathUpdateOps: copyUpdateOptions, + "updateOptions.allowBrokerDownscaleOnClusterUpscale": copyUpdateOptions, + pathBroker: func(dst, src *mkdriver.Cluster) { dst.BrokerDiskSizeGib = src.BrokerDiskSizeGib }, + "brokerCapacityConfig.diskSizeGib": func(dst, src *mkdriver.Cluster) { dst.BrokerDiskSizeGib = src.BrokerDiskSizeGib }, +} + +func copyUpdateOptions(dst, src *mkdriver.Cluster) { + dst.AllowBrokerDownscaleOnClusterUpscale = src.AllowBrokerDownscaleOnClusterUpscale +} + +// copyPrincipalRules sets tlsConfig.sslPrincipalMappingRules, creating the TLS +// block if the cluster had none. +func copyPrincipalRules(dst, src *mkdriver.Cluster) { + rules := "" + if src.TLS != nil { + rules = src.TLS.SSLPrincipalMappingRules + } + + dst.TLS = ensureTLS(dst.TLS) + dst.TLS.SSLPrincipalMappingRules = rules +} + +// copyCAPools sets tlsConfig.trustConfig.casConfigs, creating the TLS block if +// the cluster had none. +func copyCAPools(dst, src *mkdriver.Cluster) { + var pools []string + if src.TLS != nil { + pools = append([]string(nil), src.TLS.CAPools...) + } + + dst.TLS = ensureTLS(dst.TLS) + dst.TLS.CAPools = pools +} + +func ensureTLS(t *mkdriver.TLSConfig) *mkdriver.TLSConfig { + if t == nil { + return &mkdriver.TLSConfig{} + } + + return t } // clusterFixedPaths are cluster field-mask paths that exist on the resource but @@ -190,7 +301,7 @@ var clusterMaskAppliers = map[string]func(dst, src *mkdriver.Cluster){ var clusterFixedPaths = map[string]bool{ pathName: true, "state": true, "createTime": true, "updateTime": true, "satisfiesPzi": true, "satisfiesPzs": true, "gcpConfig.kmsKey": true, - "brokerDetails": true, "kafkaVersion": true, + "brokerDetails": true, } func copySubnets(dst, src *mkdriver.Cluster) { dst.Subnets = append([]string(nil), src.Subnets...) } @@ -234,7 +345,7 @@ func applyAllCluster(dst, src *mkdriver.Cluster, path string) { return } - for _, p := range []string{pathCapacity, pathRebalance, pathLabels} { + for _, p := range []string{pathCapacity, pathRebalance, pathLabels, pathVersion, pathTLS, pathUpdateOps, pathBroker} { clusterMaskAppliers[p](dst, src) } } diff --git a/server/gcp/cluster_siblings.go b/server/gcp/cluster_siblings.go new file mode 100644 index 000000000..eb60d5889 --- /dev/null +++ b/server/gcp/cluster_siblings.go @@ -0,0 +1,50 @@ +package gcp + +import ( + "context" + + gkeprov "github.com/stackshy/cloudemu/v2/providers/gcp/gke" + managedkafkasrv "github.com/stackshy/cloudemu/v2/server/gcp/managedkafka" + rdbdriver "github.com/stackshy/cloudemu/v2/services/relationaldb/driver" +) + +// GKE, AlloyDB and Managed Kafka all serve +// /v1/projects/{p}/locations/{l}/clusters[/{c}]. These adapters give the +// Managed Kafka handler a read-only view of whichever of GKE / AlloyDB is +// enabled (never both; see New), so it routes that collection by ownership. Each +// answers with the same scoping its own handler serves: GKE clusters are keyed +// by location (the GKE mock is project-agnostic), and AlloyDB's handler lists +// every cluster it holds for any project+location. + +var ( + _ managedkafkasrv.ClusterSibling = gkeClusterSibling{} + _ managedkafkasrv.ClusterSibling = alloyDBClusterSibling{} +) + +type gkeClusterSibling struct{ m *gkeprov.Mock } + +func (s gkeClusterSibling) HasClusters(ctx context.Context, _, location string) bool { + all, err := s.m.ListClusters(ctx, location) + + return err == nil && len(all) > 0 +} + +func (s gkeClusterSibling) OwnsCluster(ctx context.Context, _, location, id string) bool { + _, err := s.m.GetCluster(ctx, location, id) + + return err == nil +} + +type alloyDBClusterSibling struct{ db rdbdriver.RelationalDB } + +func (s alloyDBClusterSibling) HasClusters(ctx context.Context, _, _ string) bool { + all, err := s.db.DescribeClusters(ctx, nil) + + return err == nil && len(all) > 0 +} + +func (s alloyDBClusterSibling) OwnsCluster(ctx context.Context, _, _, id string) bool { + found, err := s.db.DescribeClusters(ctx, []string{id}) + + return err == nil && len(found) > 0 +} diff --git a/server/gcp/gcp.go b/server/gcp/gcp.go index f098f49c9..1e3ff646f 100644 --- a/server/gcp/gcp.go +++ b/server/gcp/gcp.go @@ -401,15 +401,25 @@ func New(d Drivers) *server.Server { // Managed Kafka shares the exact /v1/projects/{p}/locations/{l}/clusters[/…] // grammar with GKE and AlloyDB (all greedy on that collection), so it - // registers AHEAD of both and its Matches claims only genuinely-Kafka traffic: - // a create body carrying capacityConfig/gcpConfig, an item/list it owns, or - // the Kafka-only clusters/{c}/topics sub-collection. Everything else falls - // through. Its registry is wired below, once the shared LRO poller exists, - // which also makes it yield location operation polls to that poller. + // registers AHEAD of both and its Matches claims only genuinely-Kafka traffic, + // routed by ownership against whichever of GKE / AlloyDB is enabled: a + // Kafka-shaped create, an item it owns, a list only where the sibling owns no + // cluster, or the Kafka-only clusters/{c}/topics sub-collection. Everything + // else falls through. Its registry is wired below, once the shared LRO + // poller exists, which also makes it yield location operation polls to that + // poller. var kafkaH *managedkafkasrv.Handler if d.ManagedKafka != nil { kafkaH = managedkafkasrv.New(d.ManagedKafka) + + switch { + case d.GKE != nil: + kafkaH.SetClusterSibling(gkeClusterSibling{m: d.GKE}) + case d.AlloyDB != nil: + kafkaH.SetClusterSibling(alloyDBClusterSibling{db: d.AlloyDB}) + } + srv.Register(kafkaH) } diff --git a/server/gcp/managedkafka/handler.go b/server/gcp/managedkafka/handler.go index c709f72c2..598ed7222 100644 --- a/server/gcp/managedkafka/handler.go +++ b/server/gcp/managedkafka/handler.go @@ -21,22 +21,37 @@ // to GKE's (container/v1) and AlloyDB's cluster paths, and a custom-endpoint // client sends the emulator's own Host, so URL alone cannot tell them apart. In // an assembled server (a shared LRO registry is wired) this handler registers -// AHEAD of GKE/AlloyDB and claims a cluster request only when it is genuinely -// Managed Kafka traffic, the Filestore/Spanner content+ownership pattern: +// AHEAD of GKE/AlloyDB and is told which of them shares the path +// (SetClusterSibling). It claims a cluster request only when it is genuinely +// Managed Kafka traffic: // -// - a create whose body carries capacityConfig or gcpConfig (GKE wraps its -// body in {"cluster": …}; AlloyDB bodies carry neither key); +// - a create whose body carries a Kafka-only key (capacityConfig, gcpConfig, +// …; GKE wraps its body in {"cluster": …} and AlloyDB bodies carry none); +// - a GKE/AlloyDB-shaped create naming an id this store owns, which is +// refused 409 rather than silently shadowing one cluster with the other; // - an item request for a cluster this store owns; -// - a list in a project+location where this store owns at least one cluster; +// - a PATCH whose body is Kafka-shaped, for a cluster no sibling owns, so a +// missing Kafka cluster is Kafka's 404 (GKE has no PATCH and would 405); +// - a list in a project+location where this store owns a cluster and the +// sibling owns none. When both own clusters there the list is the +// sibling's: the response shapes differ and the request carries nothing +// that says which service it is for, so the pre-existing service keeps its +// list rather than having it replaced; // - anything under clusters/{c}/topics (no sibling service has topics). // +// A create is refused 409 ALREADY_EXISTS when the sibling already owns that id +// in the location, since one of the two clusters would be unreachable. (Real +// GCP keeps the services' namespaces apart by hostname; the emulator cannot.) +// // Everything else falls through to GKE/AlloyDB. Operation polls are yielded to -// the shared LRO poller. A standalone package server (no registry) claims every -// clusters/topics/operations path. +// the shared LRO poller. A standalone package server (no shared registry) +// claims every clusters/topics/operations path and answers operation polls +// from its own private registry. package managedkafka import ( "bytes" + "context" "encoding/json" "io" "net/http" @@ -65,26 +80,74 @@ const ( restTopic = 4 // [clusters, {c}, topics, {t}] maxProbeBytes = 1 << 20 + + // allowMissingParam is AlloyDB's upsert flag on PATCH clusters/{c}; Kafka's + // UpdateCluster has none, so a PATCH carrying it is never Kafka's. + allowMissingParam = "allowMissing" ) +// kafkaOnlyKeys are top-level Cluster fields only a Managed Kafka body carries +// (a GKE CreateClusterRequest wraps its cluster in {"cluster": …}; an AlloyDB +// Cluster has none of these). +// +//nolint:gochecknoglobals // immutable lookup set +var kafkaOnlyKeys = []string{ + "capacityConfig", "gcpConfig", "rebalanceConfig", "tlsConfig", + "updateOptions", "brokerCapacityConfig", "kafkaVersion", +} + +// ClusterSibling is the view of the service that shares the +// /v1/projects/{p}/locations/{l}/clusters collection in an assembled server +// (GKE or AlloyDB), used to route that collection by ownership. It is wired the +// way the load balancer's BucketLister is: a narrow read-only probe the +// assembling server adapts from the sibling's driver. +type ClusterSibling interface { + // HasClusters reports whether the sibling serves any cluster for a list of + // project+location. + HasClusters(ctx context.Context, project, location string) bool + // OwnsCluster reports whether the sibling serves cluster id at + // project+location. + OwnsCluster(ctx context.Context, project, location, id string) bool +} + // Handler serves managedkafka.googleapis.com v1 requests against a ManagedKafka // driver. type Handler struct { db mkdriver.ManagedKafka - // ops records created operations with the shared poller. Nil in a standalone - // package server, where this handler serves its own /operations/ poll and - // claims every clusters path. + // ops records created operations. It is the shared poller's registry in an + // assembled server, or a private one in a standalone package server. ops *lro.Registry + + // poller answers operation polls from ops in a standalone package server; + // nil once the shared registry is wired (the shared poller answers them). + poller *lro.Handler + + // sibling is the service sharing the clusters path; nil when none is wired. + sibling ClusterSibling } -// New returns a Managed Kafka handler backed by db. -func New(db mkdriver.ManagedKafka) *Handler { return &Handler{db: db} } +// New returns a standalone Managed Kafka handler backed by db: it claims every +// Managed Kafka path and answers operation polls itself. +func New(db mkdriver.ManagedKafka) *Handler { + reg := lro.NewRegistry() + + return &Handler{db: db, ops: reg, poller: lro.New(reg)} +} // SetOperationRegistry wires the shared LRO poller so created operations are -// resolvable (with their response) through the full server's operations route, -// and switches Matches to the content+ownership mode an assembled server needs. -func (h *Handler) SetOperationRegistry(reg *lro.Registry) { h.ops = reg } +// resolvable (with their response and metadata) through the full server's +// operations route, and switches Matches to the content+ownership mode an +// assembled server needs. +func (h *Handler) SetOperationRegistry(reg *lro.Registry) { + h.ops = reg + h.poller = nil +} + +// SetClusterSibling wires the service (GKE or AlloyDB) that serves the same +// clusters collection, so Matches and create route by ownership instead of +// shadowing its clusters. +func (h *Handler) SetClusterSibling(s ClusterSibling) { h.sibling = s } // route holds the parsed components of a Managed Kafka v1 path. type route struct { @@ -145,72 +208,159 @@ func (rt *route) setRest(rest []string) bool { // Matches claims Managed Kafka paths. See the package doc for how it shares the // clusters path with GKE and AlloyDB in an assembled server. func (h *Handler) Matches(r *http.Request) bool { + if h.poller != nil && h.poller.Matches(r) { + return true + } + rt, ok := parseRoute(r.URL.Path) if !ok { return false } - standalone := h.ops == nil + standalone := h.poller != nil switch { case rt.resource == operationsSeg: - return standalone + return false case standalone || rt.topics: return true case rt.cluster != "": - _, err := h.db.GetCluster(r.Context(), rt.project, rt.location, rt.cluster) - - return err == nil + return h.claimsItem(r, &rt) case r.Method == http.MethodPost: - return bodyLooksLikeKafka(r) + probe := probeBody(r) + + return isKafkaBody(probe) || h.owns(r.Context(), &rt, foreignCreateID(r, probe)) case r.Method == http.MethodGet: all, err := h.db.ListClusters(r.Context(), rt.project, rt.location) - return err == nil && len(all) > 0 + return err == nil && len(all) > 0 && + (h.sibling == nil || !h.sibling.HasClusters(r.Context(), rt.project, rt.location)) default: return false } } -// bodyLooksLikeKafka reports whether a POST .../clusters body is a Managed Kafka -// Cluster (it carries capacityConfig or gcpConfig) rather than a GKE -// CreateClusterRequest or an AlloyDB Cluster. It reads and restores the body so -// a fall-through handler still sees the full request. -func bodyLooksLikeKafka(r *http.Request) bool { - if r.Body == nil { +// claimsItem decides an item request (clusters/{c}) in an assembled server: a +// cluster this store owns is Kafka's, one the sibling owns is not, and for an +// id nobody owns a Kafka-shaped PATCH is claimed so it gets Kafka's 404. +func (h *Handler) claimsItem(r *http.Request, rt *route) bool { + if h.owns(r.Context(), rt, rt.cluster) { + return true + } + + if h.siblingOwns(r.Context(), rt, rt.cluster) || r.Method != http.MethodPatch || + r.URL.Query().Has(allowMissingParam) { + return false + } + + return isKafkaPatchBody(probeBody(r)) +} + +// owns reports whether this store has cluster id at rt's project+location. +func (h *Handler) owns(ctx context.Context, rt *route, id string) bool { + if id == "" { return false } + _, err := h.db.GetCluster(ctx, rt.project, rt.location, id) + + return err == nil +} + +// siblingOwns reports whether the wired sibling serves cluster id at rt's +// project+location. +func (h *Handler) siblingOwns(ctx context.Context, rt *route, id string) bool { + return h.sibling != nil && h.sibling.OwnsCluster(ctx, rt.project, rt.location, id) +} + +// probeBody reads a request body's top-level JSON object and restores the body +// so a fall-through handler still sees the full request. A missing or non-object +// body probes as nil. +func probeBody(r *http.Request) map[string]json.RawMessage { + if r.Body == nil { + return nil + } + raw, err := io.ReadAll(io.LimitReader(r.Body, maxProbeBytes)) _ = r.Body.Close() r.Body = io.NopCloser(bytes.NewReader(raw)) if err != nil { - return false + return nil } var probe map[string]json.RawMessage if json.Unmarshal(raw, &probe) != nil { + return nil + } + + return probe +} + +// isKafkaBody reports whether a probed body carries a Kafka-only Cluster key. +func isKafkaBody(probe map[string]json.RawMessage) bool { + for _, k := range kafkaOnlyKeys { + if _, ok := probe[k]; ok { + return true + } + } + + return false +} + +// isKafkaPatchBody reports whether a probed PATCH body is a Kafka Cluster: a +// Kafka-only key, or a labels-only body (a Kafka labels update), which carries +// no GKE {"cluster"/"update": …} wrapper. +func isKafkaPatchBody(probe map[string]json.RawMessage) bool { + if isKafkaBody(probe) { + return true + } + + if _, ok := probe["labels"]; !ok { return false } - _, hasCapacity := probe["capacityConfig"] - _, hasGcp := probe["gcpConfig"] + for k := range probe { + if k != "labels" && k != "name" { + return false + } + } - return hasCapacity || hasGcp + return true +} + +// foreignCreateID returns the cluster id a non-Kafka create names: AlloyDB's +// ?clusterId= or GKE's {"cluster":{"name": …}}. +func foreignCreateID(r *http.Request, probe map[string]json.RawMessage) string { + if id := r.URL.Query().Get(clusterIDParam); id != "" { + return id + } + + var gke struct { + Name string `json:"name"` + } + + if raw, ok := probe["cluster"]; ok && json.Unmarshal(raw, &gke) == nil { + return gke.Name + } + + return "" } // ServeHTTP routes on the parsed path and method. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if h.poller != nil && h.poller.Matches(r) { + h.poller.ServeHTTP(w, r) + return + } + rt, ok := parseRoute(r.URL.Path) - if !ok { + if !ok || rt.resource == operationsSeg { gcprest.WriteError(w, http.StatusNotFound, "notFound", "unrecognized Managed Kafka path") return } switch { - case rt.resource == operationsSeg: - h.serveOperation(w, r) case rt.topics && rt.topic == "": h.serveTopicCollection(w, r, &rt) case rt.topics: diff --git a/server/gcp/managedkafka/operations.go b/server/gcp/managedkafka/operations.go index c5bda89ba..11e6aed0a 100644 --- a/server/gcp/managedkafka/operations.go +++ b/server/gcp/managedkafka/operations.go @@ -21,14 +21,32 @@ const ( ) // createCluster handles POST .../clusters?clusterId=. Validation (clusterId -// format, capacity, network configs) lives in the driver; a failure is 400. +// format, capacity, network configs) lives in the driver; a failure is 400. In +// an assembled server an id already used on the shared clusters path by the +// other service is 409 ALREADY_EXISTS, in either direction: a Kafka create +// naming a GKE/AlloyDB cluster's id, or a GKE/AlloyDB create (claimed only for +// this) naming a Kafka cluster's id. func (h *Handler) createCluster(w http.ResponseWriter, r *http.Request, rt *route) { + id := r.URL.Query().Get(clusterIDParam) + + if h.poller == nil { + if probe := probeBody(r); !isKafkaBody(probe) { + writeIDTaken(w, rt, foreignCreateID(r, probe), "a Managed Kafka") + return + } + + if h.siblingOwns(r.Context(), rt, id) { + writeIDTaken(w, rt, id, "another service's (GKE or AlloyDB)") + return + } + } + var body clusterJSON - if !decodeBody(w, r, &body) { + if !gcprest.DecodeOptionalJSON(w, r, &body) { return } - c, op, err := h.db.CreateCluster(r.Context(), toDriverCluster(&body, rt, r.URL.Query().Get(clusterIDParam))) + c, op, err := h.db.CreateCluster(r.Context(), toDriverCluster(&body, rt, id)) if err != nil { gcprest.WriteCErr(w, err) return @@ -78,7 +96,7 @@ func (h *Handler) listClusters(w http.ResponseWriter, r *http.Request, rt *route // change; unknown, immutable and output-only paths are 400. func (h *Handler) updateCluster(w http.ResponseWriter, r *http.Request, rt *route) { var body clusterJSON - if !decodeBody(w, r, &body) { + if !gcprest.DecodeOptionalJSON(w, r, &body) { return } @@ -106,7 +124,7 @@ func (h *Handler) deleteCluster(w http.ResponseWriter, r *http.Request, rt *rout // createTopic handles POST .../topics?topicId=; synchronous, returns the Topic. func (h *Handler) createTopic(w http.ResponseWriter, r *http.Request, rt *route) { var body topicJSON - if !decodeBody(w, r, &body) { + if !gcprest.DecodeOptionalJSON(w, r, &body) { return } @@ -159,7 +177,7 @@ func (h *Handler) listTopics(w http.ResponseWriter, r *http.Request, rt *route) // updateTopic handles PATCH .../topics/{t}?updateMask=; synchronous. func (h *Handler) updateTopic(w http.ResponseWriter, r *http.Request, rt *route) { var body topicJSON - if !decodeBody(w, r, &body) { + if !gcprest.DecodeOptionalJSON(w, r, &body) { return } @@ -183,21 +201,14 @@ func (h *Handler) deleteTopic(w http.ResponseWriter, r *http.Request, rt *route) gcprest.WriteJSON(w, http.StatusOK, map[string]any{}) } -// serveOperation resolves a (done) operation poll for a standalone package -// server (no shared registry). -func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - writeMethodNotAllowed(w) - return - } - - op, err := h.db.GetOperation(r.Context(), strings.TrimPrefix(r.URL.Path, "/v1/")) - if err != nil { - gcprest.WriteCErr(w, err) - return - } - - gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true}) +// writeIDTaken writes the 409 for a cluster id another service already uses +// on the shared clusters path. Real GCP keeps GKE, AlloyDB and Managed Kafka +// apart by hostname; the emulator serves them on one path, so it refuses the +// second cluster rather than making one of the two unreachable. +func writeIDTaken(w http.ResponseWriter, rt *route, id, owner string) { + gcprest.WriteError(w, http.StatusConflict, "alreadyExists", + "cluster "+clusterName(rt.project, rt.location, id)+" already exists: the id is used by "+owner+ + " cluster on this emulator, which serves GKE, AlloyDB and Managed Kafka clusters on one path") } // parseMask splits a comma-separated updateMask query param into field paths. diff --git a/server/gcp/managedkafka/sdk_gapic_test.go b/server/gcp/managedkafka/sdk_gapic_test.go new file mode 100644 index 000000000..12f217b9a --- /dev/null +++ b/server/gcp/managedkafka/sdk_gapic_test.go @@ -0,0 +1,161 @@ +package managedkafka_test + +import ( + "context" + "net/http/httptest" + "testing" + + mkapi "cloud.google.com/go/managedkafka/apiv1" + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + "google.golang.org/protobuf/types/known/fieldmaskpb" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +// newGAPICClient returns the official cloud.google.com/go/managedkafka REST +// client against a full assembled server. The GAPIC REST transport marshals +// enums as numbers ($alt=json;enum-encoding=int, protojson UseEnumNumbers), the +// shape the discovery client never sends. +func newGAPICClient(t *testing.T) *mkapi.Client { + t.Helper() + + srv := gcpserver.NewFromProvider(cloudemu.NewGCP(config.WithClock(config.NewFakeClock(epoch)), + config.WithProjectID(project))) + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + c, err := mkapi.NewRESTClient(context.Background(), + option.WithEndpoint(ts.URL), + option.WithoutAuthentication(), + option.WithHTTPClient(ts.Client()), + ) + if err != nil { + t.Fatalf("managedkafka.NewRESTClient: %v", err) + } + + t.Cleanup(func() { _ = c.Close() }) + + return c +} + +func gapicCluster(mode managedkafkapb.RebalanceConfig_Mode) *managedkafkapb.Cluster { + return &managedkafkapb.Cluster{ + CapacityConfig: &managedkafkapb.CapacityConfig{VcpuCount: 3, MemoryBytes: 3 * gib}, + PlatformConfig: &managedkafkapb.Cluster_GcpConfig{GcpConfig: &managedkafkapb.GcpConfig{ + AccessConfig: &managedkafkapb.AccessConfig{ + NetworkConfigs: []*managedkafkapb.NetworkConfig{{Subnet: subnet}}, + }, + }}, + RebalanceConfig: &managedkafkapb.RebalanceConfig{Mode: mode}, + Labels: map[string]string{"env": "gapic"}, + } +} + +// TestGAPICClusterWithNumericEnums drives the official GAPIC REST client +// through create (Wait), get, list, update of a fetched cluster (which sends +// its output-only state back as a number) and delete (Wait). On the unfixed +// wire, CreateCluster with a RebalanceConfig 400s "cannot unmarshal number into +// ... rebalanceConfig.mode". +func TestGAPICClusterWithNumericEnums(t *testing.T) { + ctx := context.Background() + c := newGAPICClient(t) + + op, err := c.CreateCluster(ctx, &managedkafkapb.CreateClusterRequest{ + Parent: parent, + ClusterId: "gapic1", + Cluster: gapicCluster(managedkafkapb.RebalanceConfig_AUTO_REBALANCE_ON_SCALE_UP), + }) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + created, err := op.Wait(ctx) + if err != nil { + t.Fatalf("CreateCluster Wait: %v", err) + } + + if created.GetRebalanceConfig().GetMode() != managedkafkapb.RebalanceConfig_AUTO_REBALANCE_ON_SCALE_UP || + created.GetState() != managedkafkapb.Cluster_ACTIVE || created.GetName() != parent+"/clusters/gapic1" { + t.Fatalf("created = %v", created) + } + + meta, err := op.Metadata() + if err != nil || meta.GetVerb() != "create" || meta.GetTarget() != parent+"/clusters/gapic1" || + meta.GetApiVersion() != "v1" || meta.GetCreateTime() == nil || meta.GetEndTime() == nil { + t.Fatalf("create metadata = %v, %v", meta, err) + } + + got, err := c.GetCluster(ctx, &managedkafkapb.GetClusterRequest{Name: created.GetName()}) + if err != nil { + t.Fatalf("GetCluster: %v", err) + } + + // A cluster created without a rebalanceConfig reports the NO_REBALANCE + // default; MODE_UNSPECIFIED (0) is omitted on the wire, so it defaults too. + defOp, err := c.CreateCluster(ctx, &managedkafkapb.CreateClusterRequest{ + Parent: parent, ClusterId: "gapic2", + Cluster: gapicCluster(managedkafkapb.RebalanceConfig_MODE_UNSPECIFIED), + }) + if err != nil { + t.Fatalf("CreateCluster default mode: %v", err) + } + + def, err := defOp.Wait(ctx) + if err != nil || def.GetRebalanceConfig().GetMode() != managedkafkapb.RebalanceConfig_NO_REBALANCE { + t.Fatalf("default mode = %v, %v", def.GetRebalanceConfig(), err) + } + + it := c.ListClusters(ctx, &managedkafkapb.ListClustersRequest{Parent: parent}) + + var names []string + + for { + cl, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + t.Fatalf("ListClusters: %v", err) + } + + names = append(names, cl.GetName()) + } + + if len(names) != 2 { + t.Fatalf("ListClusters = %v, want 2", names) + } + + // Round-trip the fetched cluster (state is set, sent as a number) with a + // numeric rebalance mode change. + got.RebalanceConfig.Mode = managedkafkapb.RebalanceConfig_NO_REBALANCE + got.Labels = map[string]string{"env": "updated"} + + upOp, err := c.UpdateCluster(ctx, &managedkafkapb.UpdateClusterRequest{ + Cluster: got, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"rebalance_config", "labels"}}, + }) + if err != nil { + t.Fatalf("UpdateCluster: %v", err) + } + + updated, err := upOp.Wait(ctx) + if err != nil || updated.GetRebalanceConfig().GetMode() != managedkafkapb.RebalanceConfig_NO_REBALANCE || + updated.GetLabels()["env"] != "updated" { + t.Fatalf("updated = %v, %v", updated, err) + } + + delOp, err := c.DeleteCluster(ctx, &managedkafkapb.DeleteClusterRequest{Name: created.GetName()}) + if err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if err := delOp.Wait(ctx); err != nil { + t.Fatalf("DeleteCluster Wait: %v", err) + } +} diff --git a/server/gcp/managedkafka/wire.go b/server/gcp/managedkafka/wire.go index 2b1f2515e..b82ae0d2d 100644 --- a/server/gcp/managedkafka/wire.go +++ b/server/gcp/managedkafka/wire.go @@ -3,22 +3,36 @@ package managedkafka import ( "encoding/json" "errors" - "io" + "fmt" "net/http" "strconv" "strings" - "time" "github.com/stackshy/cloudemu/v2/server/wire/gcprest" mkdriver "github.com/stackshy/cloudemu/v2/services/managedkafka/driver" ) const ( - clusterTypeURL = "type.googleapis.com/google.cloud.managedkafka.v1.Cluster" - emptyTypeURL = "type.googleapis.com/google.protobuf.Empty" + clusterTypeURL = "type.googleapis.com/google.cloud.managedkafka.v1.Cluster" + emptyTypeURL = "type.googleapis.com/google.protobuf.Empty" + opMetaTypeURL = "type.googleapis.com/google.cloud.managedkafka.v1.OperationMetadata" + int64Base = 10 + int64Bits = 64 + jsonNull = "null" + jsonQuote = '"' + enumUnspecified = 0 +) + +// Proto enum name tables, indexed by the enum number (google.cloud.managedkafka.v1). +// Index 0 is the *_UNSPECIFIED value. +// +//nolint:gochecknoglobals // immutable ordinal enum tables +var ( + rebalanceModeNames = []string{"MODE_UNSPECIFIED", "NO_REBALANCE", "AUTO_REBALANCE_ON_SCALE_UP"} + clusterStateNames = []string{"STATE_UNSPECIFIED", "CREATING", "ACTIVE", "DELETING", "UPDATING"} - int64Base = 10 - int64Bits = 64 + errEnumValue = errors.New("enum value is neither a name nor a number") + errEnumRange = errors.New("enum number is out of range") ) // int64String is a proto3-JSON int64: marshaled as a decimal string, and @@ -45,6 +59,66 @@ func (v *int64String) UnmarshalJSON(b []byte) error { return nil } +// decodeEnum reads a proto3-JSON enum, which may be the value's name (a JSON +// string, as the discovery client, gcloud and Terraform send) or its number (as +// the GAPIC REST client sends under UseEnumNumbers). A number resolves through +// names; 0 (the *_UNSPECIFIED value) and null decode to "" (unset). An unknown +// number is an error, surfaced as 400 by the body decoder. A name is kept +// verbatim so the provider validates it. +func decodeEnum(b []byte, names []string) (string, error) { + s := strings.TrimSpace(string(b)) + if s == jsonNull { + return "", nil + } + + if s != "" && s[0] == jsonQuote { + var name string + err := json.Unmarshal(b, &name) + + return name, err + } + + n, err := strconv.Atoi(s) + if err != nil { + return "", fmt.Errorf("%w: %s", errEnumValue, s) + } + + if n == enumUnspecified { + return "", nil + } + + if n < 0 || n >= len(names) { + return "", fmt.Errorf("%w: %d", errEnumRange, n) + } + + return names[n], nil +} + +// rebalanceMode is RebalanceConfig.mode (name or number on input, name on +// output). +type rebalanceMode string + +// UnmarshalJSON accepts the mode's name or number. +func (m *rebalanceMode) UnmarshalJSON(b []byte) error { + s, err := decodeEnum(b, rebalanceModeNames) + *m = rebalanceMode(s) + + return err +} + +// clusterState is Cluster.state: output only, but a client that round-trips a +// fetched cluster sends it back (the GAPIC client as a number), so it must +// decode either way. Its input value is ignored. +type clusterState string + +// UnmarshalJSON accepts the state's name or number. +func (s *clusterState) UnmarshalJSON(b []byte) error { + v, err := decodeEnum(b, clusterStateNames) + *s = clusterState(v) + + return err +} + type capacityJSON struct { VcpuCount int64String `json:"vcpuCount,omitempty"` MemoryBytes int64String `json:"memoryBytes,omitempty"` @@ -64,22 +138,47 @@ type gcpConfigJSON struct { } type rebalanceJSON struct { - Mode string `json:"mode,omitempty"` + Mode rebalanceMode `json:"mode,omitempty"` +} + +type casConfigJSON struct { + CaPool string `json:"caPool,omitempty"` +} + +type trustConfigJSON struct { + CasConfigs []casConfigJSON `json:"casConfigs,omitempty"` +} + +type tlsConfigJSON struct { + SslPrincipalMappingRules string `json:"sslPrincipalMappingRules,omitempty"` + TrustConfig *trustConfigJSON `json:"trustConfig,omitempty"` +} + +type updateOptionsJSON struct { + AllowBrokerDownscaleOnClusterUpscale bool `json:"allowBrokerDownscaleOnClusterUpscale,omitempty"` +} + +type brokerCapacityJSON struct { + DiskSizeGib int64String `json:"diskSizeGib,omitempty"` } // clusterJSON mirrors the managedkafka v1 Cluster message. Output-only fields // (name, state, createTime, updateTime, satisfiesPzi/Pzs) are ignored on input. type clusterJSON struct { - Name string `json:"name,omitempty"` - CapacityConfig *capacityJSON `json:"capacityConfig,omitempty"` - GcpConfig *gcpConfigJSON `json:"gcpConfig,omitempty"` - RebalanceConfig *rebalanceJSON `json:"rebalanceConfig,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - State string `json:"state,omitempty"` - CreateTime string `json:"createTime,omitempty"` - UpdateTime string `json:"updateTime,omitempty"` - SatisfiesPzi bool `json:"satisfiesPzi,omitempty"` - SatisfiesPzs bool `json:"satisfiesPzs,omitempty"` + Name string `json:"name,omitempty"` + CapacityConfig *capacityJSON `json:"capacityConfig,omitempty"` + GcpConfig *gcpConfigJSON `json:"gcpConfig,omitempty"` + RebalanceConfig *rebalanceJSON `json:"rebalanceConfig,omitempty"` + KafkaVersion string `json:"kafkaVersion,omitempty"` + TLSConfig *tlsConfigJSON `json:"tlsConfig,omitempty"` + UpdateOptions *updateOptionsJSON `json:"updateOptions,omitempty"` + BrokerCapacityConfig *brokerCapacityJSON `json:"brokerCapacityConfig,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + State clusterState `json:"state,omitempty"` + CreateTime string `json:"createTime,omitempty"` + UpdateTime string `json:"updateTime,omitempty"` + SatisfiesPzi bool `json:"satisfiesPzi,omitempty"` + SatisfiesPzs bool `json:"satisfiesPzs,omitempty"` } // topicJSON mirrors the managedkafka v1 Topic message (int32 counts are plain @@ -95,17 +194,28 @@ type topicJSON struct { // inline, so `done` is always true. type operationJSON struct { Name string `json:"name"` + Metadata json.RawMessage `json:"metadata,omitempty"` Done bool `json:"done"` Response json.RawMessage `json:"response,omitempty"` } +// operationMetadataJSON mirrors google.cloud.managedkafka.v1.OperationMetadata. +type operationMetadataJSON struct { + CreateTime string `json:"createTime,omitempty"` + EndTime string `json:"endTime,omitempty"` + Target string `json:"target,omitempty"` + Verb string `json:"verb,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` +} + // toDriverCluster converts a request body into a driver cluster scoped to rt. func toDriverCluster(in *clusterJSON, rt *route, id string) *mkdriver.Cluster { c := &mkdriver.Cluster{ - Project: rt.project, - Location: rt.location, - ID: id, - Labels: in.Labels, + Project: rt.project, + Location: rt.location, + ID: id, + Labels: in.Labels, + KafkaVersion: in.KafkaVersion, } if in.CapacityConfig != nil { @@ -124,7 +234,25 @@ func toDriverCluster(in *clusterJSON, rt *route, id string) *mkdriver.Cluster { } if in.RebalanceConfig != nil { - c.RebalanceMode = in.RebalanceConfig.Mode + c.RebalanceMode = string(in.RebalanceConfig.Mode) + } + + if in.TLSConfig != nil { + c.TLS = &mkdriver.TLSConfig{SSLPrincipalMappingRules: in.TLSConfig.SslPrincipalMappingRules} + + if in.TLSConfig.TrustConfig != nil { + for _, cas := range in.TLSConfig.TrustConfig.CasConfigs { + c.TLS.CAPools = append(c.TLS.CAPools, cas.CaPool) + } + } + } + + if in.UpdateOptions != nil { + c.AllowBrokerDownscaleOnClusterUpscale = in.UpdateOptions.AllowBrokerDownscaleOnClusterUpscale + } + + if in.BrokerCapacityConfig != nil { + c.BrokerDiskSizeGib = int64(in.BrokerCapacityConfig.DiskSizeGib) } return c @@ -139,10 +267,11 @@ func fromDriverCluster(c *mkdriver.Cluster) clusterJSON { MemoryBytes: int64String(c.MemoryBytes), }, GcpConfig: &gcpConfigJSON{KmsKey: c.KmsKey, AccessConfig: &accessConfigJSON{}}, + KafkaVersion: c.KafkaVersion, Labels: c.Labels, - State: c.State, - CreateTime: formatTime(c.CreateTime), - UpdateTime: formatTime(c.UpdateTime), + State: clusterState(c.State), + CreateTime: gcprest.FormatTime(c.CreateTime), + UpdateTime: gcprest.FormatTime(c.UpdateTime), SatisfiesPzi: c.SatisfiesPzi, SatisfiesPzs: c.SatisfiesPzs, } @@ -153,7 +282,26 @@ func fromDriverCluster(c *mkdriver.Cluster) clusterJSON { } if c.RebalanceMode != "" { - out.RebalanceConfig = &rebalanceJSON{Mode: c.RebalanceMode} + out.RebalanceConfig = &rebalanceJSON{Mode: rebalanceMode(c.RebalanceMode)} + } + + if c.TLS != nil { + out.TLSConfig = &tlsConfigJSON{SslPrincipalMappingRules: c.TLS.SSLPrincipalMappingRules} + + if len(c.TLS.CAPools) > 0 { + out.TLSConfig.TrustConfig = &trustConfigJSON{} + for _, p := range c.TLS.CAPools { + out.TLSConfig.TrustConfig.CasConfigs = append(out.TLSConfig.TrustConfig.CasConfigs, casConfigJSON{CaPool: p}) + } + } + } + + if c.AllowBrokerDownscaleOnClusterUpscale { + out.UpdateOptions = &updateOptionsJSON{AllowBrokerDownscaleOnClusterUpscale: true} + } + + if c.BrokerDiskSizeGib != 0 { + out.BrokerCapacityConfig = &brokerCapacityJSON{DiskSizeGib: int64String(c.BrokerDiskSizeGib)} } return out @@ -182,62 +330,32 @@ func fromDriverTopic(t *mkdriver.Topic) topicJSON { } } -// decodeBody decodes a JSON request body into v; an empty body leaves v zero. -// A malformed body is 400 INVALID_ARGUMENT. -func decodeBody(w http.ResponseWriter, r *http.Request, v any) bool { - r.Body = http.MaxBytesReader(w, r.Body, gcprest.MaxBodyBytes) - - if err := json.NewDecoder(r.Body).Decode(v); err != nil && !errors.Is(err, io.EOF) { - gcprest.WriteError(w, http.StatusBadRequest, "invalid", "malformed JSON body: "+err.Error()) - return false - } - - return true -} - -// anyWithType marshals v as a google.protobuf.Any by adding the "@type" -// discriminator to its JSON object. -func anyWithType(v any, typeURL string) (json.RawMessage, error) { - raw, err := json.Marshal(v) - if err != nil { - return nil, err - } - - var fields map[string]json.RawMessage - if uErr := json.Unmarshal(raw, &fields); uErr != nil { - return nil, uErr - } - - if fields == nil { - fields = map[string]json.RawMessage{} - } - - typ, err := json.Marshal(typeURL) - if err != nil { - return nil, err - } - - fields["@type"] = typ - - return json.Marshal(fields) -} - // writeOperation writes a completed operation whose response is v (typed as -// typeURL) and records it with the shared LRO poller (a no-op on a nil -// registry), so a client polling the returned name resolves the same done -// operation with its response. +// typeURL) and whose metadata is the driver operation's OperationMetadata, and +// records both with the LRO poller, so a client polling the returned name +// resolves the same done operation. func (h *Handler) writeOperation(w http.ResponseWriter, op *mkdriver.Operation, v any, typeURL string) { - resp, err := anyWithType(v, typeURL) + resp, err := gcprest.TypedAny(v, typeURL) if err != nil { gcprest.WriteError(w, http.StatusInternalServerError, "internalError", err.Error()) return } - if h.ops != nil { - h.ops.Register(op.Name, resp) + meta, err := gcprest.TypedAny(operationMetadataJSON{ + CreateTime: gcprest.FormatTime(op.CreateTime), + EndTime: gcprest.FormatTime(op.EndTime), + Target: op.TargetName, + Verb: op.Type, + APIVersion: op.APIVersion, + }, opMetaTypeURL) + if err != nil { + gcprest.WriteError(w, http.StatusInternalServerError, "internalError", err.Error()) + return } - gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Done: true, Response: resp}) + h.ops.RegisterWithMetadata(op.Name, resp, meta) + + gcprest.WriteJSON(w, http.StatusOK, operationJSON{Name: op.Name, Metadata: meta, Done: true, Response: resp}) } // clusterName builds the full cluster resource name. @@ -249,12 +367,3 @@ func clusterName(project, location, id string) string { func topicName(project, location, clusterID, id string) string { return clusterName(project, location, clusterID) + "/" + topicsSeg + "/" + id } - -// formatTime renders t as RFC3339Nano UTC; a zero time renders as "". -func formatTime(t time.Time) string { - if t.IsZero() { - return "" - } - - return t.UTC().Format(time.RFC3339Nano) -} diff --git a/server/gcp/managedkafka/wire_test.go b/server/gcp/managedkafka/wire_test.go new file mode 100644 index 000000000..b851c1f44 --- /dev/null +++ b/server/gcp/managedkafka/wire_test.go @@ -0,0 +1,312 @@ +package managedkafka + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + mkprovider "github.com/stackshy/cloudemu/v2/providers/gcp/managedkafka" + "github.com/stackshy/cloudemu/v2/server/gcp/lro" +) + +const ( + wireSubnet = `"gcpConfig":{"accessConfig":{"networkConfigs":[{"subnet":"projects/p/regions/us-central1/subnetworks/s"}]}}` + wireCluster = `{"capacityConfig":{"vcpuCount":"3","memoryBytes":"3221225472"},` + wireSubnet +) + +func standaloneServer(t *testing.T) *httptest.Server { + t.Helper() + + ts := httptest.NewServer(New(mkprovider.New(config.NewOptions(config.WithProjectID("p"))))) + t.Cleanup(ts.Close) + + return ts +} + +func call(t *testing.T, ts *httptest.Server, method, path, body string) (int, map[string]any) { + t.Helper() + + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + + req, err := http.NewRequest(method, ts.URL+path, rdr) + if err != nil { + t.Fatalf("request: %v", err) + } + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + + var out map[string]any + _ = json.NewDecoder(resp.Body).Decode(&out) + + return resp.StatusCode, out +} + +func dig(m map[string]any, keys ...string) any { + var cur any = m + for _, k := range keys { + mm, ok := cur.(map[string]any) + if !ok { + return nil + } + + cur = mm[k] + } + + return cur +} + +// TestWireNumericEnumsAndOptionalFields: proto3 JSON allows an enum as its +// number; rebalanceConfig.mode 2 is AUTO_REBALANCE_ON_SCALE_UP, and an echoed +// output-only numeric state is accepted and ignored. The optional tlsConfig, +// updateOptions, brokerCapacityConfig and kafkaVersion round-trip; kafkaVersion +// defaults to 3.7.x. +func TestWireNumericEnumsAndOptionalFields(t *testing.T) { + ts := standaloneServer(t) + + code, op := call(t, ts, http.MethodPost, loc+"/clusters?clusterId=n1", + wireCluster+`,"rebalanceConfig":{"mode":2},"state":2}`) + if code != http.StatusOK || dig(op, "response", "rebalanceConfig", "mode") != "AUTO_REBALANCE_ON_SCALE_UP" || + dig(op, "response", "kafkaVersion") != "3.7.x" || dig(op, "response", "state") != "ACTIVE" { + t.Fatalf("create with numeric mode: %d %v", code, op) + } + + if dig(op, "metadata", "@type") != opMetaTypeURL || dig(op, "metadata", "verb") != "create" || + dig(op, "metadata", "target") != "projects/p/locations/us-central1/clusters/n1" { + t.Fatalf("operation metadata: %v", op["metadata"]) + } + + full := wireCluster + `,"kafkaVersion":"4.3.x","rebalanceConfig":{"mode":"NO_REBALANCE"},` + + `"tlsConfig":{"sslPrincipalMappingRules":"DEFAULT","trustConfig":{"casConfigs":[{"caPool":"projects/x/locations/y/caPools/z"}]}},` + + `"updateOptions":{"allowBrokerDownscaleOnClusterUpscale":true},"brokerCapacityConfig":{"diskSizeGib":"150"}}` + + if code, op = call(t, ts, http.MethodPost, loc+"/clusters?clusterId=n2", full); code != http.StatusOK { + t.Fatalf("create full: %d %v", code, op) + } + + code, got := call(t, ts, http.MethodGet, loc+"/clusters/n2", "") + if code != http.StatusOK || got["kafkaVersion"] != "4.3.x" || + dig(got, "tlsConfig", "sslPrincipalMappingRules") != "DEFAULT" || + dig(got, "updateOptions", "allowBrokerDownscaleOnClusterUpscale") != true || + dig(got, "brokerCapacityConfig", "diskSizeGib") != "150" || dig(got, "rebalanceConfig", "mode") != "NO_REBALANCE" { + t.Fatalf("get full: %d %v", code, got) + } + + cas, _ := dig(got, "tlsConfig", "trustConfig", "casConfigs").([]any) + if len(cas) != 1 || dig(cas[0].(map[string]any), "caPool") != "projects/x/locations/y/caPools/z" { + t.Fatalf("casConfigs: %v", cas) + } + + for name, body := range map[string]string{ + "out of range mode": wireCluster + `,"rebalanceConfig":{"mode":7}}`, + "negative mode": wireCluster + `,"rebalanceConfig":{"mode":-1}}`, + "bool mode": wireCluster + `,"rebalanceConfig":{"mode":true}}`, + "bad state": wireCluster + `,"state":99}`, + "bad int64": `{"capacityConfig":{"vcpuCount":"x"}}`, + } { + if code, out := call(t, ts, http.MethodPost, loc+"/clusters?clusterId=bad", body); code != http.StatusBadRequest { + t.Fatalf("%s: %d %v, want 400", name, code, out) + } + } + + // null and 0 (the unspecified value) both mean unset: the default applies. + for id, mode := range map[string]string{"z0": "0", "zn": "null"} { + code, op := call(t, ts, http.MethodPost, loc+"/clusters?clusterId="+id, + wireCluster+`,"rebalanceConfig":{"mode":`+mode+`},"state":null}`) + if code != http.StatusOK || dig(op, "response", "rebalanceConfig", "mode") != "NO_REBALANCE" { + t.Fatalf("mode %s: %d %v", mode, code, op) + } + } + + // A numeric-enum PATCH changes the mode. + code, op = call(t, ts, http.MethodPatch, loc+"/clusters/n1?updateMask=rebalanceConfig.mode", + `{"rebalanceConfig":{"mode":1},"state":"ACTIVE"}`) + if code != http.StatusOK || dig(op, "response", "rebalanceConfig", "mode") != "NO_REBALANCE" { + t.Fatalf("patch numeric mode: %d %v", code, op) + } +} + +// TestStandaloneOperationPoll: a standalone package server resolves its own +// operation polls with the typed response and metadata, and 404s a name it +// never issued (it used to answer done with no response for any name). +func TestStandaloneOperationPoll(t *testing.T) { + ts := standaloneServer(t) + + code, op := call(t, ts, http.MethodPost, loc+"/clusters?clusterId=c1", wireCluster+`}`) + if code != http.StatusOK { + t.Fatalf("create: %d %v", code, op) + } + + name, _ := op["name"].(string) + + code, polled := call(t, ts, http.MethodGet, "/v1/"+name, "") + if code != http.StatusOK || polled["done"] != true || dig(polled, "response", "@type") != clusterTypeURL || + dig(polled, "response", "name") != "projects/p/locations/us-central1/clusters/c1" || + dig(polled, "metadata", "@type") != opMetaTypeURL { + t.Fatalf("poll: %d %v", code, polled) + } + + if code, out := call(t, ts, http.MethodGet, loc+"/operations/never-issued", ""); code != http.StatusNotFound { + t.Fatalf("unknown op: %d %v, want 404", code, out) + } + + code, del := call(t, ts, http.MethodDelete, loc+"/clusters/c1", "") + if code != http.StatusOK || dig(del, "response", "@type") != emptyTypeURL || dig(del, "metadata", "verb") != "delete" { + t.Fatalf("delete: %d %v", code, del) + } + + if code, _ := call(t, ts, http.MethodGet, loc+"/operations", ""); code != http.StatusNotFound { + t.Fatalf("operations collection: %d, want 404", code) + } +} + +// TestWireRoutingErrors covers the not-found, method and malformed-body paths. +func TestWireRoutingErrors(t *testing.T) { + ts := standaloneServer(t) + + if code, _ := call(t, ts, http.MethodPost, loc+"/clusters?clusterId=c1", wireCluster+`}`); code != http.StatusOK { + t.Fatalf("seed: %d", code) + } + + if code, _ := call(t, ts, http.MethodPost, loc+"/clusters/c1/topics?topicId=t1", + `{"partitionCount":1,"replicationFactor":3}`); code != http.StatusOK { + t.Fatalf("seed topic: %d", code) + } + + cases := []struct { + method, path, body string + want int + }{ + {http.MethodPut, loc + "/clusters", "", http.StatusMethodNotAllowed}, + {http.MethodPut, loc + "/clusters/c1", "", http.StatusMethodNotAllowed}, + {http.MethodPut, loc + "/clusters/c1/topics", "", http.StatusMethodNotAllowed}, + {http.MethodPut, loc + "/clusters/c1/topics/t1", "", http.StatusMethodNotAllowed}, + {http.MethodGet, loc + "/clusters/ghost", "", http.StatusNotFound}, + {http.MethodDelete, loc + "/clusters/ghost", "", http.StatusNotFound}, + {http.MethodPatch, loc + "/clusters/ghost?updateMask=labels", `{"labels":{}}`, http.StatusNotFound}, + {http.MethodPatch, loc + "/clusters/c1?updateMask=labels", `{`, http.StatusBadRequest}, + {http.MethodPost, loc + "/clusters?clusterId=c2", `{`, http.StatusBadRequest}, + {http.MethodPost, loc + "/clusters?clusterId=c2", "", http.StatusBadRequest}, + {http.MethodPost, loc + "/clusters?clusterId=c1", wireCluster + `}`, http.StatusConflict}, + {http.MethodGet, loc + "/clusters?pageToken=garbage", "", http.StatusBadRequest}, + {http.MethodGet, loc + "/clusters/c1/topics?pageToken=garbage", "", http.StatusBadRequest}, + {http.MethodGet, loc + "/clusters/ghost/topics", "", http.StatusNotFound}, + {http.MethodGet, loc + "/clusters/c1/topics/ghost", "", http.StatusNotFound}, + {http.MethodDelete, loc + "/clusters/c1/topics/ghost", "", http.StatusNotFound}, + {http.MethodPost, loc + "/clusters/c1/topics?topicId=t2", `{`, http.StatusBadRequest}, + {http.MethodPost, loc + "/clusters/c1/topics?topicId=t2", `{"partitionCount":0}`, http.StatusBadRequest}, + {http.MethodPatch, loc + "/clusters/c1/topics/t1?updateMask=partitionCount", `{`, http.StatusBadRequest}, + {http.MethodPatch, loc + "/clusters/c1/topics/t1?updateMask=replicationFactor", `{}`, http.StatusBadRequest}, + {http.MethodGet, loc + "/clusters/c1/topics/t1", "", http.StatusOK}, + {http.MethodDelete, loc + "/clusters/c1/topics/t1", "", http.StatusOK}, + } + + for _, tc := range cases { + if code, out := call(t, ts, tc.method, tc.path, tc.body); code != tc.want { + t.Fatalf("%s %s: %d %v, want %d", tc.method, tc.path, code, out, tc.want) + } + } + + h := New(mkprovider.New(config.NewOptions())) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, request(http.MethodGet, "/v1/projects/p/regions/r/clusters", "")) + + if rec.Code != http.StatusNotFound { + t.Fatalf("unparseable path: %d, want 404", rec.Code) + } +} + +// fakeSibling is a ClusterSibling owning a fixed set of cluster ids. +type fakeSibling struct{ ids map[string]bool } + +func (f fakeSibling) HasClusters(context.Context, string, string) bool { return len(f.ids) > 0 } + +func (f fakeSibling) OwnsCluster(_ context.Context, _, _, id string) bool { return f.ids[id] } + +// TestMatchesWithSibling covers ownership routing against a wired sibling. +func TestMatchesWithSibling(t *testing.T) { + h := sharedHandler(t) + h.SetClusterSibling(fakeSibling{ids: map[string]bool{"gke": true}}) + + cases := []struct { + name, method, path, body string + want bool + }{ + {"list yields when sibling owns clusters", http.MethodGet, loc + "/clusters", "", false}, + {"owned item", http.MethodGet, loc + "/clusters/owned", "", true}, + {"sibling item", http.MethodPatch, loc + "/clusters/gke", `{"labels":{}}`, false}, + {"missing item GET falls through", http.MethodGet, loc + "/clusters/ghost", "", false}, + {"missing item kafka PATCH", http.MethodPatch, loc + "/clusters/ghost", `{"capacityConfig":{}}`, true}, + {"missing item labels PATCH", http.MethodPatch, loc + "/clusters/ghost", `{"labels":{"a":"b"},"name":"x"}`, true}, + {"missing item alloydb PATCH", http.MethodPatch, loc + "/clusters/ghost", `{"labels":{},"displayName":"d"}`, false}, + {"missing item allowMissing PATCH", http.MethodPatch, loc + "/clusters/ghost?allowMissing=true", `{"labels":{}}`, false}, + {"missing item non-object PATCH", http.MethodPatch, loc + "/clusters/ghost", `[]`, false}, + {"gke create over kafka id", http.MethodPost, loc + "/clusters", `{"cluster":{"name":"owned"}}`, true}, + {"alloydb create over kafka id", http.MethodPost, loc + "/clusters?clusterId=owned", `{"network":"n"}`, true}, + {"gke create fresh id", http.MethodPost, loc + "/clusters", `{"cluster":{"name":"fresh"}}`, false}, + {"gke create no body", http.MethodPost, loc + "/clusters", "", false}, + {"delete collection", http.MethodDelete, loc + "/clusters", "", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := request(tc.method, tc.path, tc.body) + if tc.body == "" { + r.Body = nil + } + + if got := h.Matches(r); got != tc.want { + t.Fatalf("Matches(%s %s %s) = %v, want %v", tc.method, tc.path, tc.body, got, tc.want) + } + }) + } +} + +// TestSharedCreateRefusesTakenIDs: in an assembled server a create naming an id +// the other service already uses on the shared path is 409 ALREADY_EXISTS. +func TestSharedCreateRefusesTakenIDs(t *testing.T) { + h := sharedHandler(t) + h.SetClusterSibling(fakeSibling{ids: map[string]bool{"gke": true}}) + + for name, tc := range map[string]struct{ path, body string }{ + "kafka over sibling id": {loc + "/clusters?clusterId=gke", wireCluster + `}`}, + "sibling over kafka id": {loc + "/clusters", `{"cluster":{"name":"owned"}}`}, + } { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, request(http.MethodPost, tc.path, tc.body)) + + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "ALREADY_EXISTS") { + t.Fatalf("%s: %d %s, want 409 ALREADY_EXISTS", name, rec.Code, rec.Body.String()) + } + } + + // A fresh id with a wired shared registry registers its op there. + reg := lro.NewRegistry() + h.SetOperationRegistry(reg) + + rec := httptest.NewRecorder() + h.ServeHTTP(rec, request(http.MethodPost, loc+"/clusters?clusterId=fresh", wireCluster+`}`)) + + if rec.Code != http.StatusOK { + t.Fatalf("fresh create: %d %s", rec.Code, rec.Body.String()) + } + + // Operation paths are the shared poller's, never this handler's. + rec = httptest.NewRecorder() + h.ServeHTTP(rec, request(http.MethodGet, loc+"/operations/x", "")) + + if rec.Code != http.StatusNotFound { + t.Fatalf("operations on shared handler: %d", rec.Code) + } +} diff --git a/server/gcp/operations_ownership_test.go b/server/gcp/operations_ownership_test.go index 07440c2fb..2b06a16f5 100644 --- a/server/gcp/operations_ownership_test.go +++ b/server/gcp/operations_ownership_test.go @@ -195,6 +195,101 @@ func TestFullServerManagedKafkaSharesClustersWithGKE(t *testing.T) { if code, _ := do(t, ts, http.MethodGet, "/v1/projects/demo/locations/us-central1/operations/nope", ""); code != http.StatusNotFound { t.Fatalf("bogus op GET: code=%d, want 404", code) } + + // AFTER a Kafka create, the shared-location list is still GKE's: gke1 is + // listed and the Kafka cluster does not replace it. + if code, body := do(t, ts, http.MethodGet, base, ""); code != http.StatusOK || + !strings.Contains(body, "gke1") || strings.Contains(body, "kafka1") { + t.Fatalf("GKE list after Kafka create: code=%d body=%s (want gke1, not kafka1)", code, body) + } + + // In a location where GKE owns nothing, the list is Kafka's. + const west = "/v1/projects/demo/locations/europe-west1/clusters" + + westBody := strings.ReplaceAll(kafkaBody, "us-central1", "europe-west1") + if code, body := do(t, ts, http.MethodPost, west+"?clusterId=kafka2", westBody); code != http.StatusOK { + t.Fatalf("Kafka create in europe-west1: code=%d body=%s", code, body) + } + + if code, body := do(t, ts, http.MethodGet, west, ""); code != http.StatusOK || !strings.Contains(body, "clusters/kafka2") { + t.Fatalf("Kafka list where GKE owns none: code=%d body=%s", code, body) + } + + // A Kafka create reusing a GKE cluster's id in that location is refused, and + // the GKE cluster stays reachable; so is a GKE create reusing a Kafka id. + if code, body := do(t, ts, http.MethodPost, base+"?clusterId=gke1", kafkaBody); code != http.StatusConflict || + !strings.Contains(body, "ALREADY_EXISTS") { + t.Fatalf("Kafka create over GKE id: code=%d body=%s (want 409 ALREADY_EXISTS)", code, body) + } + + if code, body := do(t, ts, http.MethodPost, base, `{"cluster":{"name":"kafka1","initialNodeCount":1}}`); code != http.StatusConflict { + t.Fatalf("GKE create over Kafka id: code=%d body=%s (want 409)", code, body) + } + + if code, body := do(t, ts, http.MethodGet, base+"/gke1", ""); code != http.StatusOK || strings.Contains(body, "capacityConfig") { + t.Fatalf("GKE get after refused Kafka create: code=%d body=%s", code, body) + } + + // A Kafka-shaped PATCH for a Kafka cluster that no longer exists is Kafka's + // 404, not GKE's 405. + if code, body := do(t, ts, http.MethodDelete, base+"/kafka1", ""); code != http.StatusOK { + t.Fatalf("Kafka delete: code=%d body=%s", code, body) + } + + for _, body := range []string{`{"labels":{"a":"b"}}`, `{"capacityConfig":{"vcpuCount":"4"}}`} { + code, got := do(t, ts, http.MethodPatch, base+"/kafka1?updateMask=labels", body) + if code != http.StatusNotFound || !strings.Contains(got, "NOT_FOUND") { + t.Fatalf("PATCH missing Kafka cluster %s: code=%d body=%s (want 404 NOT_FOUND)", body, code, got) + } + } +} + +// TestFullServerManagedKafkaSharesClustersWithAlloyDB is the AlloyDB variant: +// AlloyDB's list survives a Kafka create, and neither service can take an id +// the other already uses. +func TestFullServerManagedKafkaSharesClustersWithAlloyDB(t *testing.T) { + ts := httptest.NewServer(gcpserver.New(gcpserver.DriversFromWithAlloyDB(cloudemu.NewGCP()))) + t.Cleanup(ts.Close) + + const base = "/v1/projects/demo/locations/us-central1/clusters" + + if code, body := do(t, ts, http.MethodPost, base+"?clusterId=adb1", `{"network":"n"}`); code != http.StatusOK { + t.Fatalf("AlloyDB create: code=%d body=%s", code, body) + } + + kafkaBody := `{"capacityConfig":{"vcpuCount":"3","memoryBytes":"3221225472"},` + + `"gcpConfig":{"accessConfig":{"networkConfigs":[{"subnet":"projects/demo/regions/us-central1/subnetworks/s"}]}}}` + + if code, body := do(t, ts, http.MethodPost, base+"?clusterId=kafka1", kafkaBody); code != http.StatusOK { + t.Fatalf("Kafka create: code=%d body=%s", code, body) + } + + if code, body := do(t, ts, http.MethodGet, base, ""); code != http.StatusOK || + !strings.Contains(body, "adb1") || strings.Contains(body, "kafka1") { + t.Fatalf("AlloyDB list after Kafka create: code=%d body=%s (want adb1, not kafka1)", code, body) + } + + if code, body := do(t, ts, http.MethodGet, base+"/kafka1", ""); code != http.StatusOK || !strings.Contains(body, "capacityConfig") { + t.Fatalf("Kafka get: code=%d body=%s", code, body) + } + + if code, body := do(t, ts, http.MethodGet, base+"/adb1", ""); code != http.StatusOK || strings.Contains(body, "capacityConfig") { + t.Fatalf("AlloyDB get: code=%d body=%s", code, body) + } + + if code, body := do(t, ts, http.MethodPost, base+"?clusterId=adb1", kafkaBody); code != http.StatusConflict { + t.Fatalf("Kafka create over AlloyDB id: code=%d body=%s (want 409)", code, body) + } + + if code, body := do(t, ts, http.MethodPost, base+"?clusterId=kafka1", `{"network":"n"}`); code != http.StatusConflict { + t.Fatalf("AlloyDB create over Kafka id: code=%d body=%s (want 409)", code, body) + } + + // An AlloyDB labels PATCH on its own cluster still reaches AlloyDB. + if code, body := do(t, ts, http.MethodPatch, base+"/adb1?updateMask=labels", `{"labels":{"a":"b"}}`); code != http.StatusOK || + strings.Contains(body, "capacityConfig") { + t.Fatalf("AlloyDB PATCH own cluster: code=%d body=%s", code, body) + } } // TestFullServerBackupDROperationsResolveThroughSharedPoller proves a Backup and diff --git a/services/managedkafka/driver/driver.go b/services/managedkafka/driver/driver.go index d9df09cad..2387c551b 100644 --- a/services/managedkafka/driver/driver.go +++ b/services/managedkafka/driver/driver.go @@ -43,10 +43,25 @@ type Cluster struct { // KmsKey is gcpConfig.kmsKey (immutable after create). KmsKey string - // RebalanceMode is rebalanceConfig.mode; empty when no rebalanceConfig was - // supplied. + // RebalanceMode is rebalanceConfig.mode. The provider defaults an unset + // mode to NO_REBALANCE, as the real API does. RebalanceMode string + // KafkaVersion is the Apache Kafka version (e.g. "3.7.x"). Optional on + // create; the provider defaults it to "3.7.x", as the real API does. + KafkaVersion string + + // TLS is tlsConfig; nil when the cluster has no TLS configuration. + TLS *TLSConfig + + // AllowBrokerDownscaleOnClusterUpscale is + // updateOptions.allowBrokerDownscaleOnClusterUpscale. + AllowBrokerDownscaleOnClusterUpscale bool + + // BrokerDiskSizeGib is brokerCapacityConfig.diskSizeGib (per-broker disk, + // minimum 100 GiB); 0 when no brokerCapacityConfig was supplied. + BrokerDiskSizeGib int64 + Labels map[string]string State string @@ -68,13 +83,26 @@ type Topic struct { Configs map[string]string } +// TLSConfig is the cluster's tlsConfig block. +type TLSConfig struct { + // SSLPrincipalMappingRules is tlsConfig.sslPrincipalMappingRules. + SSLPrincipalMappingRules string + // CAPools are tlsConfig.trustConfig.casConfigs[].caPool, in order. + CAPools []string +} + // Operation is a completed long-running operation. Every CloudEmu mutation -// finishes synchronously, so Done is always true. +// finishes synchronously, so Done is always true. CreateTime, EndTime, +// TargetName, Type and APIVersion render as the operation's +// google.cloud.managedkafka.v1.OperationMetadata. type Operation struct { Name string // projects/{p}/locations/{region}/operations/{op} Done bool TargetName string // the cluster the operation acted on - Type string // create | update | delete + Type string // create | update | delete (OperationMetadata.verb) + APIVersion string // OperationMetadata.apiVersion ("v1") + CreateTime time.Time + EndTime time.Time } // ManagedKafka is the control-plane interface a provider implements. @@ -96,7 +124,7 @@ type ManagedKafka interface { UpdateTopic(ctx context.Context, t *Topic, mask []string) (*Topic, error) DeleteTopic(ctx context.Context, project, location, clusterID, id string) error - // GetOperation resolves a (done) long-running operation by name, for a - // standalone package server's own operations poll. + // GetOperation returns an operation this driver created; unknown is NOT_FOUND. + // The store is bounded, so a very old (evicted) name is NOT_FOUND too. GetOperation(ctx context.Context, name string) (*Operation, error) } From 9f7b22edaa92839a340fd2b8b5b9e0def2db10d3 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:16:17 +0530 Subject: [PATCH 14/24] fix(gcp-lb): keep synthetic compute ids within int64 numericID returned the full 64-bit FNV hash, so roughly half of all forwarding rules, backend services, url maps and backend buckets got an id above MaxInt64 (e.g. 16018893947941779668). The proto type is uint64, so the gapic client accepted it, but Terraform's google provider reads every compute id into an int and failed the forwarding-rule read with "forwarding_rule_id: expected type 'int', got unconvertible type 'string'", which blocked the PSC flow in Terraform entirely. Real GCP ids never set the top bit. Mask the hash to a non-zero 63-bit value (positiveID), the same fix clouddns already carries, and apply it to pscConnectionId too. --- .../gcp/loadbalancer/forwardingrules_psc.go | 5 +- server/gcp/loadbalancer/numericid_sdk_test.go | 61 +++++++++++++++++++ server/gcp/loadbalancer/operations.go | 18 +++++- 3 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 server/gcp/loadbalancer/numericid_sdk_test.go diff --git a/server/gcp/loadbalancer/forwardingrules_psc.go b/server/gcp/loadbalancer/forwardingrules_psc.go index 576414b55..510a7721e 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc.go +++ b/server/gcp/loadbalancer/forwardingrules_psc.go @@ -60,10 +60,7 @@ func applyPSCFields(out *forwardingRuleResponse, lb *lbdriver.LBInfo) { return } - id := fnvHash("psc:" + lb.ID) - if id == 0 { - id = 1 - } + id := positiveID(fnvHash("psc:" + lb.ID)) out.PscConnectionStatus = pscStatusAccepted out.PscConnectionID = strconv.FormatUint(id, 10) diff --git a/server/gcp/loadbalancer/numericid_sdk_test.go b/server/gcp/loadbalancer/numericid_sdk_test.go new file mode 100644 index 000000000..48676500b --- /dev/null +++ b/server/gcp/loadbalancer/numericid_sdk_test.go @@ -0,0 +1,61 @@ +package loadbalancer_test + +import ( + "context" + "fmt" + "math" + "testing" + + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/option" +) + +// numericIDSamples is how many resources each id test creates. Ids are FNV +// hashes of a random driver id, so on the unmasked code each has a 1-in-2 +// chance of setting the top bit; 16 of them all missing it is a 1-in-65536 +// event. +const numericIDSamples = 16 + +// TestSDKGCPForwardingRuleIDsFitInt64 proves every forwarding-rule id and +// pscConnectionId fits in an int64. Terraform's google provider reads +// forwarding_rule_id as an int, and a full-range uint64 id failed the read +// with "expected type 'int', got unconvertible type 'string'". +func TestSDKGCPForwardingRuleIDsFitInt64(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + client := newForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + for i := range numericIDSamples { + name := fmt.Sprintf("pscid%d", i) + + op, err := client.Insert(ctx, &computepb.InsertGlobalForwardingRuleRequest{ + Project: testProject, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr(name), + Network: ptrStr(pscNetwork), + IPAddress: ptrStr(fmt.Sprintf("10.3.0.%d", i+1)), + Target: ptrStr("all-apis"), + }, + }) + if err != nil { + t.Fatalf("Insert %s: %v", name, err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("Insert %s wait: %v", name, err) + } + + got, err := client.Get(ctx, &computepb.GetGlobalForwardingRuleRequest{Project: testProject, ForwardingRule: name}) + if err != nil { + t.Fatalf("Get %s: %v", name, err) + } + + if got.GetId() == 0 || got.GetId() > math.MaxInt64 { + t.Errorf("%s: id = %d, want a non-zero value <= MaxInt64", name, got.GetId()) + } + + if got.GetPscConnectionId() == 0 || got.GetPscConnectionId() > math.MaxInt64 { + t.Errorf("%s: pscConnectionId = %d, want a non-zero value <= MaxInt64", name, got.GetPscConnectionId()) + } + } +} diff --git a/server/gcp/loadbalancer/operations.go b/server/gcp/loadbalancer/operations.go index 48e9464e2..8d9443b81 100644 --- a/server/gcp/loadbalancer/operations.go +++ b/server/gcp/loadbalancer/operations.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/binary" "encoding/json" + "math" "net/http" "sort" "strconv" @@ -934,8 +935,23 @@ func fnvHash(s string) uint64 { // numericID returns a stable uint64-shaped string derived from a driver ID. // GCP wire IDs are uint64 and proto JSON unmarshalling rejects anything else. +// The value is kept within int64 as well: Terraform's google provider reads +// every compute id into an int (e.g. forwarding_rule_id), and a full-range +// uint64 fails that read with "expected type 'int', got unconvertible type +// 'string'". Real GCP ids never set the top bit. func numericID(driverID string) string { - return strconv.FormatUint(fnvHash(driverID), 10) + return strconv.FormatUint(positiveID(fnvHash(driverID)), 10) +} + +// positiveID masks a hash to a non-zero 63-bit value, so a synthetic numeric +// id fits both uint64 (the proto type) and int64 (the Terraform schema type). +func positiveID(h uint64) uint64 { + n := h & math.MaxInt64 + if n == 0 { + n = 1 + } + + return n } // fingerprintOf returns a stable base64 fingerprint for a resource. GCP returns From 7ada44a3b7b2f8dfd736b9d5a7a077abb9adb19c Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:18:03 +0530 Subject: [PATCH 15/24] fix(gcp-lb): refuse PSC consumer rules without a network or with a scheme A Private Service Connect consumer rule was accepted with no network and was handed an external 34.x IPAddress, and an explicit loadBalancingScheme (EXTERNAL, INTERNAL) on a service-attachment or Google APIs bundle target was stored as sent. In GCP a PSC endpoint is an internal address in the consumer's VPC: the network is required and loadBalancingScheme must be empty. validatePSCTarget now refuses both with the repo's "Invalid value for field 'resource.'" 400, and a PSC rule sent without an IPAddress gets a stable internal 10.x address. --- .../gcp/loadbalancer/forwardingrules_psc.go | 46 +++++++- .../forwardingrules_psc_validate_sdk_test.go | 109 ++++++++++++++++++ server/gcp/loadbalancer/operations.go | 6 +- server/gcp/loadbalancer/refvalidation.go | 12 +- 4 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go diff --git a/server/gcp/loadbalancer/forwardingrules_psc.go b/server/gcp/loadbalancer/forwardingrules_psc.go index 510a7721e..c131cf12d 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc.go +++ b/server/gcp/loadbalancer/forwardingrules_psc.go @@ -39,20 +39,56 @@ func isPSCTarget(target string) bool { return isGoogleAPIsBundle(target) || strings.Contains(target, pscServiceAttachmentsSegment) } -// validatePSCTarget rejects a Google APIs bundle target on a regional rule: -// all-apis / vpc-sc are only valid on global forwarding rules. +// validatePSCTarget checks the fields GCP constrains on a Private Service +// Connect consumer rule; a rule whose target is not a PSC target passes: +// +// - a Google APIs bundle (all-apis / vpc-sc) is only valid on a global rule; +// - the consumer VPC must be named: the endpoint is an internal address in +// that network, so a PSC rule without `network` is refused; +// - loadBalancingScheme must be empty — a PSC endpoint is not a load +// balancer, and GCP refuses any explicit scheme (EXTERNAL, INTERNAL, …). // //nolint:gocritic // rp is a request-scoped value -func validatePSCTarget(rp gcprest.ResourcePath, target string) error { - if isGoogleAPIsBundle(target) && rp.Scope != gcprest.ScopeGlobal { +func validatePSCTarget(rp gcprest.ResourcePath, req *forwardingRuleRequest) error { + if !isPSCTarget(req.Target) { + return nil + } + + if isGoogleAPIsBundle(req.Target) && rp.Scope != gcprest.ScopeGlobal { return cerrors.Newf(cerrors.InvalidArgument, "Invalid value for field 'resource.target': '%s'. A Google APIs bundle target is only valid on a global forwarding rule.", - target) + req.Target) + } + + if req.Network == "" { + return cerrors.New(cerrors.InvalidArgument, + "Invalid value for field 'resource.network': ''. A network must be specified for a Private Service Connect forwarding rule.") + } + + if req.LoadBalancingScheme != "" { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.loadBalancingScheme': '%s'. The load balancing scheme must be empty for a Private Service Connect forwarding rule.", + req.LoadBalancingScheme) } return nil } +// pscInternalIP derives the stable internal (RFC 1918) address a PSC consumer +// rule sent without an IPAddress gets: the endpoint lives in the consumer's +// VPC, so it is never an external 34.x address. +func pscInternalIP(lb *lbdriver.LBInfo) string { + h := fnvHash("pscip:" + lb.ID + lb.Name) + + const octetMod = 254 + + o2 := byte(h%octetMod) + 1 + o3 := byte((h>>8)%octetMod) + 1 + o4 := byte((h>>16)%octetMod) + 1 + + return "10." + strconv.Itoa(int(o2)) + "." + strconv.Itoa(int(o3)) + "." + strconv.Itoa(int(o4)) +} + // applyPSCFields sets pscConnectionStatus and a stable pscConnectionId on a // PSC consumer rule's response; other rules are left untouched. func applyPSCFields(out *forwardingRuleResponse, lb *lbdriver.LBInfo) { diff --git a/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go b/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go new file mode 100644 index 000000000..e3a136c81 --- /dev/null +++ b/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go @@ -0,0 +1,109 @@ +package loadbalancer_test + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" +) + +// wantBadRequest fails unless err is a googleapi 400 whose message names field. +func wantBadRequest(t *testing.T, err error, field string) { + t.Helper() + + var gerr *googleapi.Error + if !errors.As(err, &gerr) || gerr.Code != http.StatusBadRequest { + t.Fatalf("err = %v, want 400", err) + } + + if !strings.Contains(gerr.Message, "'resource."+field+"'") { + t.Errorf("message = %q, want it to name resource.%s", gerr.Message, field) + } +} + +// TestSDKGCPForwardingRulePSCRequiresNetwork: a PSC consumer endpoint is an +// internal address in the consumer's VPC, so GCP refuses one with no network, +// for both a service-attachment and a Google APIs bundle target. +func TestSDKGCPForwardingRulePSCRequiresNetwork(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + regional := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + global := newForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + _, err := regional.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{Name: ptrStr("psc-nonet"), Target: ptrStr(pscAttachment)}, + }) + wantBadRequest(t, err, "network") + + _, err = global.Insert(ctx, &computepb.InsertGlobalForwardingRuleRequest{ + Project: testProject, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("apisnonet"), IPAddress: ptrStr("10.3.0.9"), Target: ptrStr("all-apis"), + }, + }) + wantBadRequest(t, err, "network") +} + +// TestSDKGCPForwardingRulePSCRejectsScheme: a PSC rule's loadBalancingScheme +// must be empty; an explicit EXTERNAL (or INTERNAL) is refused. +func TestSDKGCPForwardingRulePSCRejectsScheme(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + regional := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + global := newForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + _, err := regional.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("psc-ext"), Network: ptrStr(pscNetwork), Target: ptrStr(pscAttachment), + LoadBalancingScheme: ptrStr("EXTERNAL"), + }, + }) + wantBadRequest(t, err, "loadBalancingScheme") + + _, err = global.Insert(ctx, &computepb.InsertGlobalForwardingRuleRequest{ + Project: testProject, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("apisint"), Network: ptrStr(pscNetwork), IPAddress: ptrStr("10.3.0.9"), + Target: ptrStr("vpc-sc"), LoadBalancingScheme: ptrStr("INTERNAL"), + }, + }) + wantBadRequest(t, err, "loadBalancingScheme") +} + +// TestSDKGCPForwardingRulePSCInternalAddress: a PSC rule sent without an +// IPAddress gets an internal (10.x) address, never an external 34.x one. +func TestSDKGCPForwardingRulePSCInternalAddress(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + op, err := client.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("psc-noip"), Network: ptrStr(pscNetwork), Target: ptrStr(pscAttachment), + }, + }) + if err != nil { + t.Fatalf("Insert: %v", err) + } + + if err := op.Wait(ctx); err != nil { + t.Fatalf("Insert wait: %v", err) + } + + got, err := client.Get(ctx, &computepb.GetForwardingRuleRequest{Project: testProject, Region: testRegion, ForwardingRule: "psc-noip"}) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if !strings.HasPrefix(got.GetIPAddress(), "10.") { + t.Errorf("IPAddress = %q, want an internal 10.x address for a PSC endpoint", got.GetIPAddress()) + } +} diff --git a/server/gcp/loadbalancer/operations.go b/server/gcp/loadbalancer/operations.go index 8d9443b81..c2c1e650f 100644 --- a/server/gcp/loadbalancer/operations.go +++ b/server/gcp/loadbalancer/operations.go @@ -328,7 +328,7 @@ func (h *Handler) insertForwardingRule(w http.ResponseWriter, r *http.Request, r return } - if err := h.validateForwardingRuleTarget(r.Context(), rp, req.Target); err != nil { + if err := h.validateForwardingRuleTarget(r.Context(), rp, &req); err != nil { gcprest.WriteCErr(w, err) return } @@ -1033,6 +1033,10 @@ func forwardingRuleIP(lb *lbdriver.LBInfo) string { return ip } + if isPSCTarget(lb.Tags[frTargetTag]) { + return pscInternalIP(lb) + } + // Derive a deterministic public-looking IPv4 from the LB identity. h := fnvHash("ip:" + lb.ID + lb.Name) diff --git a/server/gcp/loadbalancer/refvalidation.go b/server/gcp/loadbalancer/refvalidation.go index 730de9845..65c088abe 100644 --- a/server/gcp/loadbalancer/refvalidation.go +++ b/server/gcp/loadbalancer/refvalidation.go @@ -249,15 +249,17 @@ func invalidRefErr(field, ref, noun string) error { // unvalidated rather than falsely rejected. // //nolint:gocritic // rp is a request-scoped value -func (h *Handler) validateForwardingRuleTarget(ctx context.Context, rp gcprest.ResourcePath, target string) error { +func (h *Handler) validateForwardingRuleTarget(ctx context.Context, rp gcprest.ResourcePath, + req *forwardingRuleRequest, +) error { + target := req.Target if target == "" { return nil } - // A PSC service-attachment target names a producer resource the emulator - // does not model, so it is accepted unvalidated (targetCollectionFor does not - // recognize it); a Google APIs bundle is only valid on a global rule. - if err := validatePSCTarget(rp, target); err != nil { + // A PSC consumer rule has its own field rules (validatePSCTarget); + // targetCollectionFor does not recognize a PSC target. + if err := validatePSCTarget(rp, req); err != nil { return err } From b205f8cd00f8f41fc2a2871a27c7622d28aa509e Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:20:36 +0530 Subject: [PATCH 16/24] fix(gcp-cdn): enforce cdnPolicy cross-field TTL rules and fill TTL defaults backendBuckets accepted TTL combinations GCP refuses: USE_ORIGIN_HEADERS with defaultTtl/maxTtl/clientTtl, FORCE_CACHE_ALL with maxTtl, and clientTtl above maxTtl (maxTtl 100 + clientTtl 5000 inserted fine). defaultTtl was only compared with an explicit maxTtl, never the 86400 default, and applyBackendBucketDefaults filled cacheMode but none of the documented TTLs. Now: - USE_ORIGIN_HEADERS refuses any non-zero TTL; FORCE_CACHE_ALL refuses a non-zero maxTtl. - Under CACHE_ALL_STATIC, defaultTtl and clientTtl are capped by the effective maxTtl (explicit, else 86400). - Defaults: CACHE_ALL_STATIC gets defaultTtl 3600 / maxTtl 86400 / clientTtl 3600, FORCE_CACHE_ALL defaultTtl and clientTtl 3600, USE_ORIGIN_HEADERS none. A default never exceeds an explicit smaller maxTtl, so a default cannot be why a request is refused. - A merge patch that switches cacheMode drops stored TTLs the new mode forbids unless the patch itself sends them. The rules stay in the handler, next to the existing urlMaps and healthChecks validation. The lifecycle fixture used FORCE_CACHE_ALL with maxTtl 600, which GCP refuses; it now uses CACHE_ALL_STATIC with the same TTL assertions. --- server/gcp/loadbalancer/backendbuckets.go | 1 + .../backendbuckets_cdnttl_sdk_test.go | 116 ++++++++++++++++ .../loadbalancer/backendbuckets_sdk_test.go | 6 +- .../loadbalancer/backendbuckets_validate.go | 126 ++++++++++++++++-- 4 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 server/gcp/loadbalancer/backendbuckets_cdnttl_sdk_test.go diff --git a/server/gcp/loadbalancer/backendbuckets.go b/server/gcp/loadbalancer/backendbuckets.go index 64cf908cd..0956f5ae0 100644 --- a/server/gcp/loadbalancer/backendbuckets.go +++ b/server/gcp/loadbalancer/backendbuckets.go @@ -261,6 +261,7 @@ func nextBackendBucketBody(stored, req map[string]any, merge bool) map[string]an if merge { next := deepCopyMap(stored) mergePatch(next, req) + dropTTLsForbiddenByMode(next, req) applyBackendBucketDefaults(next) return next diff --git a/server/gcp/loadbalancer/backendbuckets_cdnttl_sdk_test.go b/server/gcp/loadbalancer/backendbuckets_cdnttl_sdk_test.go new file mode 100644 index 000000000..3b3f4b6fd --- /dev/null +++ b/server/gcp/loadbalancer/backendbuckets_cdnttl_sdk_test.go @@ -0,0 +1,116 @@ +package loadbalancer_test + +import ( + "context" + "testing" + + gcpcompute "cloud.google.com/go/compute/apiv1" + computepb "cloud.google.com/go/compute/apiv1/computepb" +) + +// TestSDKGCPBackendBucketCDNModeTTLRules covers the cdnPolicy cross-field TTL +// rules GCP enforces: USE_ORIGIN_HEADERS takes every TTL from the origin, +// FORCE_CACHE_ALL has no maxTtl, clientTtl cannot exceed maxTtl, and +// defaultTtl is capped by the 86400 default maxTtl when none is sent. +func TestSDKGCPBackendBucketCDNModeTTLRules(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + withPolicy := func(p *computepb.BackendBucketCdnPolicy) *computepb.BackendBucket { + return &computepb.BackendBucket{Name: ptrStr("ttl-bb"), BucketName: ptrStr(gcsBucket), CdnPolicy: p} + } + + origin := ptrStr("USE_ORIGIN_HEADERS") + forceAll := ptrStr("FORCE_CACHE_ALL") + + cases := map[string]*computepb.BackendBucket{ + "USE_ORIGIN_HEADERS defaultTtl": withPolicy(&computepb.BackendBucketCdnPolicy{CacheMode: origin, DefaultTtl: ptrI32(60)}), + "USE_ORIGIN_HEADERS maxTtl": withPolicy(&computepb.BackendBucketCdnPolicy{CacheMode: origin, MaxTtl: ptrI32(60)}), + "USE_ORIGIN_HEADERS clientTtl": withPolicy(&computepb.BackendBucketCdnPolicy{CacheMode: origin, ClientTtl: ptrI32(60)}), + "FORCE_CACHE_ALL maxTtl": withPolicy(&computepb.BackendBucketCdnPolicy{CacheMode: forceAll, MaxTtl: ptrI32(600)}), + "clientTtl>maxTtl": withPolicy(&computepb.BackendBucketCdnPolicy{MaxTtl: ptrI32(100), ClientTtl: ptrI32(5000)}), + "defaultTtl>default maxTtl": withPolicy(&computepb.BackendBucketCdnPolicy{DefaultTtl: ptrI32(90000)}), + } + + for name, bb := range cases { + t.Run(name, func(t *testing.T) { + err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertBackendBucketRequest{Project: testProject, BackendBucketResource: bb}) + }) + assertHTTPCode(t, err, 400) + }) + } +} + +// TestSDKGCPBackendBucketCDNTTLDefaults: the TTLs GCP reports for each +// cacheMode when the caller leaves them out. +func TestSDKGCPBackendBucketCDNTTLDefaults(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + type ttls struct{ def, max, client int32 } + + cases := []struct { + name string + mode *string + max *int32 + want ttls + }{ + {"static-bb", nil, nil, ttls{3600, 86400, 3600}}, + {"static-lowmax-bb", ptrStr("CACHE_ALL_STATIC"), ptrI32(100), ttls{100, 100, 100}}, + {"force-bb", ptrStr("FORCE_CACHE_ALL"), nil, ttls{3600, 0, 3600}}, + {"origin-bb", ptrStr("USE_ORIGIN_HEADERS"), nil, ttls{0, 0, 0}}, + } + + for _, tc := range cases { + insertBB(ctx, t, c, &computepb.BackendBucket{ + Name: ptrStr(tc.name), BucketName: ptrStr(gcsBucket), EnableCdn: ptrBool(true), + CdnPolicy: &computepb.BackendBucketCdnPolicy{CacheMode: tc.mode, MaxTtl: tc.max}, + }) + + p := getBB(ctx, t, c, tc.name).GetCdnPolicy() + got := ttls{p.GetDefaultTtl(), p.GetMaxTtl(), p.GetClientTtl()} + + if got != tc.want { + t.Errorf("%s: (defaultTtl, maxTtl, clientTtl) = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestSDKGCPBackendBucketCDNModeSwitchPatch: patching a CACHE_ALL_STATIC +// bucket (which carries defaulted TTLs) to USE_ORIGIN_HEADERS succeeds and +// clears those TTLs, while a TTL the patch itself sends is still refused. +func TestSDKGCPBackendBucketCDNModeSwitchPatch(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + insertBB(ctx, t, c, &computepb.BackendBucket{ + Name: ptrStr("switch-bb"), BucketName: ptrStr(gcsBucket), EnableCdn: ptrBool(true), + }) + + patch := func(p *computepb.BackendBucketCdnPolicy) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendBucketRequest{ + Project: testProject, BackendBucket: "switch-bb", + BackendBucketResource: &computepb.BackendBucket{CdnPolicy: p}, + }) + }) + } + + assertHTTPCode(t, patch(&computepb.BackendBucketCdnPolicy{ + CacheMode: ptrStr("USE_ORIGIN_HEADERS"), DefaultTtl: ptrI32(60), + }), 400) + + if err := patch(&computepb.BackendBucketCdnPolicy{CacheMode: ptrStr("USE_ORIGIN_HEADERS")}); err != nil { + t.Fatalf("Patch to USE_ORIGIN_HEADERS: %v", err) + } + + p := getBB(ctx, t, c, "switch-bb").GetCdnPolicy() + if p.GetCacheMode() != "USE_ORIGIN_HEADERS" || p.DefaultTtl != nil || p.MaxTtl != nil || p.ClientTtl != nil { + t.Errorf("after switch: cacheMode=%q defaultTtl=%v maxTtl=%v clientTtl=%v, want USE_ORIGIN_HEADERS with no TTLs", + p.GetCacheMode(), p.DefaultTtl, p.MaxTtl, p.ClientTtl) + } +} diff --git a/server/gcp/loadbalancer/backendbuckets_sdk_test.go b/server/gcp/loadbalancer/backendbuckets_sdk_test.go index d7481e2a9..2940eb410 100644 --- a/server/gcp/loadbalancer/backendbuckets_sdk_test.go +++ b/server/gcp/loadbalancer/backendbuckets_sdk_test.go @@ -111,7 +111,7 @@ func cdnBucket(name string) *computepb.BackendBucket { CompressionMode: ptrStr("AUTOMATIC"), CustomResponseHeaders: []string{"X-Cache-Status: {cdn_cache_status}"}, CdnPolicy: &computepb.BackendBucketCdnPolicy{ - CacheMode: ptrStr("FORCE_CACHE_ALL"), + CacheMode: ptrStr("CACHE_ALL_STATIC"), DefaultTtl: ptrI32(60), MaxTtl: ptrI32(600), ClientTtl: ptrI32(30), @@ -189,7 +189,7 @@ func assertInsertedBB(t *testing.T, got *computepb.BackendBucket) { } p := got.GetCdnPolicy() - if p.GetCacheMode() != "FORCE_CACHE_ALL" || p.GetDefaultTtl() != 60 || p.GetMaxTtl() != 600 || + if p.GetCacheMode() != "CACHE_ALL_STATIC" || p.GetDefaultTtl() != 60 || p.GetMaxTtl() != 600 || p.GetClientTtl() != 30 || !p.GetNegativeCaching() || p.GetSignedUrlCacheMaxAgeSec() != 7200 || !p.GetRequestCoalescing() { t.Errorf("cdnPolicy did not round-trip: %v", p) @@ -208,7 +208,7 @@ func assertPatchedBB(t *testing.T, got *computepb.BackendBucket) { } p := got.GetCdnPolicy() - if p.GetDefaultTtl() != 120 || p.GetCacheMode() != "FORCE_CACHE_ALL" || p.GetMaxTtl() != 600 { + if p.GetDefaultTtl() != 120 || p.GetCacheMode() != "CACHE_ALL_STATIC" || p.GetMaxTtl() != 600 { t.Errorf("cdnPolicy after patch = %v, want defaultTtl=120 with cacheMode/maxTtl kept", p) } } diff --git a/server/gcp/loadbalancer/backendbuckets_validate.go b/server/gcp/loadbalancer/backendbuckets_validate.go index bbc56d62d..582b6bb2d 100644 --- a/server/gcp/loadbalancer/backendbuckets_validate.go +++ b/server/gcp/loadbalancer/backendbuckets_validate.go @@ -19,11 +19,19 @@ const ( // maxBypassCacheHeaders is how many bypassCacheOnRequestHeaders are allowed. maxBypassCacheHeaders = 5 - cacheModeCacheAllStatic = "CACHE_ALL_STATIC" + cacheModeCacheAllStatic = "CACHE_ALL_STATIC" + cacheModeUseOriginHeaders = "USE_ORIGIN_HEADERS" + cacheModeForceCacheAll = "FORCE_CACHE_ALL" + + // defaultCDNTTL is the documented defaultTtl and clientTtl (1 hour), and + // defaultCDNMaxTTL the documented maxTtl (1 day), for a mode that uses them. + defaultCDNTTL = 3600 + defaultCDNMaxTTL = 86400 fieldCacheMode = "cacheMode" fieldDefaultTTL = "defaultTtl" fieldMaxTTL = "maxTtl" + fieldClientTTL = "clientTtl" fieldService = "service" fieldDefaultService = "defaultService" @@ -38,9 +46,9 @@ var rfc1035Name = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) // //nolint:gochecknoglobals // immutable lookup table, not mutable state var validCacheModes = map[string]bool{ - cacheModeCacheAllStatic: true, - "USE_ORIGIN_HEADERS": true, - "FORCE_CACHE_ALL": true, + cacheModeCacheAllStatic: true, + cacheModeUseOriginHeaders: true, + cacheModeForceCacheAll: true, } // validCompressionModes are the compressionMode values the API accepts. @@ -75,7 +83,7 @@ func validateRFC1035Name(name string) error { // applyBackendBucketDefaults fills cdnPolicy.cacheMode with the documented // default (CACHE_ALL_STATIC) when Cloud CDN is enabled or a cdnPolicy is given -// without one. +// without one, then the TTLs the chosen mode uses (applyCDNTTLDefaults). func applyBackendBucketDefaults(body map[string]any) { enabled, _ := body["enableCdn"].(bool) policy, hasPolicy := body["cdnPolicy"].(map[string]any) @@ -92,6 +100,74 @@ func applyBackendBucketDefaults(body map[string]any) { if _, ok := policy[fieldCacheMode]; !ok { policy[fieldCacheMode] = cacheModeCacheAllStatic } + + applyCDNTTLDefaults(policy) +} + +// applyCDNTTLDefaults fills the TTLs GCP reports for a cacheMode when the +// caller left them out: CACHE_ALL_STATIC gets defaultTtl 3600, maxTtl 86400 and +// clientTtl 3600; FORCE_CACHE_ALL gets defaultTtl and clientTtl 3600 (it has no +// maxTtl); USE_ORIGIN_HEADERS gets none, because it takes every TTL from the +// origin. A filled defaultTtl/clientTtl never exceeds an explicit smaller +// maxTtl, so a default can never be the reason a request is refused. +func applyCDNTTLDefaults(policy map[string]any) { + mode, _ := policy[fieldCacheMode].(string) + if mode == cacheModeUseOriginHeaders { + return + } + + limit := int64(math.MaxInt64) + + if mode == cacheModeCacheAllStatic { + if _, ok := policy[fieldMaxTTL]; !ok { + policy[fieldMaxTTL] = float64(defaultCDNMaxTTL) + } + + if m, ok := jsonInt(policy[fieldMaxTTL]); ok { + limit = m + } + } + + for _, f := range []string{fieldDefaultTTL, fieldClientTTL} { + if _, ok := policy[f]; !ok { + policy[f] = float64(min(defaultCDNTTL, limit)) + } + } +} + +// dropTTLsForbiddenByMode removes, from a merge-patched cdnPolicy, the TTLs the +// new cacheMode forbids that the patch itself did not send. Switching a stored +// CACHE_ALL_STATIC policy (which carries defaulted TTLs) to USE_ORIGIN_HEADERS +// must not be refused over TTLs the caller never chose; a TTL the patch does +// send is still validated. +func dropTTLsForbiddenByMode(next, patch map[string]any) { + patchPolicy, _ := patch["cdnPolicy"].(map[string]any) + if _, changed := patchPolicy[fieldCacheMode]; !changed { + return + } + + policy, _ := next["cdnPolicy"].(map[string]any) + mode, _ := policy[fieldCacheMode].(string) + + for _, f := range cdnTTLsForbiddenBy(mode) { + if _, sent := patchPolicy[f]; !sent { + delete(policy, f) + } + } +} + +// cdnTTLsForbiddenBy lists the TTL fields a cacheMode refuses a non-zero value +// for: USE_ORIGIN_HEADERS takes every TTL from the origin, and FORCE_CACHE_ALL +// caches for defaultTtl so it has no maxTtl. +func cdnTTLsForbiddenBy(mode string) []string { + switch mode { + case cacheModeUseOriginHeaders: + return []string{fieldDefaultTTL, fieldMaxTTL, fieldClientTTL} + case cacheModeForceCacheAll: + return []string{fieldMaxTTL} + default: + return nil + } } // validateCompressionMode rejects an unrecognized compressionMode. @@ -133,7 +209,7 @@ func validateCDNPolicy(v any) error { return validateCDNLists(policy) } -// validateCDNRanges checks the TTL bounds and that defaultTtl <= maxTtl. +// validateCDNRanges checks the TTL bounds, then the cross-field TTL rules. func validateCDNRanges(policy map[string]any) error { limits := []struct { field string @@ -141,7 +217,7 @@ func validateCDNRanges(policy map[string]any) error { }{ {fieldDefaultTTL, maxCDNTTLSeconds}, {fieldMaxTTL, maxCDNTTLSeconds}, - {"clientTtl", maxCDNTTLSeconds}, + {fieldClientTTL, maxCDNTTLSeconds}, {"serveWhileStale", maxServeWhileStaleSeconds}, {"signedUrlCacheMaxAgeSec", math.MaxInt64}, } @@ -159,13 +235,39 @@ func validateCDNRanges(policy map[string]any) error { } } - defTTL, hasDef := jsonInt(policy[fieldDefaultTTL]) + return validateCDNModeTTLs(policy) +} + +// validateCDNModeTTLs applies the cross-field TTL rules: a cacheMode refuses a +// non-zero TTL it does not use (cdnTTLsForbiddenBy), and under +// CACHE_ALL_STATIC both defaultTtl and clientTtl are capped by the effective +// maxTtl — the explicit one, or the 86400 default when it is unset. +func validateCDNModeTTLs(policy map[string]any) error { + mode, _ := policy[fieldCacheMode].(string) + + for _, f := range cdnTTLsForbiddenBy(mode) { + if n, ok := jsonInt(policy[f]); ok && n != 0 { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.%s': '%d'. %s cannot be specified with the %s cache mode.", + f, n, f, mode) + } + } + + if mode != cacheModeCacheAllStatic && mode != "" { + return nil + } + maxTTL, hasMax := jsonInt(policy[fieldMaxTTL]) + if !hasMax { + maxTTL = defaultCDNMaxTTL + } - if hasDef && hasMax && defTTL > maxTTL { - return cerrors.Newf(cerrors.InvalidArgument, - "Invalid value for field 'resource.cdnPolicy.defaultTtl': '%d'. defaultTtl cannot be greater than maxTtl (%d).", - defTTL, maxTTL) + for _, f := range []string{fieldDefaultTTL, fieldClientTTL} { + if n, ok := jsonInt(policy[f]); ok && n > maxTTL { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.cdnPolicy.%s': '%d'. %s cannot be greater than maxTtl (%d).", + f, n, f, maxTTL) + } } return nil From ba607cd75ef7aa3ae682729dbb960c007a17b0a6 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:27:23 +0530 Subject: [PATCH 17/24] fix(gcp-vpc): keep reserved addresses and their labels in the provider compute addresses / globalAddresses, and the labels and labelFingerprint setLabels gives them, lived in a server-side addressStore inside the networking handler. They were not in /_cloudemu/snapshot, so a restore lost every address (and its labels), and a Go-library caller could not see what `serve` held. The GCP networking provider now owns them through a new optional capability, driver.GCPAddressStore (insert, get, list, delete, IP allocation, setLabels). The provider stamps labelFingerprint on insert, does the fingerprint check-and-replace under its store lock, and snapshots the records together with the IP allocator counter, so a restored emulator neither loses an address nor hands its IP out again. The handler keeps only the wire shape (kind, id, selfLink, IN_USE overlay, list filters) and adapts to the capability; a networking driver without it answers addresses with 501. Coverage docs regenerated. --- docs/coverage/coverage.json | 30 +++ docs/coverage/gcp/vpc.md | 17 ++ providers/gcp/vpc/addresses.go | 229 ++++++++++++++++++++ providers/gcp/vpc/addresses_test.go | 124 +++++++++++ providers/gcp/vpc/snapshot.go | 8 + providers/gcp/vpc/vpc.go | 8 +- server/gcp/vpc/address_labels.go | 104 ++------- server/gcp/vpc/address_snapshot_sdk_test.go | 105 +++++++++ server/gcp/vpc/address_unsupported_test.go | 32 +++ server/gcp/vpc/addresses.go | 180 +++++++-------- server/gcp/vpc/handler.go | 2 +- services/networking/driver/gcp_addresses.go | 50 +++++ 12 files changed, 716 insertions(+), 173 deletions(-) create mode 100644 providers/gcp/vpc/addresses.go create mode 100644 providers/gcp/vpc/addresses_test.go create mode 100644 server/gcp/vpc/address_snapshot_sdk_test.go create mode 100644 server/gcp/vpc/address_unsupported_test.go create mode 100644 services/networking/driver/gcp_addresses.go diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index b45ba2c30..d42e8ccbe 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -11425,6 +11425,36 @@ } ] }, + { + "name": "GCPAddressStore", + "doc": "GCPAddressStore is the GCP-only reserved-address surface.", + "operations": [ + { + "name": "AllocateGCPAddressIP", + "doc": "AllocateGCPAddressIP hands out the next IP of the provider's synthetic" + }, + { + "name": "DeleteGCPAddress", + "doc": "DeleteGCPAddress removes the address, or returns NotFound." + }, + { + "name": "GetGCPAddress", + "doc": "GetGCPAddress returns the address, or NotFound." + }, + { + "name": "InsertGCPAddress", + "doc": "InsertGCPAddress stores a new address, returning AlreadyExists when the" + }, + { + "name": "ListGCPAddresses", + "doc": "ListGCPAddresses returns every address of a project in scope, or in every" + }, + { + "name": "SetGCPAddressLabels", + "doc": "SetGCPAddressLabels replaces the address's whole label set (an empty set" + } + ] + }, { "name": "IPAM", "doc": "IPAM is an OPTIONAL AWS capability (type-asserted on the vpc driver).", diff --git a/docs/coverage/gcp/vpc.md b/docs/coverage/gcp/vpc.md index fd9ada870..9c4db637f 100644 --- a/docs/coverage/gcp/vpc.md +++ b/docs/coverage/gcp/vpc.md @@ -65,6 +65,23 @@ GCP's `networking` service · portable interface `driver.Networking` · [GCP ind | `UpdateSubnetTags` | | | `UpdateVPCTags` | Tag mutation. Update* merges keys into the resource's existing Tags | +## Optional capabilities + +Discovered by type assertion; only some providers implement these. + +### GCPAddressStore + +GCPAddressStore is the GCP-only reserved-address surface. + +| Operation | Description | +| --- | --- | +| `AllocateGCPAddressIP` | AllocateGCPAddressIP hands out the next IP of the provider's synthetic | +| `DeleteGCPAddress` | DeleteGCPAddress removes the address, or returns NotFound. | +| `GetGCPAddress` | GetGCPAddress returns the address, or NotFound. | +| `InsertGCPAddress` | InsertGCPAddress stores a new address, returning AlreadyExists when the | +| `ListGCPAddresses` | ListGCPAddresses returns every address of a project in scope, or in every | +| `SetGCPAddressLabels` | SetGCPAddressLabels replaces the address's whole label set (an empty set | + ## Not in scope _Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ diff --git a/providers/gcp/vpc/addresses.go b/providers/gcp/vpc/addresses.go new file mode 100644 index 000000000..fe81f0aac --- /dev/null +++ b/providers/gcp/vpc/addresses.go @@ -0,0 +1,229 @@ +package vpc + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "hash/fnv" + "net" + "sort" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/networking/driver" +) + +// Compile-time check that Mock implements the reserved-address capability. +var _ driver.GCPAddressStore = (*Mock)(nil) + +// reservedIPBase is the start of the synthetic range handed out to reserved +// addresses the caller did not pin to a specific IP. +const reservedIPBase = "10.128.0.0" + +// fieldLabels / fieldLabelFingerprint are the address body members the +// provider owns. +const ( + fieldLabels = "labels" + fieldLabelFingerprint = "labelFingerprint" +) + +func addressKey(project, scope, name string) string { + return project + "/" + scope + "/" + name +} + +// cloneAddress deep-copies a stored address so a caller can never alias the +// stored body. +func cloneAddress(a *driver.GCPAddress) driver.GCPAddress { + out := *a + out.Body = append(json.RawMessage(nil), a.Body...) + + return out +} + +// InsertGCPAddress stores a new reserved address and stamps its +// labelFingerprint from the labels it was created with. +// +//nolint:gocritic // hugeParam: interface method signature is fixed. +func (m *Mock) InsertGCPAddress(_ context.Context, addr driver.GCPAddress) error { + obj, err := addressObject(addr.Body) + if err != nil { + return err + } + + obj[fieldLabelFingerprint] = addressLabelFingerprint(labelsOf(obj)) + + body, err := json.Marshal(obj) + if err != nil { + return cerrors.Newf(cerrors.Internal, "encode address %q: %v", addr.Name, err) + } + + stored := addr + stored.Body = body + + if !m.addresses.SetIfAbsent(addressKey(addr.Project, addr.Scope, addr.Name), &stored) { + return cerrors.Newf(cerrors.AlreadyExists, "The resource 'addresses/%s' already exists", addr.Name) + } + + return nil +} + +// GetGCPAddress returns the named address, or NotFound. +func (m *Mock) GetGCPAddress(_ context.Context, project, scope, name string) (*driver.GCPAddress, error) { + a, ok := m.addresses.Get(addressKey(project, scope, name)) + if !ok { + return nil, addressNotFound(name) + } + + out := cloneAddress(a) + + return &out, nil +} + +// ListGCPAddresses returns a project's addresses in scope (every scope when +// scope is empty), ordered by scope then name. +func (m *Mock) ListGCPAddresses(_ context.Context, project, scope string) ([]driver.GCPAddress, error) { + matched := m.addresses.Filter(func(_ string, a *driver.GCPAddress) bool { + return a.Project == project && (scope == "" || a.Scope == scope) + }) + + out := make([]driver.GCPAddress, 0, len(matched)) + for _, a := range matched { + out = append(out, cloneAddress(a)) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].Scope != out[j].Scope { + return out[i].Scope < out[j].Scope + } + + return out[i].Name < out[j].Name + }) + + return out, nil +} + +// DeleteGCPAddress removes the named address, or returns NotFound. +func (m *Mock) DeleteGCPAddress(_ context.Context, project, scope, name string) error { + if !m.addresses.Delete(addressKey(project, scope, name)) { + return addressNotFound(name) + } + + return nil +} + +// AllocateGCPAddressIP hands out the next IP of the synthetic reserved range. +// The counter is part of the snapshot, so a restored emulator never hands out +// an IP a restored address already holds. +func (m *Mock) AllocateGCPAddressIP(_ context.Context) (string, error) { + n := m.addressIPSeq.Add(1) + + out := make(net.IP, net.IPv4len) + binary.BigEndian.PutUint32(out, binary.BigEndian.Uint32(net.ParseIP(reservedIPBase).To4())+n) + + return out.String(), nil +} + +// SetGCPAddressLabels replaces an address's labels under the store lock, so the +// fingerprint check and the write are atomic against a concurrent setLabels. +func (m *Mock) SetGCPAddressLabels(_ context.Context, project, scope, name string, + labels map[string]string, fingerprint string, +) error { + var opErr error + + found := m.addresses.Update(addressKey(project, scope, name), func(a *driver.GCPAddress) *driver.GCPAddress { + obj, err := addressObject(a.Body) + if err != nil { + opErr = err + return a + } + + if fingerprint == "" || fingerprint != addressLabelFingerprint(labelsOf(obj)) { + opErr = cerrors.New(cerrors.FailedPrecondition, + "Labels fingerprint either invalid or resource labels have changed") + + return a + } + + if len(labels) == 0 { + delete(obj, fieldLabels) + } else { + obj[fieldLabels] = labels + } + + obj[fieldLabelFingerprint] = addressLabelFingerprint(labels) + + body, err := json.Marshal(obj) + if err != nil { + opErr = cerrors.Newf(cerrors.Internal, "encode address %q: %v", name, err) + return a + } + + next := *a + next.Body = body + + return &next + }) + if !found { + return addressNotFound(name) + } + + return opErr +} + +// addressObject decodes a stored or submitted address body into an object. +func addressObject(body json.RawMessage) (map[string]any, error) { + var obj map[string]any + if err := json.Unmarshal(body, &obj); err != nil || obj == nil { + return nil, cerrors.New(cerrors.InvalidArgument, "address body is not a JSON object") + } + + return obj, nil +} + +// labelsOf reads the string labels of a decoded address body. +func labelsOf(obj map[string]any) map[string]string { + raw, _ := obj[fieldLabels].(map[string]any) + out := make(map[string]string, len(raw)) + + for k, v := range raw { + if s, ok := v.(string); ok { + out[k] = s + } + } + + return out +} + +// addressLabelFingerprint is a pure function of the label set, so it changes +// exactly when the labels do, and an address with no labels still has a +// stable, non-empty fingerprint the caller must echo back, as real Compute +// Engine requires. +func addressLabelFingerprint(labels map[string]string) string { + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + + sort.Strings(keys) + + h := fnv.New64a() + _, _ = h.Write([]byte("labels\x00")) + + for _, k := range keys { + _, _ = h.Write([]byte(k)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(labels[k])) + _, _ = h.Write([]byte{0}) + } + + var b [8]byte + + binary.BigEndian.PutUint64(b[:], h.Sum64()) + + return base64.StdEncoding.EncodeToString(b[:]) +} + +// addressNotFound renders compute's not-found message for an address. +func addressNotFound(name string) error { + return cerrors.Newf(cerrors.NotFound, "The resource 'addresses/%s' was not found", name) +} diff --git a/providers/gcp/vpc/addresses_test.go b/providers/gcp/vpc/addresses_test.go new file mode 100644 index 000000000..385f3a12d --- /dev/null +++ b/providers/gcp/vpc/addresses_test.go @@ -0,0 +1,124 @@ +package vpc + +import ( + "context" + "encoding/json" + "testing" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/networking/driver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func addressLabelsOf(t *testing.T, a *driver.GCPAddress) (map[string]string, string) { + t.Helper() + + var body struct { + Labels map[string]string `json:"labels"` + LabelFingerprint string `json:"labelFingerprint"` + } + + require.NoError(t, json.Unmarshal(a.Body, &body)) + + return body.Labels, body.LabelFingerprint +} + +// TestGCPAddressStoreLifecycle drives the reserved-address capability directly +// (the Go library path): insert stamps a fingerprint, duplicates and absent +// names are refused, setLabels enforces the fingerprint, lists filter by +// project and scope, and delete removes the record. +func TestGCPAddressStoreLifecycle(t *testing.T) { + ctx := context.Background() + m := newTestMock() + + insert := func(scope, name, body string) error { + return m.InsertGCPAddress(ctx, driver.GCPAddress{Project: "p", Scope: scope, Name: name, Body: json.RawMessage(body)}) + } + + require.NoError(t, insert("global", "a", `{"name":"a","labels":{"team":"net"}}`)) + require.NoError(t, insert("us-central1", "a", `{"name":"a"}`)) + require.NoError(t, m.InsertGCPAddress(ctx, driver.GCPAddress{Project: "q", Scope: "global", Name: "z", Body: json.RawMessage(`{"name":"z"}`)})) + + assert.True(t, cerrors.IsAlreadyExists(insert("global", "a", `{"name":"a"}`))) + assert.True(t, cerrors.IsInvalidArgument(insert("global", "bad", `[1]`))) + + got, err := m.GetGCPAddress(ctx, "p", "global", "a") + require.NoError(t, err) + + labels, fp := addressLabelsOf(t, got) + assert.Equal(t, map[string]string{"team": "net"}, labels) + assert.Equal(t, addressLabelFingerprint(labels), fp) + + _, err = m.GetGCPAddress(ctx, "p", "global", "nope") + assert.True(t, cerrors.IsNotFound(err)) + + assert.True(t, cerrors.IsFailedPrecondition(m.SetGCPAddressLabels(ctx, "p", "global", "a", nil, ""))) + assert.True(t, cerrors.IsFailedPrecondition(m.SetGCPAddressLabels(ctx, "p", "global", "a", nil, "stale"))) + assert.True(t, cerrors.IsNotFound(m.SetGCPAddressLabels(ctx, "p", "global", "nope", nil, fp))) + + require.NoError(t, m.SetGCPAddressLabels(ctx, "p", "global", "a", map[string]string{"env": "prod"}, fp)) + + got, err = m.GetGCPAddress(ctx, "p", "global", "a") + require.NoError(t, err) + + labels, fp2 := addressLabelsOf(t, got) + assert.Equal(t, map[string]string{"env": "prod"}, labels) + assert.NotEqual(t, fp, fp2) + + require.NoError(t, m.SetGCPAddressLabels(ctx, "p", "global", "a", nil, fp2)) + + got, err = m.GetGCPAddress(ctx, "p", "global", "a") + require.NoError(t, err) + + labels, _ = addressLabelsOf(t, got) + assert.Empty(t, labels, "an empty setLabels removes every label") + + all, err := m.ListGCPAddresses(ctx, "p", "") + require.NoError(t, err) + require.Len(t, all, 2) + assert.Equal(t, "global", all[0].Scope) + assert.Equal(t, "us-central1", all[1].Scope) + + regional, err := m.ListGCPAddresses(ctx, "p", "us-central1") + require.NoError(t, err) + assert.Len(t, regional, 1) + + require.NoError(t, m.DeleteGCPAddress(ctx, "p", "global", "a")) + assert.True(t, cerrors.IsNotFound(m.DeleteGCPAddress(ctx, "p", "global", "a"))) +} + +// TestGCPAddressStoreSnapshot: addresses and the IP allocator are part of the +// provider snapshot, and a returned body never aliases the stored one. +func TestGCPAddressStoreSnapshot(t *testing.T) { + ctx := context.Background() + src := newTestMock() + + ip1, err := src.AllocateGCPAddressIP(ctx) + require.NoError(t, err) + assert.Equal(t, "10.128.0.1", ip1) + + require.NoError(t, src.InsertGCPAddress(ctx, driver.GCPAddress{ + Project: "p", Scope: "global", Name: "a", Body: json.RawMessage(`{"name":"a","address":"10.128.0.1"}`), + })) + + got, err := src.GetGCPAddress(ctx, "p", "global", "a") + require.NoError(t, err) + + got.Body[0] = 'X' + + data, err := src.Snapshot(ctx, false) + require.NoError(t, err) + + dst := newTestMock() + require.NoError(t, dst.Restore(ctx, data)) + + restored, err := dst.GetGCPAddress(ctx, "p", "global", "a") + require.NoError(t, err) + assert.JSONEq(t, `{"name":"a","address":"10.128.0.1","labelFingerprint":"`+addressLabelFingerprint(nil)+`"}`, + string(restored.Body)) + + ip2, err := dst.AllocateGCPAddressIP(ctx) + require.NoError(t, err) + assert.Equal(t, "10.128.0.2", ip2) +} diff --git a/providers/gcp/vpc/snapshot.go b/providers/gcp/vpc/snapshot.go index 20d00ddb4..559cf473d 100644 --- a/providers/gcp/vpc/snapshot.go +++ b/providers/gcp/vpc/snapshot.go @@ -29,6 +29,8 @@ type vpcSnapshot struct { EIPs json.RawMessage `json:"eips,omitempty"` RTAssocs json.RawMessage `json:"rtAssocs,omitempty"` Endpoints json.RawMessage `json:"endpoints,omitempty"` + Addresses json.RawMessage `json:"addresses,omitempty"` + AddressIPSeq uint32 `json:"addressIpSeq,omitempty"` } // Snapshot captures the mock's entire state as JSON. includeAssets is unused: @@ -60,8 +62,11 @@ func (m *Mock) snapshotStores(snap *vpcSnapshot) error { {&snap.EIPs, m.eips.Snapshot}, {&snap.RTAssocs, m.rtAssocs.Snapshot}, {&snap.Endpoints, m.endpoints.Snapshot}, + {&snap.Addresses, m.addresses.Snapshot}, } + snap.AddressIPSeq = m.addressIPSeq.Load() + for _, d := range dumps { b, err := d.fn() if err != nil { @@ -104,8 +109,11 @@ func (m *Mock) restoreStores(snap *vpcSnapshot) error { {snap.EIPs, m.eips.LoadSnapshot}, {snap.RTAssocs, m.rtAssocs.LoadSnapshot}, {snap.Endpoints, m.endpoints.LoadSnapshot}, + {snap.Addresses, m.addresses.LoadSnapshot}, } + m.addressIPSeq.Store(snap.AddressIPSeq) + for _, l := range loads { if len(l.src) == 0 { continue diff --git a/providers/gcp/vpc/vpc.go b/providers/gcp/vpc/vpc.go index f134aa8cc..05d535464 100644 --- a/providers/gcp/vpc/vpc.go +++ b/providers/gcp/vpc/vpc.go @@ -3,6 +3,7 @@ package vpc import ( "context" + "sync/atomic" "time" "github.com/stackshy/cloudemu/v2/config" @@ -62,7 +63,11 @@ type Mock struct { eips *memstore.Store[*eipData] rtAssocs *memstore.Store[*rtAssocData] endpoints *memstore.Store[*driver.VPCEndpoint] - opts *config.Options + // addresses holds compute reserved addresses (driver.GCPAddressStore), + // keyed project/scope/name; addressIPSeq is the synthetic IP allocator. + addresses *memstore.Store[*driver.GCPAddress] + addressIPSeq atomic.Uint32 + opts *config.Options } // New creates a new GCP VPC mock. @@ -80,6 +85,7 @@ func New(opts *config.Options) *Mock { eips: memstore.New[*eipData](), rtAssocs: memstore.New[*rtAssocData](), endpoints: memstore.New[*driver.VPCEndpoint](), + addresses: memstore.New[*driver.GCPAddress](), opts: opts, } } diff --git a/server/gcp/vpc/address_labels.go b/server/gcp/vpc/address_labels.go index a26b2b270..35be5ca43 100644 --- a/server/gcp/vpc/address_labels.go +++ b/server/gcp/vpc/address_labels.go @@ -3,9 +3,9 @@ package vpc import ( "encoding/json" "net/http" - "sort" "strings" + cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/server/wire/gcprest" ) @@ -27,26 +27,6 @@ type addressSetLabelsRequest struct { LabelFingerprint string `json:"labelFingerprint"` } -// addressLabelFingerprint returns the fingerprint of an address's label set. -// It is a pure function of the labels, so it changes exactly when they do and -// an address with no labels still has a (stable, non-empty) fingerprint the -// caller must echo back, as real Compute Engine requires. -func addressLabelFingerprint(labels map[string]string) string { - keys := make([]string, 0, len(labels)) - for k := range labels { - keys = append(keys, k) - } - - sort.Strings(keys) - - parts := make([]string, 0, len(keys)*2) //nolint:mnd // key and value per label - for _, k := range keys { - parts = append(parts, k, labels[k]) - } - - return fingerprintOf(append([]string{"labels"}, parts...)...) -} - // addressLabels extracts the labels map from a stored address body. func addressLabels(body json.RawMessage) map[string]string { var withLabels struct { @@ -61,9 +41,11 @@ func addressLabels(body json.RawMessage) map[string]string { // setAddressLabels handles setLabels on a regional or global address. The // request's labels REPLACE the whole set; the caller must send the current // labelFingerprint (read from a Get), and a missing or stale one is rejected -// 412 conditionNotMet with no change applied. Success returns a DONE compute -// Operation recorded in the shared registry, and a later Get shows the new -// labels under a new labelFingerprint. +// 412 conditionNotMet with no change applied. The check-and-replace is the +// provider's (driver.GCPAddressStore.SetGCPAddressLabels), so the labels are +// part of the snapshot. Success returns a DONE compute Operation recorded in +// the shared registry, and a later Get shows the new labels under a new +// labelFingerprint. // //nolint:gocritic // rp is a request-scoped value func (h *Handler) setAddressLabels(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { @@ -72,79 +54,31 @@ func (h *Handler) setAddressLabels(w http.ResponseWriter, r *http.Request, rp gc return } - scope := scopeOf(rp) + if h.addresses.store == nil { + writeAddressErr(w, errAddressesUnsupported) + return + } - status := h.addresses.replaceLabels(rp.Project, scope, rp.ResourceName, req) + err := h.addresses.store.SetGCPAddressLabels(r.Context(), rp.Project, scopeOf(rp), rp.ResourceName, + req.Labels, req.LabelFingerprint) - switch status { - case labelsNotFound: + switch { + case err == nil: + case cerrors.IsNotFound(err): gcprest.WriteError(w, http.StatusNotFound, "notFound", "address "+rp.ResourceName+" not found") return - case labelsConditionNotMet: - gcprest.WriteError(w, http.StatusPreconditionFailed, "conditionNotMet", - "Labels fingerprint either invalid or resource labels have changed") - + case cerrors.IsFailedPrecondition(err): + gcprest.WriteError(w, http.StatusPreconditionFailed, "conditionNotMet", cerrors.Message(err)) return - case labelsInvalid: - gcprest.WriteError(w, http.StatusBadRequest, "invalid", "address body is not a JSON object") + default: + writeAddressErr(w, err) return - case labelsOK: } gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, resourceAddresses, rp.ResourceName, setLabelsAction)) } -// labelsResult is the outcome of a replaceLabels attempt. -type labelsResult int - -const ( - labelsOK labelsResult = iota - labelsNotFound - labelsConditionNotMet - labelsInvalid -) - -// replaceLabels swaps an address's label set under the store lock, so the -// fingerprint check and the write are atomic against a concurrent setLabels. -func (s *addressStore) replaceLabels( - project, scope, name string, req addressSetLabelsRequest, -) labelsResult { - s.mu.Lock() - defer s.mu.Unlock() - - body, ok := s.addresses[s.key(project, scope)][name] - if !ok { - return labelsNotFound - } - - if req.LabelFingerprint == "" || req.LabelFingerprint != addressLabelFingerprint(addressLabels(body)) { - return labelsConditionNotMet - } - - var obj map[string]any - if err := json.Unmarshal(body, &obj); err != nil || obj == nil { - return labelsInvalid - } - - if len(req.Labels) == 0 { - delete(obj, "labels") - } else { - obj["labels"] = req.Labels - } - - obj["labelFingerprint"] = addressLabelFingerprint(req.Labels) - - out, err := json.Marshal(obj) - if err != nil { - return labelsInvalid - } - - s.addresses[s.key(project, scope)][name] = out - - return labelsOK -} - // addressMatches applies a compute list filter to a stored address. It extends // the shared name-only matcher with `labels.=` (and `!=`, `eq`, // `ne`) so a label-scoped list returns only the addresses carrying that label. diff --git a/server/gcp/vpc/address_snapshot_sdk_test.go b/server/gcp/vpc/address_snapshot_sdk_test.go new file mode 100644 index 000000000..5e546f194 --- /dev/null +++ b/server/gcp/vpc/address_snapshot_sdk_test.go @@ -0,0 +1,105 @@ +package vpc_test + +import ( + "context" + "maps" + "net/http/httptest" + "testing" + + computev1 "google.golang.org/api/compute/v1" + "google.golang.org/api/option" + + "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/providers/gcp" + gcpserver "github.com/stackshy/cloudemu/v2/server/gcp" +) + +// serveGCPNet serves cloudP's networking + compute drivers and returns a +// compute/v1 client against it. +func serveGCPNet(t *testing.T, cloudP *gcp.Provider) *computev1.Service { + t.Helper() + + ts := httptest.NewServer(gcpserver.New(gcpserver.Drivers{Networking: cloudP.VPC, Compute: cloudP.GCE})) + t.Cleanup(ts.Close) + + svc, err := computev1.NewService(context.Background(), + option.WithEndpoint(ts.URL+"/compute/v1/"), option.WithoutAuthentication(), option.WithHTTPClient(ts.Client())) + if err != nil { + t.Fatalf("compute.NewService: %v", err) + } + + return svc +} + +// TestSDKAddressLabelsSurviveSnapshotRestore: a reserved address and the +// labels / labelFingerprint setLabels gave it live in the provider, so they are +// in the provider snapshot and read back identically from a restored emulator; +// the restored fingerprint still authorizes the next setLabels, and the IP +// allocator does not hand a restored address's IP out again. +func TestSDKAddressLabelsSurviveSnapshotRestore(t *testing.T) { + ctx := context.Background() + src := cloudemu.NewGCP() + svc := serveGCPNet(t, src) + + if _, err := svc.GlobalAddresses.Insert(testProject, &computev1.Address{Name: "psa-range"}).Context(ctx).Do(); err != nil { + t.Fatalf("Insert: %v", err) + } + + created, err := svc.GlobalAddresses.Get(testProject, "psa-range").Context(ctx).Do() + if err != nil { + t.Fatalf("Get: %v", err) + } + + if _, err := svc.GlobalAddresses.SetLabels(testProject, "psa-range", &computev1.GlobalSetLabelsRequest{ + Labels: map[string]string{"env": "prod"}, LabelFingerprint: created.LabelFingerprint, + }).Context(ctx).Do(); err != nil { + t.Fatalf("SetLabels: %v", err) + } + + before, err := svc.GlobalAddresses.Get(testProject, "psa-range").Context(ctx).Do() + if err != nil { + t.Fatalf("Get after SetLabels: %v", err) + } + + data, err := src.VPC.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + dst := cloudemu.NewGCP() + if err := dst.VPC.Restore(ctx, data); err != nil { + t.Fatalf("Restore: %v", err) + } + + restored := serveGCPNet(t, dst) + + after, err := restored.GlobalAddresses.Get(testProject, "psa-range").Context(ctx).Do() + if err != nil { + t.Fatalf("Get after restore: %v", err) + } + + if !maps.Equal(after.Labels, map[string]string{"env": "prod"}) || after.LabelFingerprint != before.LabelFingerprint || + after.Address != before.Address { + t.Fatalf("after restore: labels=%v fingerprint=%q address=%q, want %v %q %q", + after.Labels, after.LabelFingerprint, after.Address, before.Labels, before.LabelFingerprint, before.Address) + } + + if _, err := restored.GlobalAddresses.SetLabels(testProject, "psa-range", &computev1.GlobalSetLabelsRequest{ + Labels: map[string]string{"env": "dev"}, LabelFingerprint: after.LabelFingerprint, + }).Context(ctx).Do(); err != nil { + t.Fatalf("SetLabels with the restored fingerprint: %v", err) + } + + if _, err := restored.GlobalAddresses.Insert(testProject, &computev1.Address{Name: "next-range"}).Context(ctx).Do(); err != nil { + t.Fatalf("Insert after restore: %v", err) + } + + next, err := restored.GlobalAddresses.Get(testProject, "next-range").Context(ctx).Do() + if err != nil { + t.Fatalf("Get next-range: %v", err) + } + + if next.Address == after.Address { + t.Errorf("new address got IP %s, already held by the restored address", next.Address) + } +} diff --git a/server/gcp/vpc/address_unsupported_test.go b/server/gcp/vpc/address_unsupported_test.go new file mode 100644 index 000000000..1d76c1429 --- /dev/null +++ b/server/gcp/vpc/address_unsupported_test.go @@ -0,0 +1,32 @@ +package vpc_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + awsvpc "github.com/stackshy/cloudemu/v2/providers/aws/vpc" + "github.com/stackshy/cloudemu/v2/server/gcp/vpc" +) + +// TestAddressesNeedTheProviderCapability: addresses live in the provider's +// GCPAddressStore, so a networking driver without it answers 501 rather than +// silently keeping them somewhere a snapshot cannot see. +func TestAddressesNeedTheProviderCapability(t *testing.T) { + h := vpc.New(awsvpc.New(config.NewOptions()), nil) + + for _, req := range []*http.Request{ + httptest.NewRequest(http.MethodPost, "/compute/v1/projects/p/global/addresses", strings.NewReader(`{"name":"a"}`)), + httptest.NewRequest(http.MethodPost, "/compute/v1/projects/p/global/addresses/a/setLabels", + strings.NewReader(`{"labelFingerprint":"x"}`)), + } { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotImplemented { + t.Errorf("%s %s = %d, want 501 (%s)", req.Method, req.URL.Path, rec.Code, rec.Body.String()) + } + } +} diff --git a/server/gcp/vpc/addresses.go b/server/gcp/vpc/addresses.go index 916a90030..7f4ff508b 100644 --- a/server/gcp/vpc/addresses.go +++ b/server/gcp/vpc/addresses.go @@ -2,90 +2,75 @@ package vpc import ( "context" - "encoding/binary" "encoding/json" - "net" "net/http" "sort" - "strings" - "sync" + cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/internal/pagination" "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + netdriver "github.com/stackshy/cloudemu/v2/services/networking/driver" ) -// reservedIPBase is the start of the synthetic range CloudEmu hands out for -// reserved addresses that the caller didn't pin to a specific IP. -const reservedIPBase = "10.128.0.0" - // Addresses are reserved IP ranges. Private services access uses a global one // to carve out the block a managed service is peered into, so a caller // reserves it while building a network and releases it while tearing one // down, which is where its absence stops the work. // -// Like routers, these are held in the handler rather than the networking -// driver: a reserved range with a purpose and prefix length is specific to -// this provider's shape, not part of the portable subset. +// The records live in the GCP networking provider (driver.GCPAddressStore), +// not in this handler, so they (and their labels / labelFingerprint) are in the +// emulator snapshot, survive a restore, and read the same through the Go +// library and `serve`. addressStore adapts that capability to the handler: a +// networking driver without it serves addresses as 501. type addressStore struct { - mu sync.RWMutex - addresses map[string]map[string]json.RawMessage // project/scope -> name -> body - seq uint32 // monotonic IP allocator -} - -func newAddressStore() *addressStore { - return &addressStore{addresses: map[string]map[string]json.RawMessage{}} + store netdriver.GCPAddressStore } -// allocIP hands out the next IP from the synthetic reserved range. Real GCP -// allocates an address at reservation time; a caller reading back status -// RESERVED with an actual IP is what unblocks PSA/VPC-peering range setup. -func (s *addressStore) allocIP() string { - s.mu.Lock() - s.seq++ - n := s.seq - s.mu.Unlock() - - base := net.ParseIP(reservedIPBase).To4() - - v := binary.BigEndian.Uint32(base) + n - out := make(net.IP, net.IPv4len) - binary.BigEndian.PutUint32(out, v) +func newAddressStore(n netdriver.Networking) *addressStore { + s, _ := n.(netdriver.GCPAddressStore) - return out.String() + return &addressStore{store: s} } -func (s *addressStore) key(project, scope string) string { return project + "/" + scope } +// errAddressesUnsupported is returned when the networking driver does not +// implement the reserved-address capability. +var errAddressesUnsupported = cerrors.New(cerrors.Unimplemented, + "compute addresses are not supported by this networking driver") -func (s *addressStore) put(project, scope, name string, body json.RawMessage) { - s.mu.Lock() - defer s.mu.Unlock() - - k := s.key(project, scope) - if s.addresses[k] == nil { - s.addresses[k] = map[string]json.RawMessage{} +func (s *addressStore) insert(ctx context.Context, project, scope, name string, body json.RawMessage) error { + if s.store == nil { + return errAddressesUnsupported } - s.addresses[k][name] = body + return s.store.InsertGCPAddress(ctx, netdriver.GCPAddress{Project: project, Scope: scope, Name: name, Body: body}) } -func (s *addressStore) get(project, scope, name string) (json.RawMessage, bool) { - s.mu.RLock() - defer s.mu.RUnlock() +func (s *addressStore) get(ctx context.Context, project, scope, name string) (json.RawMessage, bool) { + if s.store == nil { + return nil, false + } - b, ok := s.addresses[s.key(project, scope)][name] + a, err := s.store.GetGCPAddress(ctx, project, scope, name) + if err != nil { + return nil, false + } - return b, ok + return a.Body, true } -func (s *addressStore) list(project, scope string) []json.RawMessage { - s.mu.RLock() - defer s.mu.RUnlock() +func (s *addressStore) list(ctx context.Context, project, scope string) []json.RawMessage { + if s.store == nil { + return nil + } - byName := s.addresses[s.key(project, scope)] - out := make([]json.RawMessage, 0, len(byName)) + all, err := s.store.ListGCPAddresses(ctx, project, scope) + if err != nil { + return nil + } - for _, b := range byName { - out = append(out, b) + out := make([]json.RawMessage, 0, len(all)) + for i := range all { + out = append(out, all[i].Body) } return out @@ -94,39 +79,40 @@ func (s *addressStore) list(project, scope string) []json.RawMessage { // allByScope returns every stored address for a project grouped by the scope // ("global" or a region name) it was reserved in. The grouping aggregatedList // projects into per-scope buckets. -func (s *addressStore) allByScope(project string) map[string][]json.RawMessage { - s.mu.RLock() - defer s.mu.RUnlock() - +func (s *addressStore) allByScope(ctx context.Context, project string) map[string][]json.RawMessage { out := map[string][]json.RawMessage{} - prefix := project + "/" - for k, byName := range s.addresses { - if !strings.HasPrefix(k, prefix) { - continue - } + if s.store == nil { + return out + } - scope := strings.TrimPrefix(k, prefix) - for _, b := range byName { - out[scope] = append(out[scope], b) - } + all, err := s.store.ListGCPAddresses(ctx, project, "") + if err != nil { + return out + } + + for i := range all { + out[all[i].Scope] = append(out[all[i].Scope], all[i].Body) } return out } -func (s *addressStore) delete(project, scope, name string) bool { - s.mu.Lock() - defer s.mu.Unlock() +func (s *addressStore) delete(ctx context.Context, project, scope, name string) bool { + return s.store != nil && s.store.DeleteGCPAddress(ctx, project, scope, name) == nil +} - k := s.key(project, scope) - if _, ok := s.addresses[k][name]; !ok { - return false +// allocIP hands out the next IP from the provider's synthetic reserved range. +// Real GCP allocates an address at reservation time; a caller reading back +// status RESERVED with an actual IP is what unblocks PSA/VPC-peering range setup. +func (s *addressStore) allocIP(ctx context.Context) string { + if s.store == nil { + return "" } - delete(s.addresses[k], name) + ip, _ := s.store.AllocateGCPAddressIP(ctx) - return true + return ip } // scopeOf keys an address by the scope it was reserved in, so a global @@ -202,15 +188,24 @@ func (h *Handler) insertAddress(w http.ResponseWriter, r *http.Request, rp gcpre return } - if _, exists := h.addresses.get(rp.Project, scopeOf(rp), named.Name); exists { + if h.addresses.store == nil { + writeAddressErr(w, errAddressesUnsupported) + return + } + + if _, exists := h.addresses.get(r.Context(), rp.Project, scopeOf(rp), named.Name); exists { gcprest.WriteError(w, http.StatusConflict, "alreadyExists", "address "+named.Name+" already exists") return } - h.addresses.put(rp.Project, scopeOf(rp), named.Name, - h.enrichAddress(raw, rp, hostOf(r), named.Name)) + err := h.addresses.insert(r.Context(), rp.Project, scopeOf(rp), named.Name, + h.enrichAddress(r.Context(), raw, rp, hostOf(r), named.Name)) + if err != nil { + writeAddressErr(w, err) + return + } gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, resourceAddresses, named.Name, "insert")) @@ -222,7 +217,9 @@ func (h *Handler) insertAddress(w http.ResponseWriter, r *http.Request, rp gcpre // prefixLength, addressType, …). Without this a Get reads back all-empty. // //nolint:gocritic // rp is a request-scoped value -func (h *Handler) enrichAddress(raw json.RawMessage, rp gcprest.ResourcePath, host, name string) json.RawMessage { +func (h *Handler) enrichAddress(ctx context.Context, raw json.RawMessage, rp gcprest.ResourcePath, + host, name string, +) json.RawMessage { var body map[string]any if err := json.Unmarshal(raw, &body); err != nil || body == nil { return raw @@ -233,10 +230,10 @@ func (h *Handler) enrichAddress(raw json.RawMessage, rp gcprest.ResourcePath, ho body["status"] = "RESERVED" body["selfLink"] = gcprest.SelfLink(host, rp.Project, rp.Scope, rp.ScopeName, resourceAddresses, name) body["creationTimestamp"] = nowRFC3339() - body["labelFingerprint"] = addressLabelFingerprint(addressLabels(raw)) + // labelFingerprint is stamped by the provider on insert. if addr, ok := body["address"].(string); !ok || addr == "" { - body["address"] = h.addresses.allocIP() + body["address"] = h.addresses.allocIP(ctx) } if rp.Scope == gcprest.ScopeRegions { @@ -253,7 +250,7 @@ func (h *Handler) enrichAddress(raw json.RawMessage, rp gcprest.ResourcePath, ho //nolint:gocritic // rp is a request-scoped value func (h *Handler) getAddress(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { - body, ok := h.addresses.get(rp.Project, scopeOf(rp), rp.ResourceName) + body, ok := h.addresses.get(r.Context(), rp.Project, scopeOf(rp), rp.ResourceName) if !ok { gcprest.WriteError(w, http.StatusNotFound, "notFound", "address "+rp.ResourceName+" not found") @@ -268,7 +265,7 @@ func (h *Handler) getAddress(w http.ResponseWriter, r *http.Request, rp gcprest. //nolint:gocritic // rp is a request-scoped value func (h *Handler) listAddresses(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { - all := h.addresses.list(rp.Project, scopeOf(rp)) + all := h.addresses.list(r.Context(), rp.Project, scopeOf(rp)) filter := r.URL.Query().Get("filter") usersByIP := h.addressUsersByIP(r.Context(), hostOf(r), rp.Project) @@ -303,7 +300,7 @@ func (h *Handler) listAddresses(w http.ResponseWriter, r *http.Request, rp gcpre //nolint:gocritic // rp is a request-scoped value func (h *Handler) aggregatedListAddresses(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { - byScope := h.addresses.allByScope(rp.Project) + byScope := h.addresses.allByScope(r.Context(), rp.Project) filter := r.URL.Query().Get("filter") host := hostOf(r) usersByIP := h.addressUsersByIP(r.Context(), host, rp.Project) @@ -363,7 +360,7 @@ func (h *Handler) deleteAddress(w http.ResponseWriter, r *http.Request, rp gcpre // an accessConfig natIP, returning 400 resourceInUseByAnotherResource (the // same in-use guard the disk/subnetwork deletes carry). The address deletes // cleanly once the instance releasing it is gone. - body, ok := h.addresses.get(rp.Project, scopeOf(rp), rp.ResourceName) + body, ok := h.addresses.get(r.Context(), rp.Project, scopeOf(rp), rp.ResourceName) if !ok { gcprest.WriteError(w, http.StatusNotFound, "notFound", "address "+rp.ResourceName+" not found") @@ -381,7 +378,7 @@ func (h *Handler) deleteAddress(w http.ResponseWriter, r *http.Request, rp gcpre } } - if !h.addresses.delete(rp.Project, scopeOf(rp), rp.ResourceName) { + if !h.addresses.delete(r.Context(), rp.Project, scopeOf(rp), rp.ResourceName) { gcprest.WriteError(w, http.StatusNotFound, "notFound", "address "+rp.ResourceName+" not found") @@ -497,3 +494,14 @@ func addressIP(body json.RawMessage) string { return a.Address } + +// writeAddressErr writes a provider error for an address call, answering a +// driver without the address capability with 501. +func writeAddressErr(w http.ResponseWriter, err error) { + if cerrors.GetCode(err) == cerrors.Unimplemented { + gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", cerrors.Message(err)) + return + } + + gcprest.WriteCErr(w, err) +} diff --git a/server/gcp/vpc/handler.go b/server/gcp/vpc/handler.go index 811cefd9d..5442c7365 100644 --- a/server/gcp/vpc/handler.go +++ b/server/gcp/vpc/handler.go @@ -126,7 +126,7 @@ func New(n netdriver.Networking, compute instanceLister) *Handler { net: n, compute: compute, routers: newRouterStore(), - addresses: newAddressStore(), + addresses: newAddressStore(n), routes: newRouteStore(), } } diff --git a/services/networking/driver/gcp_addresses.go b/services/networking/driver/gcp_addresses.go new file mode 100644 index 000000000..1fc52fe93 --- /dev/null +++ b/services/networking/driver/gcp_addresses.go @@ -0,0 +1,50 @@ +package driver + +import ( + "context" + "encoding/json" +) + +// compute.addresses (regional) and compute.globalAddresses reserve an IP: a +// static external IP, an internal IP in a subnetwork, or a Private Services +// Access / Private Service Connect range. The cross-cloud Networking model has +// no reserved-address resource with GCP's purpose/prefixLength shape, so the +// GCP provider stores it through this OPTIONAL, type-asserted capability, +// alongside its other state so it snapshots and restores with it. AWS and Azure +// do not implement it. + +// GCPAddress is one reserved address, addressed by (Project, Scope, Name). +// Scope is "global" for a global address, otherwise the region name. Body is +// the address resource as served (compute#address JSON), including the labels +// and labelFingerprint the provider maintains. +type GCPAddress struct { + Project string + Scope string + Name string + Body json.RawMessage +} + +// GCPAddressStore is the GCP-only reserved-address surface. +type GCPAddressStore interface { + // InsertGCPAddress stores a new address, returning AlreadyExists when the + // (project, scope, name) is taken. The provider stamps labelFingerprint + // from the body's labels, so a freshly inserted address already carries the + // fingerprint setLabels requires. + InsertGCPAddress(ctx context.Context, addr GCPAddress) error + // GetGCPAddress returns the address, or NotFound. + GetGCPAddress(ctx context.Context, project, scope, name string) (*GCPAddress, error) + // ListGCPAddresses returns every address of a project in scope, or in every + // scope when scope is empty. + ListGCPAddresses(ctx context.Context, project, scope string) ([]GCPAddress, error) + // DeleteGCPAddress removes the address, or returns NotFound. + DeleteGCPAddress(ctx context.Context, project, scope, name string) error + // AllocateGCPAddressIP hands out the next IP of the provider's synthetic + // reserved range, for an address the caller did not pin to an IP. + AllocateGCPAddressIP(ctx context.Context) (string, error) + // SetGCPAddressLabels replaces the address's whole label set (an empty set + // removes every label) and recomputes labelFingerprint. fingerprint must be + // the current labelFingerprint: a missing or stale one returns + // FailedPrecondition with nothing changed. Returns NotFound when absent. + SetGCPAddressLabels(ctx context.Context, project, scope, name string, + labels map[string]string, fingerprint string) error +} From 8065665ffe3988e6c1e3c18525befffd61a07e0f Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:30:48 +0530 Subject: [PATCH 18/24] feat(gcp-cdn): signed URL keys on backend buckets and services, urlMaps.invalidateCache backendBuckets.addSignedUrlKey / deleteSignedUrlKey returned 501, the same pair on backendServices returned 405, and urlMaps.invalidateCache returned 405. So Terraform google_compute_backend_bucket_signed_url_key failed and cdnPolicy.signedUrlKeyNames was never populated. - add/deleteSignedUrlKey (both backends) take {keyName, keyValue} / ?keyName=, list the name under cdnPolicy.signedUrlKeyNames on GET and never store or echo the value. keyName follows the compute name grammar, keyValue must be a base64url 128-bit key, a backend holds at most 3 keys, a duplicate name is 409 and deleting an unknown one 404. The names are output-only: a patch or update keeps them and a client echo of the list is ignored. - urlMaps.invalidateCache checks the url map exists and the rule names a path starting with "/" (or cache tags), and returns a DONE operation. --- server/gcp/loadbalancer/backendbuckets.go | 11 +- server/gcp/loadbalancer/handler.go | 5 + server/gcp/loadbalancer/operations.go | 6 +- server/gcp/loadbalancer/resources.go | 5 + server/gcp/loadbalancer/signedurlkeys.go | 342 ++++++++++++++++++ .../loadbalancer/signedurlkeys_sdk_test.go | 202 +++++++++++ server/gcp/loadbalancer/types.go | 2 + 7 files changed, 571 insertions(+), 2 deletions(-) create mode 100644 server/gcp/loadbalancer/signedurlkeys.go create mode 100644 server/gcp/loadbalancer/signedurlkeys_sdk_test.go diff --git a/server/gcp/loadbalancer/backendbuckets.go b/server/gcp/loadbalancer/backendbuckets.go index 0956f5ae0..0d25f26a0 100644 --- a/server/gcp/loadbalancer/backendbuckets.go +++ b/server/gcp/loadbalancer/backendbuckets.go @@ -9,7 +9,8 @@ package loadbalancer // resources. // // Surface: insert, get, list, patch (JSON merge patch), update (full replace), -// delete, setEdgeSecurityPolicy. Every mutation answers a DONE compute#operation +// delete, setEdgeSecurityPolicy, addSignedUrlKey, deleteSignedUrlKey. Every +// mutation answers a DONE compute#operation // recorded in the shared OperationRegistry, polled at // /compute/v1/projects/{p}/global/operations/{op}. @@ -129,6 +130,11 @@ func (h *Handler) routeBackendBucketItem(w http.ResponseWriter, r *http.Request, func (h *Handler) backendBucketAction(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, store lbdriver.GCPBackendBucketStore, ) { + if r.Method == http.MethodPost && isSignedURLKeyAction(rp.Action) { + h.backendBucketSignedURLKey(w, r, rp, store) + return + } + if r.Method != http.MethodPost || rp.Action != actionSetEdgeSecurityPolicy { gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", "backendBuckets."+rp.Action+" is not implemented") @@ -182,6 +188,7 @@ func (h *Handler) insertBackendBucket(w http.ResponseWriter, r *http.Request, rp } stripOutputOnly(body) + stripClientKeyNames(body) applyBackendBucketDefaults(body) if err := h.validateBackendBucket(r.Context(), body, true); err != nil { @@ -220,6 +227,7 @@ func (h *Handler) mutateBackendBucket(w http.ResponseWriter, r *http.Request, rp } stripOutputOnly(body) + stripClientKeyNames(body) // The GCS lookup happens outside the store lock; only a changed bucketName // needs it. @@ -233,6 +241,7 @@ func (h *Handler) mutateBackendBucket(w http.ResponseWriter, r *http.Request, rp err := store.UpdateGCPBackendBucket(r.Context(), rp.ResourceName, func(res *lbdriver.GCPResource) error { next := nextBackendBucketBody(res.Body, body, merge) next["name"] = res.Name + carryKeyNames(res.Body, next) if err := h.validateBackendBucket(r.Context(), next, false); err != nil { return err diff --git a/server/gcp/loadbalancer/handler.go b/server/gcp/loadbalancer/handler.go index 1918ed477..c7a9d0fa9 100644 --- a/server/gcp/loadbalancer/handler.go +++ b/server/gcp/loadbalancer/handler.go @@ -155,6 +155,11 @@ func (h *Handler) routeBackendServices(w http.ResponseWriter, r *http.Request, r return } + if r.Method == http.MethodPost && isSignedURLKeyAction(rp.Action) { + h.backendServiceSignedURLKey(w, r, rp) + return + } + switch r.Method { case http.MethodGet: h.getBackendService(w, r, rp) diff --git a/server/gcp/loadbalancer/operations.go b/server/gcp/loadbalancer/operations.go index c2c1e650f..9b4ef5b79 100644 --- a/server/gcp/loadbalancer/operations.go +++ b/server/gcp/loadbalancer/operations.go @@ -543,6 +543,7 @@ func toBackendServiceResponse(tg *lbdriver.TargetGroupInfo, rp gcprest.ResourceP decodeJSONTag(tg.Tags, bsBackendsTag, &resp.Backends) decodeJSONTag(tg.Tags, bsConnDrainTag, &resp.ConnectionDraining) decodeJSONTag(tg.Tags, bsCdnPolicyTag, &resp.CdnPolicy) + backendServiceKeyNames(tg.Tags, &resp) resp.EnableCDN = boolTag(tg.Tags, bsEnableCDNTag) return resp @@ -658,7 +659,10 @@ func mergeBackendServiceCDNTags(tags map[string]string, req *backendServiceReque } if req.CdnPolicy != nil { - encodeJSONTag(tags, bsCdnPolicyTag, req.CdnPolicy) + // signedUrlKeyNames is output-only; the names live in bsSignedURLKeysTag. + policy := *req.CdnPolicy + policy.SignedURLKeyNames = nil + encodeJSONTag(tags, bsCdnPolicyTag, &policy) } if req.EnableCDN != nil { diff --git a/server/gcp/loadbalancer/resources.go b/server/gcp/loadbalancer/resources.go index d14d7cb72..a1b79aa64 100644 --- a/server/gcp/loadbalancer/resources.go +++ b/server/gcp/loadbalancer/resources.go @@ -74,6 +74,11 @@ func (h *Handler) routeGCPResource(w http.ResponseWriter, r *http.Request, rp gc return } + if r.Method == http.MethodPost && rp.ResourceType == resourceURLMaps && rp.Action == actionInvalidateCache { + h.invalidateURLMapCache(w, r, rp) + return + } + switch r.Method { case http.MethodGet: h.getGCPResource(w, r, rp) diff --git a/server/gcp/loadbalancer/signedurlkeys.go b/server/gcp/loadbalancer/signedurlkeys.go new file mode 100644 index 000000000..212c18406 --- /dev/null +++ b/server/gcp/loadbalancer/signedurlkeys.go @@ -0,0 +1,342 @@ +package loadbalancer + +import ( + "encoding/base64" + "net/http" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + lbdriver "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +// Cloud CDN signed URL keys: backendBuckets and backendServices +// addSignedUrlKey (POST, body {keyName, keyValue}) and deleteSignedUrlKey +// (POST, ?keyName=). The key name is then listed under +// cdnPolicy.signedUrlKeyNames on a Get; the key value is write-only in GCP, so +// it is validated and dropped — never stored, never echoed. +const ( + actionAddSignedURLKey = "addSignedUrlKey" + actionDeleteSignedURLKey = "deleteSignedUrlKey" + + fieldSignedURLKeyNames = "signedUrlKeyNames" + + // maxSignedURLKeys is how many signed URL keys one backend may hold. + maxSignedURLKeys = 3 + // signedURLKeyBytes is the decoded size of a key value (a 128-bit key). + signedURLKeyBytes = 16 + + // bsSignedURLKeysTag holds a backend service's signed URL key names apart + // from its cdnPolicy tag, so a cdnPolicy patch cannot drop them. + bsSignedURLKeysTag = "cloudemu:gcpBsSignedUrlKeyNames" +) + +// signedURLKeyRequest is the SignedUrlKey body of addSignedUrlKey. +type signedURLKeyRequest struct { + KeyName string `json:"keyName"` + KeyValue string `json:"keyValue"` +} + +// validateSignedURLKey checks an addSignedUrlKey body: keyName follows the +// compute name grammar and keyValue is an RFC 4648 §5 base64url 128-bit key. +func validateSignedURLKey(req *signedURLKeyRequest) error { + if !rfc1035Name.MatchString(req.KeyName) { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'signedUrlKeyResource.keyName': '%s'. Must be a match of regex '(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)'", + req.KeyName) + } + + key, err := base64.URLEncoding.DecodeString(req.KeyValue) + if err != nil { + key, err = base64.RawURLEncoding.DecodeString(req.KeyValue) + } + + if err != nil || len(key) != signedURLKeyBytes { + return cerrors.New(cerrors.InvalidArgument, + "Invalid value for field 'signedUrlKeyResource.keyValue'. The key value must be a 128-bit key encoded as RFC 4648 Section 5 base64url.") + } + + return nil +} + +// addKeyName appends name to a backend's key names, refusing a duplicate or a +// fourth key. +func addKeyName(names []string, name string) ([]string, error) { + for _, n := range names { + if n == name { + return nil, cerrors.Newf(cerrors.AlreadyExists, "The signed URL key '%s' already exists", name) + } + } + + if len(names) >= maxSignedURLKeys { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'signedUrlKeyResource.keyName': '%s'. A backend can have at most %d signed URL keys.", + name, maxSignedURLKeys) + } + + return append(append([]string(nil), names...), name), nil +} + +// removeKeyName drops name from a backend's key names, or returns NotFound. +func removeKeyName(names []string, name string) ([]string, error) { + out := make([]string, 0, len(names)) + + for _, n := range names { + if n != name { + out = append(out, n) + } + } + + if len(out) == len(names) { + return nil, cerrors.Newf(cerrors.NotFound, "The signed URL key '%s' was not found", name) + } + + return out, nil +} + +// signedURLKeyChange decodes a signed-URL-key action and returns the function +// that applies it to a backend's current key names. It writes the 400 itself +// and returns nil when the request is malformed. +func signedURLKeyChange(w http.ResponseWriter, r *http.Request, action string) func([]string) ([]string, error) { + if action == actionDeleteSignedURLKey { + name := r.URL.Query().Get("keyName") + if name == "" { + gcprest.WriteError(w, http.StatusBadRequest, "required", "Required parameter 'keyName' is missing.") + return nil + } + + return func(names []string) ([]string, error) { return removeKeyName(names, name) } + } + + var req signedURLKeyRequest + if !gcprest.DecodeJSON(w, r, &req) { + return nil + } + + if err := validateSignedURLKey(&req); err != nil { + gcprest.WriteCErr(w, err) + return nil + } + + return func(names []string) ([]string, error) { return addKeyName(names, req.KeyName) } +} + +// isSignedURLKeyAction reports whether action is add/deleteSignedUrlKey. +func isSignedURLKeyAction(action string) bool { + return action == actionAddSignedURLKey || action == actionDeleteSignedURLKey +} + +// backendBucketSignedURLKey serves backendBuckets.addSignedUrlKey and +// deleteSignedUrlKey: the key names live under the stored body's +// cdnPolicy.signedUrlKeyNames, changed under the store lock. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) backendBucketSignedURLKey(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPBackendBucketStore, +) { + change := signedURLKeyChange(w, r, rp.Action) + if change == nil { + return + } + + err := store.UpdateGCPBackendBucket(r.Context(), rp.ResourceName, func(res *lbdriver.GCPResource) error { + body := deepCopyMap(res.Body) + + policy, _ := body["cdnPolicy"].(map[string]any) + if policy == nil { + policy = map[string]any{} + } + + names, err := change(bodyKeyNames(policy)) + if err != nil { + return err + } + + setBodyKeyNames(policy, names) + body["cdnPolicy"] = policy + res.Body = body + + return nil + }) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeBackendBucketOp(w, r, rp, rp.ResourceName, rp.Action) +} + +// bodyKeyNames reads cdnPolicy.signedUrlKeyNames from a decoded policy. +func bodyKeyNames(policy map[string]any) []string { + raw, _ := policy[fieldSignedURLKeyNames].([]any) + out := make([]string, 0, len(raw)) + + for _, v := range raw { + if s, ok := v.(string); ok { + out = append(out, s) + } + } + + return out +} + +// setBodyKeyNames writes names as cdnPolicy.signedUrlKeyNames, removing the +// member when no key is left. +func setBodyKeyNames(policy map[string]any, names []string) { + if len(names) == 0 { + delete(policy, fieldSignedURLKeyNames) + return + } + + list := make([]any, 0, len(names)) + for _, n := range names { + list = append(list, n) + } + + policy[fieldSignedURLKeyNames] = list +} + +// carryKeyNames keeps the stored signed URL key names across a patch or +// update body: they are output-only, set solely through add/deleteSignedUrlKey. +func carryKeyNames(stored, next map[string]any) { + storedPolicy, _ := stored["cdnPolicy"].(map[string]any) + names := bodyKeyNames(storedPolicy) + + nextPolicy, _ := next["cdnPolicy"].(map[string]any) + if nextPolicy == nil { + if len(names) == 0 { + return + } + + nextPolicy = map[string]any{} + next["cdnPolicy"] = nextPolicy + } + + setBodyKeyNames(nextPolicy, names) +} + +// stripClientKeyNames drops a client-sent cdnPolicy.signedUrlKeyNames. +func stripClientKeyNames(body map[string]any) { + if policy, ok := body["cdnPolicy"].(map[string]any); ok { + delete(policy, fieldSignedURLKeyNames) + } +} + +// backendServiceSignedURLKey serves backendServices.addSignedUrlKey and +// deleteSignedUrlKey, keeping the key names in their own tag under the +// driver's patch lock. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) backendServiceSignedURLKey(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { + patcher, ok := h.lb.(lbdriver.GCPBackendServicePatcher) + if !ok { + gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", "load balancer driver cannot patch backend services") + return + } + + change := signedURLKeyChange(w, r, rp.Action) + if change == nil { + return + } + + var changeErr error + + err := patcher.PatchGCPBackendService(r.Context(), scopedDriverName(rp, rp.ResourceName), func(tg *lbdriver.TargetGroupInfo) { + var names []string + + decodeJSONTag(tg.Tags, bsSignedURLKeysTag, &names) + + next, cerr := change(names) + if cerr != nil { + changeErr = cerr + return + } + + if tg.Tags == nil { + tg.Tags = map[string]string{} + } + + if len(next) == 0 { + delete(tg.Tags, bsSignedURLKeysTag) + } else { + encodeJSONTag(tg.Tags, bsSignedURLKeysTag, next) + } + }) + if err == nil { + err = changeErr + } + + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, + resourceBackendServices, rp.ResourceName, rp.Action)) +} + +// backendServiceKeyNames overlays a backend service's signed URL key names onto +// its response cdnPolicy. +func backendServiceKeyNames(tags map[string]string, resp *backendServiceResponse) { + var names []string + + decodeJSONTag(tags, bsSignedURLKeysTag, &names) + + if len(names) == 0 { + if resp.CdnPolicy != nil { + resp.CdnPolicy.SignedURLKeyNames = nil + } + + return + } + + if resp.CdnPolicy == nil { + resp.CdnPolicy = &cdnPolicy{} + } + + resp.CdnPolicy.SignedURLKeyNames = names +} + +// --- urlMaps.invalidateCache --- + +const actionInvalidateCache = "invalidateCache" + +// cacheInvalidationRule is the urlMaps.invalidateCache body. +type cacheInvalidationRule struct { + Path string `json:"path"` + Host string `json:"host,omitempty"` + CacheTags []string `json:"cacheTags,omitempty"` +} + +// invalidateURLMapCache serves urlMaps.invalidateCache: the url map must exist +// and the rule must name a path (starting with "/") or cache tags. The emulator +// caches nothing, so a valid request just records a DONE operation. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) invalidateURLMapCache(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { + store, ok := h.gcpStore() + if !ok { + gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", "load balancer driver has no GCP resource store") + return + } + + var rule cacheInvalidationRule + if !gcprest.DecodeJSON(w, r, &rule) { + return + } + + if len(rule.CacheTags) == 0 && !strings.HasPrefix(rule.Path, "/") { + gcprest.WriteCErr(w, cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.path': '%s'. The path must start with '/'.", rule.Path)) + + return + } + + if _, err := store.GetGCPResource(r.Context(), resourceURLMaps, scopeKeyOf(rp), rp.ResourceName); err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, + resourceURLMaps, rp.ResourceName, actionInvalidateCache)) +} diff --git a/server/gcp/loadbalancer/signedurlkeys_sdk_test.go b/server/gcp/loadbalancer/signedurlkeys_sdk_test.go new file mode 100644 index 000000000..4208cac72 --- /dev/null +++ b/server/gcp/loadbalancer/signedurlkeys_sdk_test.go @@ -0,0 +1,202 @@ +package loadbalancer_test + +import ( + "context" + "slices" + "strings" + "testing" + + gcpcompute "cloud.google.com/go/compute/apiv1" + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/option" +) + +// signingKey is a valid 128-bit base64url signed URL key value. The tests +// assert it never appears in a response. +const signingKey = "nZtRohdNF9m3cKM24IcK4w==" + +// TestSDKGCPBackendBucketSignedURLKeys drives backendBuckets.addSignedUrlKey +// and deleteSignedUrlKey through the real BackendBucketsClient (Terraform's +// google_compute_backend_bucket_signed_url_key uses the same calls): the key +// name is listed under cdnPolicy.signedUrlKeyNames, the value is never echoed, +// a patch keeps the names, and bad requests are refused. +func TestSDKGCPBackendBucketSignedURLKeys(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + c := newBackendBucketsClient(t, ts) + + insertBB(ctx, t, c, &computepb.BackendBucket{Name: ptrStr("signed-bb"), BucketName: ptrStr(gcsBucket), EnableCdn: ptrBool(true)}) + + add := func(bucket, name, value string) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.AddSignedUrlKey(ctx, &computepb.AddSignedUrlKeyBackendBucketRequest{ + Project: testProject, BackendBucket: bucket, + SignedUrlKeyResource: &computepb.SignedUrlKey{KeyName: ptrStr(name), KeyValue: ptrStr(value)}, + }) + }) + } + + del := func(name string) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.DeleteSignedUrlKey(ctx, &computepb.DeleteSignedUrlKeyBackendBucketRequest{ + Project: testProject, BackendBucket: "signed-bb", KeyName: name, + }) + }) + } + + for _, name := range []string{"key-a", "key-b"} { + if err := add("signed-bb", name, signingKey); err != nil { + t.Fatalf("AddSignedUrlKey %s: %v", name, err) + } + } + + if got := getBB(ctx, t, c, "signed-bb").GetCdnPolicy().GetSignedUrlKeyNames(); !slices.Equal(got, []string{"key-a", "key-b"}) { + t.Fatalf("signedUrlKeyNames = %v, want [key-a key-b]", got) + } + + _, raw := doJSON(t, ts, "GET", ts.URL+"/compute/v1/projects/"+testProject+"/global/backendBuckets/signed-bb", "") + if strings.Contains(raw, signingKey) { + t.Fatalf("GET echoes the signed URL key value: %s", raw) + } + + assertHTTPCode(t, add("signed-bb", "key-a", signingKey), 409) + assertHTTPCode(t, add("signed-bb", "key-c", "not-base64!"), 400) + assertHTTPCode(t, add("signed-bb", "key-c", "c2hvcnQ="), 400) + assertHTTPCode(t, add("signed-bb", "Bad_Name", signingKey), 400) + assertHTTPCode(t, add("ghost-bb", "key-c", signingKey), 404) + + if err := add("signed-bb", "key-c", signingKey); err != nil { + t.Fatalf("AddSignedUrlKey key-c: %v", err) + } + + assertHTTPCode(t, add("signed-bb", "key-d", signingKey), 400) + + // A patch that touches cdnPolicy (and even echoes a bogus key list) keeps + // the names: they are output-only. + if err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendBucketRequest{ + Project: testProject, BackendBucket: "signed-bb", + BackendBucketResource: &computepb.BackendBucket{CdnPolicy: &computepb.BackendBucketCdnPolicy{ + DefaultTtl: ptrI32(60), SignedUrlKeyNames: []string{"forged"}, + }}, + }) + }); err != nil { + t.Fatalf("Patch: %v", err) + } + + if err := del("key-b"); err != nil { + t.Fatalf("DeleteSignedUrlKey: %v", err) + } + + assertHTTPCode(t, del("key-b"), 404) + + if got := getBB(ctx, t, c, "signed-bb").GetCdnPolicy().GetSignedUrlKeyNames(); !slices.Equal(got, []string{"key-a", "key-c"}) { + t.Fatalf("signedUrlKeyNames after patch + delete = %v, want [key-a key-c]", got) + } +} + +// TestSDKGCPBackendServiceSignedURLKeys: the same pair on backendServices +// (405 before), with the names kept across a cdnPolicy patch. +func TestSDKGCPBackendServiceSignedURLKeys(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + c := newBackendServicesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + insertBS(ctx, t, c, "signed-bs") + + if err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.AddSignedUrlKey(ctx, &computepb.AddSignedUrlKeyBackendServiceRequest{ + Project: testProject, BackendService: "signed-bs", + SignedUrlKeyResource: &computepb.SignedUrlKey{KeyName: ptrStr("key-a"), KeyValue: ptrStr(signingKey)}, + }) + }); err != nil { + t.Fatalf("AddSignedUrlKey: %v", err) + } + + if err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchBackendServiceRequest{ + Project: testProject, BackendService: "signed-bs", + BackendServiceResource: &computepb.BackendService{CdnPolicy: &computepb.BackendServiceCdnPolicy{ + CacheMode: ptrStr("CACHE_ALL_STATIC"), + }}, + }) + }); err != nil { + t.Fatalf("Patch: %v", err) + } + + got, err := c.Get(ctx, &computepb.GetBackendServiceRequest{Project: testProject, BackendService: "signed-bs"}) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if names := got.GetCdnPolicy().GetSignedUrlKeyNames(); !slices.Equal(names, []string{"key-a"}) { + t.Fatalf("signedUrlKeyNames = %v, want [key-a]", names) + } + + _, raw := doJSON(t, ts, "GET", ts.URL+"/compute/v1/projects/"+testProject+"/global/backendServices/signed-bs", "") + if strings.Contains(raw, signingKey) { + t.Fatalf("GET echoes the signed URL key value: %s", raw) + } + + del := func(name string) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.DeleteSignedUrlKey(ctx, &computepb.DeleteSignedUrlKeyBackendServiceRequest{ + Project: testProject, BackendService: "signed-bs", KeyName: name, + }) + }) + } + + if err := del("key-a"); err != nil { + t.Fatalf("DeleteSignedUrlKey: %v", err) + } + + assertHTTPCode(t, del("key-a"), 404) + + got, err = c.Get(ctx, &computepb.GetBackendServiceRequest{Project: testProject, BackendService: "signed-bs"}) + if err != nil { + t.Fatalf("Get after delete: %v", err) + } + + if names := got.GetCdnPolicy().GetSignedUrlKeyNames(); len(names) != 0 { + t.Fatalf("signedUrlKeyNames after delete = %v, want none", names) + } +} + +// TestSDKGCPURLMapInvalidateCache drives urlMaps.invalidateCache through the +// real UrlMapsClient: it returns an operation that completes, and is refused +// for a missing url map or a path that does not start with "/". +func TestSDKGCPURLMapInvalidateCache(t *testing.T) { + ts := newCDNServer(t, gcsBucket) + ctx := context.Background() + bb := newBackendBucketsClient(t, ts) + + um, err := gcpcompute.NewUrlMapsRESTClient(ctx, clientOpts(ts)...) + if err != nil { + t.Fatalf("NewUrlMapsRESTClient: %v", err) + } + + t.Cleanup(func() { _ = um.Close() }) + + insertBB(ctx, t, bb, &computepb.BackendBucket{Name: ptrStr("site-bb"), BucketName: ptrStr(gcsBucket)}) + waitOp(ctx, t, "UrlMap Insert", func() (*gcpcompute.Operation, error) { + return um.Insert(ctx, &computepb.InsertUrlMapRequest{Project: testProject, UrlMapResource: &computepb.UrlMap{ + Name: ptrStr("cdn-map"), DefaultService: ptrStr(bbRef("site-bb")), + }}) + }) + + invalidate := func(m, path string) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return um.InvalidateCache(ctx, &computepb.InvalidateCacheUrlMapRequest{ + Project: testProject, UrlMap: m, + CacheInvalidationRuleResource: &computepb.CacheInvalidationRule{Path: ptrStr(path)}, + }) + }) + } + + if err := invalidate("cdn-map", "/images/*"); err != nil { + t.Fatalf("InvalidateCache: %v", err) + } + + assertHTTPCode(t, invalidate("ghost-map", "/images/*"), 404) + assertHTTPCode(t, invalidate("cdn-map", "images"), 400) +} diff --git a/server/gcp/loadbalancer/types.go b/server/gcp/loadbalancer/types.go index bf50859c9..26c5537d9 100644 --- a/server/gcp/loadbalancer/types.go +++ b/server/gcp/loadbalancer/types.go @@ -28,6 +28,8 @@ type cdnPolicy struct { DefaultTTL int `json:"defaultTtl,omitempty"` ClientTTL int `json:"clientTtl,omitempty"` MaxTTL int `json:"maxTtl,omitempty"` + // SignedURLKeyNames is output-only: set through add/deleteSignedUrlKey. + SignedURLKeyNames []string `json:"signedUrlKeyNames,omitempty"` } type backendServiceRequest struct { From ad1bb5b7468ba0d79ab19c6b1def93e0987adf08 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:40:43 +0530 Subject: [PATCH 19/24] feat(gcp-psc): regional serviceAttachments, and PSC status from the attachment compute.serviceAttachments did not exist (POST .../regions/r/ serviceAttachments returned 501, so google_compute_service_attachment failed), and every PSC consumer rule reported pscConnectionStatus ACCEPTED, even for a target like .../serviceAttachments/nosuch. serviceAttachments now has insert, get, list, patch (JSON merge patch) and delete with DONE operations. The records and the connection decisions live in the GCP load-balancer provider behind a new optional capability, driver.GCPServiceAttachmentStore, stored with the other opaque GCP resources so they snapshot and restore. The provider validates connectionPreference (ACCEPT_AUTOMATIC / ACCEPT_MANUAL), targetService, natSubnets and consumerAcceptLists entries. A consumer rule targeting an attachment now: - is refused (400, "The referenced serviceAttachment resource cannot be found") when the attachment does not exist or is in another region; - is recorded in the attachment's connectedEndpoints with a status: ACCEPT_AUTOMATIC accepts every consumer; ACCEPT_MANUAL rejects a project or network in consumerRejectLists, accepts one matched by a consumerAcceptLists entry (projectIdOrNum, networkUrl or endpointUrl) while its connectionLimit has room, and leaves the rest PENDING; - is re-evaluated, in connection order, whenever the attachment changes or another endpoint disconnects, and reads CLOSED once the attachment is deleted. Google APIs bundle endpoints (all-apis, vpc-sc) stay ACCEPTED. The existing PSC tests targeted an attachment that was never created; they now create it first. Coverage docs regenerated. --- docs/coverage/coverage.json | 38 ++ docs/coverage/gcp/lb.md | 15 + .../gcp/loadbalancer/service_attachments.go | 446 ++++++++++++++++++ .../loadbalancer/service_attachments_test.go | 154 ++++++ .../gcp/loadbalancer/forwardingrules_psc.go | 73 ++- .../forwardingrules_psc_sdk_test.go | 7 +- .../forwardingrules_psc_validate_sdk_test.go | 2 + server/gcp/loadbalancer/handler.go | 4 +- server/gcp/loadbalancer/operations.go | 19 +- server/gcp/loadbalancer/refvalidation.go | 4 + server/gcp/loadbalancer/resources.go | 1 + server/gcp/loadbalancer/serviceattachments.go | 206 ++++++++ .../serviceattachments_sdk_test.go | 289 ++++++++++++ .../driver/gcp_service_attachments.go | 80 ++++ 14 files changed, 1325 insertions(+), 13 deletions(-) create mode 100644 providers/gcp/loadbalancer/service_attachments.go create mode 100644 providers/gcp/loadbalancer/service_attachments_test.go create mode 100644 server/gcp/loadbalancer/serviceattachments.go create mode 100644 server/gcp/loadbalancer/serviceattachments_sdk_test.go create mode 100644 services/loadbalancer/driver/gcp_service_attachments.go diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index d42e8ccbe..6c75ba993 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -9694,6 +9694,44 @@ } ] }, + { + "name": "GCPServiceAttachmentStore", + "doc": "GCPServiceAttachmentStore is an OPTIONAL, type-asserted capability", + "operations": [ + { + "name": "ConnectGCPServiceAttachment", + "doc": "ConnectGCPServiceAttachment records a consumer endpoint on the attachment" + }, + { + "name": "DeleteGCPServiceAttachment", + "doc": "DeleteGCPServiceAttachment removes the attachment, or returns NotFound." + }, + { + "name": "DisconnectGCPServiceAttachment", + "doc": "DisconnectGCPServiceAttachment removes a consumer endpoint (by" + }, + { + "name": "GCPPSCConnectionStatus", + "doc": "GCPPSCConnectionStatus returns the current status of a consumer" + }, + { + "name": "GetGCPServiceAttachment", + "doc": "GetGCPServiceAttachment returns the attachment, or NotFound." + }, + { + "name": "InsertGCPServiceAttachment", + "doc": "InsertGCPServiceAttachment validates and stores a new attachment," + }, + { + "name": "ListGCPServiceAttachments", + "doc": "ListGCPServiceAttachments returns every attachment in a region." + }, + { + "name": "UpdateGCPServiceAttachment", + "doc": "UpdateGCPServiceAttachment applies mutate under the store lock, keeps the" + } + ] + }, { "name": "LBAttributeUpdater", "doc": "LBAttributeUpdater is implemented by drivers that can apply a partial", diff --git a/docs/coverage/gcp/lb.md b/docs/coverage/gcp/lb.md index ebe0e7bab..615585343 100644 --- a/docs/coverage/gcp/lb.md +++ b/docs/coverage/gcp/lb.md @@ -63,6 +63,21 @@ GCPComputeResourceStore is an OPTIONAL, type-asserted capability implemented | `PutGCPResource` | PutGCPResource stores res, returning AlreadyExists when a resource with | | `UpdateGCPResource` | UpdateGCPResource applies mutate to the stored resource in place under the | +### GCPServiceAttachmentStore + +GCPServiceAttachmentStore is an OPTIONAL, type-asserted capability + +| Operation | Description | +| --- | --- | +| `ConnectGCPServiceAttachment` | ConnectGCPServiceAttachment records a consumer endpoint on the attachment | +| `DeleteGCPServiceAttachment` | DeleteGCPServiceAttachment removes the attachment, or returns NotFound. | +| `DisconnectGCPServiceAttachment` | DisconnectGCPServiceAttachment removes a consumer endpoint (by | +| `GCPPSCConnectionStatus` | GCPPSCConnectionStatus returns the current status of a consumer | +| `GetGCPServiceAttachment` | GetGCPServiceAttachment returns the attachment, or NotFound. | +| `InsertGCPServiceAttachment` | InsertGCPServiceAttachment validates and stores a new attachment, | +| `ListGCPServiceAttachments` | ListGCPServiceAttachments returns every attachment in a region. | +| `UpdateGCPServiceAttachment` | UpdateGCPServiceAttachment applies mutate under the store lock, keeps the | + ## Not in scope _Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ diff --git a/providers/gcp/loadbalancer/service_attachments.go b/providers/gcp/loadbalancer/service_attachments.go new file mode 100644 index 000000000..d92f8398b --- /dev/null +++ b/providers/gcp/loadbalancer/service_attachments.go @@ -0,0 +1,446 @@ +package loadbalancer + +import ( + "context" + "encoding/json" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +// Compile-time check that Mock implements the service-attachment capability. +var _ driver.GCPServiceAttachmentStore = (*Mock)(nil) + +// Service attachment body members the provider reads or owns. +const ( + saConnectionPreference = "connectionPreference" + saNatSubnets = "natSubnets" + saTargetService = "targetService" + saAcceptLists = "consumerAcceptLists" + saRejectLists = "consumerRejectLists" + saConnectedEndpoints = "connectedEndpoints" + + epEndpoint = "endpoint" + epPscConnectionID = "pscConnectionId" + epStatus = "status" + epConsumerNetwork = "consumerNetwork" + + acceptProject = "projectIdOrNum" + acceptNetwork = "networkUrl" + acceptLimit = "connectionLimit" + acceptEndpoint = "endpointUrl" +) + +// InsertGCPServiceAttachment validates and stores a new service attachment +// with no connected endpoints. +// +//nolint:gocritic // hugeParam: interface method signature is fixed. +func (m *Mock) InsertGCPServiceAttachment(ctx context.Context, res driver.GCPResource) error { + body := cloneBody(res.Body) + delete(body, saConnectedEndpoints) + + if err := validateServiceAttachment(body); err != nil { + return err + } + + res.Collection = driver.GCPServiceAttachmentCollection + res.Body = body + + err := m.PutGCPResource(ctx, res) + if cerrors.IsAlreadyExists(err) { + return cerrors.Newf(cerrors.AlreadyExists, "The resource 'serviceAttachments/%s' already exists", res.Name) + } + + return err +} + +// GetGCPServiceAttachment returns the named attachment, or NotFound. +func (m *Mock) GetGCPServiceAttachment(_ context.Context, region, name string) (*driver.GCPResource, error) { + res, ok := m.gcpResources.Get(gcpResourceKey(driver.GCPServiceAttachmentCollection, region, name)) + if !ok { + return nil, serviceAttachmentNotFound(name) + } + + res.Body = cloneBody(res.Body) + + return &res, nil +} + +// ListGCPServiceAttachments returns every attachment in region. +func (m *Mock) ListGCPServiceAttachments(ctx context.Context, region string) ([]driver.GCPResource, error) { + items, err := m.ListGCPResources(ctx, driver.GCPServiceAttachmentCollection, region) + for i := range items { + items[i].Body = cloneBody(items[i].Body) + } + + return items, err +} + +// UpdateGCPServiceAttachment applies mutate under the store lock, keeps the +// connected endpoints (they are output-only), validates the result and +// re-evaluates every connection against the new lists. +func (m *Mock) UpdateGCPServiceAttachment(_ context.Context, region, name string, + mutate func(*driver.GCPResource) error, +) error { + return m.updateServiceAttachment(region, name, func(res *driver.GCPResource) error { + endpoints := cloneBody(res.Body)[saConnectedEndpoints] + + next := *res + next.Body = cloneBody(res.Body) + + if err := mutate(&next); err != nil { + return err + } + + next.Body = cloneBody(next.Body) + delete(next.Body, saConnectedEndpoints) + + if endpoints != nil { + next.Body[saConnectedEndpoints] = endpoints + } + + if err := validateServiceAttachment(next.Body); err != nil { + return err + } + + *res = next + + return nil + }) +} + +// DeleteGCPServiceAttachment removes the named attachment, or NotFound. +func (m *Mock) DeleteGCPServiceAttachment(_ context.Context, region, name string) error { + if !m.gcpResources.Delete(gcpResourceKey(driver.GCPServiceAttachmentCollection, region, name)) { + return serviceAttachmentNotFound(name) + } + + return nil +} + +// ConnectGCPServiceAttachment appends a consumer endpoint and returns the +// status the evaluation gave it. +// +//nolint:gocritic // hugeParam: interface method signature is fixed. +func (m *Mock) ConnectGCPServiceAttachment(_ context.Context, region, name string, + ep driver.GCPPSCEndpoint, +) (string, error) { + status := "" + + err := m.updateServiceAttachment(region, name, func(res *driver.GCPResource) error { + body := cloneBody(res.Body) + eps := endpointsOf(body) + + eps = append(eps, map[string]any{ + epEndpoint: ep.Endpoint, + epPscConnectionID: ep.PscConnectionID, + epConsumerNetwork: ep.ConsumerNetwork, + }) + + setEndpoints(body, eps) + res.Body = body + + return nil + }) + if err != nil { + return "", err + } + + res, _ := m.gcpResources.Get(gcpResourceKey(driver.GCPServiceAttachmentCollection, region, name)) + for _, e := range endpointsOf(res.Body) { + if e[epPscConnectionID] == ep.PscConnectionID { + status, _ = e[epStatus].(string) + } + } + + return status, nil +} + +// DisconnectGCPServiceAttachment removes a consumer endpoint by +// pscConnectionId; a missing attachment or endpoint is not an error. +func (m *Mock) DisconnectGCPServiceAttachment(_ context.Context, region, name, pscConnectionID string) error { + err := m.updateServiceAttachment(region, name, func(res *driver.GCPResource) error { + body := cloneBody(res.Body) + eps := endpointsOf(body) + kept := eps[:0] + + for _, e := range eps { + if e[epPscConnectionID] != pscConnectionID { + kept = append(kept, e) + } + } + + setEndpoints(body, kept) + res.Body = body + + return nil + }) + if cerrors.IsNotFound(err) { + return nil + } + + return err +} + +// GCPPSCConnectionStatus returns a consumer endpoint's current status, or +// CLOSED when the attachment or the endpoint is gone. +func (m *Mock) GCPPSCConnectionStatus(_ context.Context, region, name, pscConnectionID string) string { + res, ok := m.gcpResources.Get(gcpResourceKey(driver.GCPServiceAttachmentCollection, region, name)) + if !ok { + return driver.PSCStatusClosed + } + + for _, e := range endpointsOf(res.Body) { + if e[epPscConnectionID] == pscConnectionID { + if s, _ := e[epStatus].(string); s != "" { + return s + } + } + } + + return driver.PSCStatusClosed +} + +// updateServiceAttachment runs mutate under the store lock and then +// re-evaluates every connection. A mutate error leaves the record unchanged. +func (m *Mock) updateServiceAttachment(region, name string, mutate func(*driver.GCPResource) error) error { + var mutateErr error + + updated := m.gcpResources.Update(gcpResourceKey(driver.GCPServiceAttachmentCollection, region, name), + func(res driver.GCPResource) driver.GCPResource { + next := res + if err := mutate(&next); err != nil { + mutateErr = err + return res + } + + evaluateConnections(next.Body) + + return next + }) + if !updated { + return serviceAttachmentNotFound(name) + } + + return mutateErr +} + +// validateServiceAttachment checks the members GCP requires or constrains. +func validateServiceAttachment(body map[string]any) error { + pref, _ := body[saConnectionPreference].(string) + if pref != driver.PSCAcceptAutomatic && pref != driver.PSCAcceptManual { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.connectionPreference': '%v'. Must be one of [ACCEPT_AUTOMATIC, ACCEPT_MANUAL].", + body[saConnectionPreference]) + } + + if target, _ := body[saTargetService].(string); target == "" { + return cerrors.New(cerrors.InvalidArgument, + "Invalid value for field 'resource.targetService': ''. The target service must be specified.") + } + + if subnets, _ := body[saNatSubnets].([]any); len(subnets) == 0 { + return cerrors.New(cerrors.InvalidArgument, + "Invalid value for field 'resource.natSubnets': ''. At least one NAT subnetwork must be specified.") + } + + accept, _ := body[saAcceptLists].([]any) + for i, raw := range accept { + entry, _ := raw.(map[string]any) + project, _ := entry[acceptProject].(string) + network, _ := entry[acceptNetwork].(string) + endpoint, _ := entry[acceptEndpoint].(string) + + if project == "" && network == "" && endpoint == "" { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.consumerAcceptLists[%d]': an entry must name a projectIdOrNum, networkUrl or endpointUrl.", i) + } + + if limit, present := entry[acceptLimit]; present { + if n, ok := jsonNumber(limit); !ok || n < 0 { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.consumerAcceptLists[%d].connectionLimit': '%v'.", i, limit) + } + } + } + + return nil +} + +// evaluateConnections recomputes every connected endpoint's status in +// connection order, so an earlier consumer keeps its place under a limit. +func evaluateConnections(body map[string]any) { + eps := endpointsOf(body) + if len(eps) == 0 { + return + } + + manual := body[saConnectionPreference] == driver.PSCAcceptManual + accept, _ := body[saAcceptLists].([]any) + used := make([]int64, len(accept)) + + for _, e := range eps { + endpoint, _ := e[epEndpoint].(string) + network, _ := e[epConsumerNetwork].(string) + + e[epStatus] = connectionStatus(manual, body, accept, used, pscConsumer{ + project: projectOf(endpoint), network: network, endpoint: endpoint, + }) + } + + setEndpoints(body, eps) +} + +// connectionStatus decides one endpoint's status, charging an accepted one +// against the accept-list entry that admitted it. +func connectionStatus(manual bool, body map[string]any, accept []any, used []int64, c pscConsumer) string { + if !manual { + return driver.PSCStatusAccepted + } + + reject, _ := body[saRejectLists].([]any) + for _, r := range reject { + if s, _ := r.(string); s != "" && (s == c.project || sameNetwork(s, c.network)) { + return driver.PSCStatusRejected + } + } + + for i, raw := range accept { + entry, _ := raw.(map[string]any) + if !c.matches(entry) { + continue + } + + if limit, limited := jsonNumber(entry[acceptLimit]); limited && used[i] >= limit { + return driver.PSCStatusPending + } + + used[i]++ + + return driver.PSCStatusAccepted + } + + return driver.PSCStatusPending +} + +// pscConsumer identifies a connecting endpoint for the accept/reject lists. +type pscConsumer struct { + project, network, endpoint string +} + +// matches reports whether an accept-list entry names this consumer by +// project, network or endpoint URL. +func (c pscConsumer) matches(entry map[string]any) bool { + p, _ := entry[acceptProject].(string) + n, _ := entry[acceptNetwork].(string) + e, _ := entry[acceptEndpoint].(string) + + return (p != "" && p == c.project) || sameNetwork(n, c.network) || (e != "" && sameNetwork(e, c.endpoint)) +} + +// sameNetwork compares two network references by their +// projects/{p}/global/networks/{n} tail, so a full URL and a relative path of +// the same network match. +func sameNetwork(a, b string) bool { + return a != "" && b != "" && networkTail(a) == networkTail(b) +} + +func networkTail(ref string) string { + if i := strings.Index(ref, "projects/"); i >= 0 { + return ref[i:] + } + + return ref +} + +// projectOf extracts the project id from a ".../projects/{p}/..." reference. +func projectOf(ref string) string { + const marker = "projects/" + + i := strings.Index(ref, marker) + if i < 0 { + return "" + } + + rest := ref[i+len(marker):] + if j := strings.IndexByte(rest, '/'); j >= 0 { + return rest[:j] + } + + return rest +} + +// endpointsOf reads the connectedEndpoints list of a body. +func endpointsOf(body map[string]any) []map[string]any { + raw, _ := body[saConnectedEndpoints].([]any) + out := make([]map[string]any, 0, len(raw)) + + for _, r := range raw { + if e, ok := r.(map[string]any); ok { + out = append(out, e) + } + } + + return out +} + +// setEndpoints writes the connectedEndpoints list, removing it when empty. +func setEndpoints(body map[string]any, eps []map[string]any) { + if len(eps) == 0 { + delete(body, saConnectedEndpoints) + return + } + + list := make([]any, 0, len(eps)) + for _, e := range eps { + list = append(list, e) + } + + body[saConnectedEndpoints] = list +} + +// jsonNumber reads an integral JSON value: a number, or a decimal string (the +// proto JSON encoding of an int64/uint32 field may be either). +func jsonNumber(v any) (int64, bool) { + switch t := v.(type) { + case float64: + return int64(t), true + case int64: + return t, true + case int: + return int64(t), true + case json.Number: + n, err := t.Int64() + return n, err == nil + case string: + n, err := json.Number(t).Int64() + return n, err == nil + default: + return 0, false + } +} + +// cloneBody deep-copies a decoded JSON body through a JSON round trip, so a +// caller never aliases the stored maps. +func cloneBody(body map[string]any) map[string]any { + out := map[string]any{} + + if body == nil { + return out + } + + b, err := json.Marshal(body) + if err != nil { + return out + } + + _ = json.Unmarshal(b, &out) + + return out +} + +// serviceAttachmentNotFound renders compute's not-found message. +func serviceAttachmentNotFound(name string) error { + return cerrors.Newf(cerrors.NotFound, "The resource 'serviceAttachments/%s' was not found", name) +} diff --git a/providers/gcp/loadbalancer/service_attachments_test.go b/providers/gcp/loadbalancer/service_attachments_test.go new file mode 100644 index 000000000..bbc35b370 --- /dev/null +++ b/providers/gcp/loadbalancer/service_attachments_test.go @@ -0,0 +1,154 @@ +package loadbalancer + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +const ( + testSARegion = "us-central1" + consumerNet = "projects/consumer/global/networks/vpc" + consumerEPURL = "https://compute.googleapis.com/compute/v1/projects/consumer/regions/us-central1/forwardingRules/" +) + +func saBody(pref string, extra map[string]any) map[string]any { + body := map[string]any{ + "connectionPreference": pref, + "targetService": "projects/p/regions/us-central1/forwardingRules/ilb", + "natSubnets": []any{"projects/p/regions/us-central1/subnetworks/nat"}, + } + + for k, v := range extra { + body[k] = v + } + + return body +} + +func endpoint(name string) driver.GCPPSCEndpoint { + return driver.GCPPSCEndpoint{Endpoint: consumerEPURL + name, PscConnectionID: name, ConsumerNetwork: consumerNet} +} + +// TestGCPServiceAttachmentStoreLibrary drives the capability directly (the Go +// library path): validation, CRUD, connection decisions by project, network +// and endpoint URL, a mutate error leaving the record untouched, and a +// snapshot round trip that keeps connectedEndpoints. +func TestGCPServiceAttachmentStoreLibrary(t *testing.T) { + ctx := context.Background() + m := newTestMock() + + insert := func(name string, body map[string]any) error { + return m.InsertGCPServiceAttachment(ctx, driver.GCPResource{Scope: testSARegion, Name: name, ID: "1", Body: body}) + } + + for name, body := range map[string]map[string]any{ + "preference": saBody("SOMETIMES", nil), + "target": saBody(driver.PSCAcceptAutomatic, map[string]any{"targetService": ""}), + "nat": saBody(driver.PSCAcceptAutomatic, map[string]any{"natSubnets": []any{}}), + "entry": saBody(driver.PSCAcceptManual, map[string]any{"consumerAcceptLists": []any{map[string]any{}}}), + "limit": saBody(driver.PSCAcceptManual, map[string]any{"consumerAcceptLists": []any{ + map[string]any{"projectIdOrNum": "consumer", "connectionLimit": -1.0}, + }}), + } { + assert.True(t, cerrors.IsInvalidArgument(insert("bad", body)), name) + } + + require.NoError(t, insert("by-network", saBody(driver.PSCAcceptManual, map[string]any{ + "consumerAcceptLists": []any{map[string]any{"networkUrl": "https://x/compute/v1/" + consumerNet}}, + }))) + require.NoError(t, insert("by-endpoint", saBody(driver.PSCAcceptManual, map[string]any{ + "consumerAcceptLists": []any{map[string]any{"endpointUrl": consumerEPURL + "ep-b", "connectionLimit": "1"}}, + }))) + require.NoError(t, insert("reject-net", saBody(driver.PSCAcceptManual, map[string]any{ + "consumerRejectLists": []any{consumerNet}, + // a client-sent connectedEndpoints is output-only and dropped. + "connectedEndpoints": []any{map[string]any{"pscConnectionId": "forged"}}, + }))) + assert.True(t, cerrors.IsAlreadyExists(insert("by-network", saBody(driver.PSCAcceptAutomatic, nil)))) + + for _, tc := range []struct{ sa, ep, want string }{ + {"by-network", "ep-a", driver.PSCStatusAccepted}, + {"by-endpoint", "ep-b", driver.PSCStatusAccepted}, + {"by-endpoint", "ep-c", driver.PSCStatusPending}, + {"reject-net", "ep-d", driver.PSCStatusRejected}, + } { + got, err := m.ConnectGCPServiceAttachment(ctx, testSARegion, tc.sa, endpoint(tc.ep)) + require.NoError(t, err) + assert.Equal(t, tc.want, got, tc.sa+"/"+tc.ep) + assert.Equal(t, tc.want, m.GCPPSCConnectionStatus(ctx, testSARegion, tc.sa, tc.ep)) + } + + _, err := m.ConnectGCPServiceAttachment(ctx, testSARegion, "ghost", endpoint("x")) + assert.True(t, cerrors.IsNotFound(err)) + assert.Equal(t, driver.PSCStatusClosed, m.GCPPSCConnectionStatus(ctx, testSARegion, "ghost", "x")) + assert.Equal(t, driver.PSCStatusClosed, m.GCPPSCConnectionStatus(ctx, testSARegion, "by-network", "unknown")) + + rejected, err := m.GetGCPServiceAttachment(ctx, testSARegion, "reject-net") + require.NoError(t, err) + assert.Len(t, rejected.Body["connectedEndpoints"], 1, "the forged endpoint was dropped") + + errReject := cerrors.New(cerrors.InvalidArgument, "rejected") + err = m.UpdateGCPServiceAttachment(ctx, testSARegion, "by-network", func(res *driver.GCPResource) error { + res.Body["connectionPreference"] = driver.PSCAcceptAutomatic + return errReject + }) + require.ErrorIs(t, err, errReject) + + got, err := m.GetGCPServiceAttachment(ctx, testSARegion, "by-network") + require.NoError(t, err) + assert.Equal(t, driver.PSCAcceptManual, got.Body["connectionPreference"]) + + // A successful update keeps connectedEndpoints (output-only) even when the + // mutation replaces the body, and re-evaluates them. + require.NoError(t, m.UpdateGCPServiceAttachment(ctx, testSARegion, "by-network", func(res *driver.GCPResource) error { + res.Body = saBody(driver.PSCAcceptManual, map[string]any{"consumerRejectLists": []any{"consumer"}}) + return nil + })) + assert.Equal(t, driver.PSCStatusRejected, m.GCPPSCConnectionStatus(ctx, testSARegion, "by-network", "ep-a")) + + assert.True(t, cerrors.IsInvalidArgument(m.UpdateGCPServiceAttachment(ctx, testSARegion, "by-network", + func(res *driver.GCPResource) error { + res.Body["natSubnets"] = nil + return nil + }))) + + require.NoError(t, insert("by-project", saBody(driver.PSCAcceptManual, map[string]any{ + "consumerAcceptLists": []any{map[string]any{"projectIdOrNum": "consumer", "connectionLimit": 1.0}}, + }))) + + for _, ep := range []string{"ep-p1", "ep-p2"} { + _, err := m.ConnectGCPServiceAttachment(ctx, testSARegion, "by-project", endpoint(ep)) + require.NoError(t, err) + } + + assert.Equal(t, driver.PSCStatusPending, m.GCPPSCConnectionStatus(ctx, testSARegion, "by-project", "ep-p2")) + require.NoError(t, m.DisconnectGCPServiceAttachment(ctx, testSARegion, "by-project", "ep-p1")) + assert.Equal(t, driver.PSCStatusAccepted, m.GCPPSCConnectionStatus(ctx, testSARegion, "by-project", "ep-p2"), + "a freed connection slot is re-evaluated") + require.NoError(t, m.DisconnectGCPServiceAttachment(ctx, testSARegion, "ghost", "x")) + + items, err := m.ListGCPServiceAttachments(ctx, testSARegion) + require.NoError(t, err) + assert.Len(t, items, 4) + + data, err := m.Snapshot(ctx, false) + require.NoError(t, err) + + restored := newTestMock() + require.NoError(t, restored.Restore(ctx, data)) + assert.Equal(t, driver.PSCStatusRejected, restored.GCPPSCConnectionStatus(ctx, testSARegion, "by-network", "ep-a")) + + require.NoError(t, m.DeleteGCPServiceAttachment(ctx, testSARegion, "by-network")) + assert.True(t, cerrors.IsNotFound(m.DeleteGCPServiceAttachment(ctx, testSARegion, "by-network"))) + assert.True(t, cerrors.IsNotFound(m.UpdateGCPServiceAttachment(ctx, testSARegion, "by-network", + func(*driver.GCPResource) error { return nil }))) + + _, err = m.GetGCPServiceAttachment(ctx, testSARegion, "by-network") + assert.True(t, cerrors.IsNotFound(err)) +} diff --git a/server/gcp/loadbalancer/forwardingrules_psc.go b/server/gcp/loadbalancer/forwardingrules_psc.go index c131cf12d..d7ddb8b13 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc.go +++ b/server/gcp/loadbalancer/forwardingrules_psc.go @@ -1,6 +1,7 @@ package loadbalancer import ( + "context" "strconv" "strings" @@ -23,9 +24,10 @@ const ( pscServiceAttachmentsSegment = "/serviceAttachments/" - // pscStatusAccepted is the connection status of a PSC rule the emulator - // created: there is no producer-side acceptance list to reject it. - pscStatusAccepted = "ACCEPTED" + // pscStatusAccepted is the connection status of a Google APIs bundle + // endpoint: there is no producer to decide it. A service-attachment + // endpoint's status comes from the attachment (GCPServiceAttachmentStore). + pscStatusAccepted = lbdriver.PSCStatusAccepted ) // isGoogleAPIsBundle reports whether target names a PSC Google APIs bundle. @@ -89,15 +91,68 @@ func pscInternalIP(lb *lbdriver.LBInfo) string { return "10." + strconv.Itoa(int(o2)) + "." + strconv.Itoa(int(o3)) + "." + strconv.Itoa(int(o4)) } +// pscConnectionID is the stable pscConnectionId of a PSC consumer rule. +func pscConnectionID(lb *lbdriver.LBInfo) string { + return strconv.FormatUint(positiveID(fnvHash("psc:"+lb.ID)), 10) +} + // applyPSCFields sets pscConnectionStatus and a stable pscConnectionId on a -// PSC consumer rule's response; other rules are left untouched. -func applyPSCFields(out *forwardingRuleResponse, lb *lbdriver.LBInfo) { - if !isPSCTarget(lb.Tags[frTargetTag]) { +// PSC consumer rule's response; other rules are left untouched. A Google APIs +// bundle endpoint is always ACCEPTED; a service-attachment endpoint reports +// the status the attachment gave it (CLOSED once the attachment is gone). +func (h *Handler) applyPSCFields(ctx context.Context, out *forwardingRuleResponse, lb *lbdriver.LBInfo) { + target := lb.Tags[frTargetTag] + if !isPSCTarget(target) { return } - id := positiveID(fnvHash("psc:" + lb.ID)) - + out.PscConnectionID = pscConnectionID(lb) out.PscConnectionStatus = pscStatusAccepted - out.PscConnectionID = strconv.FormatUint(id, 10) + + store, ok := h.serviceAttachmentStore() + if !ok { + return + } + + if region, name, parsed := attachmentRef(target); parsed { + out.PscConnectionStatus = store.GCPPSCConnectionStatus(ctx, region, name, out.PscConnectionID) + } +} + +// connectPSCEndpoint records a newly created service-attachment consumer rule +// on its attachment, which decides the connection's status. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) connectPSCEndpoint(ctx context.Context, rp gcprest.ResourcePath, host string, + req *forwardingRuleRequest, lb *lbdriver.LBInfo, +) error { + store, ok := h.serviceAttachmentStore() + if !ok { + return nil + } + + region, name, parsed := attachmentRef(req.Target) + if !parsed { + return nil + } + + _, err := store.ConnectGCPServiceAttachment(ctx, region, name, lbdriver.GCPPSCEndpoint{ + Endpoint: gcprest.SelfLink(host, rp.Project, rp.Scope, rp.ScopeName, resourceForwardingRules, req.Name), + PscConnectionID: pscConnectionID(lb), + ConsumerNetwork: req.Network, + }) + + return err +} + +// disconnectPSCEndpoint removes a deleted consumer rule from its attachment. +func (h *Handler) disconnectPSCEndpoint(ctx context.Context, lb *lbdriver.LBInfo) { + store, ok := h.serviceAttachmentStore() + if !ok { + return + } + + if region, name, parsed := attachmentRef(lb.Tags[frTargetTag]); parsed { + _ = store.DisconnectGCPServiceAttachment(ctx, region, name, pscConnectionID(lb)) + } } diff --git a/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go b/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go index 737ff08e7..2c6ae2b88 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go +++ b/server/gcp/loadbalancer/forwardingrules_psc_sdk_test.go @@ -65,13 +65,16 @@ func assertPSCRule(t *testing.T, got *computepb.ForwardingRule, wantIP, wantTarg // TestSDKGCPForwardingRulePSCServiceAttachment drives a regional Private // Service Connect consumer rule (target = a producer's serviceAttachments -// self-link) through the real ForwardingRulesRESTClient. The attachment lives -// in another project the emulator does not model, so it must not be rejected. +// self-link) through the real ForwardingRulesRESTClient. The producer +// attachment must exist (GCP refuses an endpoint for one it cannot find); it +// accepts automatically, so the connection is ACCEPTED. func TestSDKGCPForwardingRulePSCServiceAttachment(t *testing.T) { ts := newGCPLBServer(t) ctx := context.Background() client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + insertAttachment(ctx, t, ts, attachment("producer-sa", "ACCEPT_AUTOMATIC")) + op, err := client.Insert(ctx, &computepb.InsertForwardingRuleRequest{ Project: testProject, Region: testRegion, diff --git a/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go b/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go index e3a136c81..f26a129ea 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go +++ b/server/gcp/loadbalancer/forwardingrules_psc_validate_sdk_test.go @@ -84,6 +84,8 @@ func TestSDKGCPForwardingRulePSCInternalAddress(t *testing.T) { ctx := context.Background() client := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + insertAttachment(ctx, t, ts, attachment("producer-sa", "ACCEPT_AUTOMATIC")) + op, err := client.Insert(ctx, &computepb.InsertForwardingRuleRequest{ Project: testProject, Region: testRegion, ForwardingRuleResource: &computepb.ForwardingRule{ diff --git a/server/gcp/loadbalancer/handler.go b/server/gcp/loadbalancer/handler.go index c7a9d0fa9..c1c7cc74a 100644 --- a/server/gcp/loadbalancer/handler.go +++ b/server/gcp/loadbalancer/handler.go @@ -98,7 +98,7 @@ func (*Handler) Matches(r *http.Request) bool { } switch rp.ResourceType { - case resourceBackendServices, resourceForwardingRules, resourceBackendBuckets, + case resourceBackendServices, resourceForwardingRules, resourceBackendBuckets, resourceServiceAttachments, resourceHealthChecks, resourceTargetPools, resourceURLMaps, resourceTargetHTTPProxies, resourceTargetHTTPSProxies, resourceSslCertificates, resourceInstanceGroups, resourceRegionInstanceGroups: @@ -123,6 +123,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.routeForwardingRules(w, r, rp) case resourceBackendBuckets: h.routeBackendBuckets(w, r, rp) + case resourceServiceAttachments: + h.routeServiceAttachments(w, r, rp) case resourceHealthChecks, resourceTargetPools, resourceURLMaps: h.routeGCPResource(w, r, rp) case resourceTargetHTTPProxies, resourceTargetHTTPSProxies, resourceSslCertificates, diff --git a/server/gcp/loadbalancer/operations.go b/server/gcp/loadbalancer/operations.go index 9b4ef5b79..1bddfd86a 100644 --- a/server/gcp/loadbalancer/operations.go +++ b/server/gcp/loadbalancer/operations.go @@ -352,6 +352,21 @@ func (h *Handler) insertForwardingRule(w http.ResponseWriter, r *http.Request, r return } + // A service-attachment consumer rule connects to its attachment, which + // decides pscConnectionStatus. If the attachment vanished since validation + // the rule is not left behind. + if err := h.connectPSCEndpoint(r.Context(), rp, hostOf(r), &req, lb); err != nil { + _ = h.lb.DeleteLoadBalancer(r.Context(), lb.ARN) + + if cerrors.IsNotFound(err) { + err = invalidRefErr("target", req.Target, "serviceAttachment") + } + + gcprest.WriteCErr(w, err) + + return + } + // A forwarding rule that references a backend service becomes a listener // linking the load balancer to that target group. A dangling reference to a // non-existent backend service is an error (as in real GCP), and a failed @@ -448,6 +463,8 @@ func (h *Handler) deleteForwardingRule(w http.ResponseWriter, r *http.Request, r return } + h.disconnectPSCEndpoint(r.Context(), lb) + op := h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, resourceForwardingRules, rp.ResourceName, "delete") @@ -691,7 +708,7 @@ func (h *Handler) toForwardingRuleResponse(ctx context.Context, lb *lbdriver.LBI SelfLink: gcprest.SelfLink(host, rp.Project, rp.Scope, rp.ScopeName, resourceForwardingRules, name), } - applyPSCFields(&out, lb) + h.applyPSCFields(ctx, &out, lb) // A linked listener (a rule referencing a backend service) supersedes the // round-tripped protocol/portRange and adds the backendService self-link. diff --git a/server/gcp/loadbalancer/refvalidation.go b/server/gcp/loadbalancer/refvalidation.go index 65c088abe..f3f87f4f6 100644 --- a/server/gcp/loadbalancer/refvalidation.go +++ b/server/gcp/loadbalancer/refvalidation.go @@ -263,6 +263,10 @@ func (h *Handler) validateForwardingRuleTarget(ctx context.Context, rp gcprest.R return err } + if err := h.validateAttachmentTarget(ctx, rp, target); err != nil { + return err + } + collection := targetCollectionFor(target) if collection == "" { return nil diff --git a/server/gcp/loadbalancer/resources.go b/server/gcp/loadbalancer/resources.go index a1b79aa64..2523611ec 100644 --- a/server/gcp/loadbalancer/resources.go +++ b/server/gcp/loadbalancer/resources.go @@ -32,6 +32,7 @@ var resourceKind = map[string]string{ resourceTargetPools: "compute#targetPool", resourceURLMaps: "compute#urlMap", resourceBackendBuckets: "compute#backendBucket", + resourceServiceAttachments: "compute#serviceAttachment", resourceTargetHTTPProxies: "compute#targetHttpProxy", resourceTargetHTTPSProxies: "compute#targetHttpsProxy", resourceSslCertificates: "compute#sslCertificate", diff --git a/server/gcp/loadbalancer/serviceattachments.go b/server/gcp/loadbalancer/serviceattachments.go new file mode 100644 index 000000000..2d7c81ac2 --- /dev/null +++ b/server/gcp/loadbalancer/serviceattachments.go @@ -0,0 +1,206 @@ +package loadbalancer + +import ( + "context" + "net/http" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/server/wire/gcprest" + lbdriver "github.com/stackshy/cloudemu/v2/services/loadbalancer/driver" +) + +// compute.serviceAttachments: the producer side of Private Service Connect. +// A regional resource publishing a producer forwarding rule (targetService) +// through NAT subnets, with a connectionPreference and consumer accept/reject +// lists that decide which consumer PSC endpoints connect. +// +// Surface: insert, get, list, patch (JSON merge patch), delete. The records and +// the connection decisions live in the provider (GCPServiceAttachmentStore); +// this handler only shapes the wire. Every mutation answers a DONE +// compute#operation in the shared OperationRegistry. +const resourceServiceAttachments = lbdriver.GCPServiceAttachmentCollection + +// serviceAttachmentOutputOnly are members a client may echo back from a get +// but the server owns. +// +//nolint:gochecknoglobals // immutable lookup table, not mutable state +var serviceAttachmentOutputOnly = []string{ + fieldID, fieldKind, fieldSelfLink, fieldCreationTimestamp, "region", "fingerprint", + "connectedEndpoints", "pscServiceAttachmentId", +} + +// serviceAttachmentStore returns the capability, or false when the driver +// does not implement it. +func (h *Handler) serviceAttachmentStore() (lbdriver.GCPServiceAttachmentStore, bool) { + s, ok := h.lb.(lbdriver.GCPServiceAttachmentStore) + + return s, ok +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeServiceAttachments(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath) { + store, ok := h.serviceAttachmentStore() + if !ok { + gcprest.WriteError(w, http.StatusNotImplemented, "notImplemented", + "load balancer driver has no service attachment store") + + return + } + + if rp.Scope != gcprest.ScopeRegions { + gcprest.WriteError(w, http.StatusNotFound, "notFound", "serviceAttachments are regional resources") + return + } + + switch { + case rp.ResourceName == "" && r.Method == http.MethodPost: + h.insertServiceAttachment(w, r, rp, store) + case rp.ResourceName == "" && r.Method == http.MethodGet: + items, err := store.ListGCPServiceAttachments(r.Context(), rp.ScopeName) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + writeGCPResourceList(w, r, rp, items) + case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodGet: + res, err := store.GetGCPServiceAttachment(r.Context(), rp.ScopeName, rp.ResourceName) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + gcprest.WriteJSON(w, http.StatusOK, gcpResourceJSON(res, rp, hostOf(r))) + case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodPatch: + h.patchServiceAttachment(w, r, rp, store) + case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodDelete: + if err := store.DeleteGCPServiceAttachment(r.Context(), rp.ScopeName, rp.ResourceName); err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeServiceAttachmentOp(w, r, rp, rp.ResourceName, opDelete) + default: + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) insertServiceAttachment(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPServiceAttachmentStore, +) { + var body map[string]any + if !gcprest.DecodeJSON(w, r, &body) { + return + } + + name, _ := body["name"].(string) + if err := validateRFC1035Name(name); err != nil { + gcprest.WriteCErr(w, err) + return + } + + for _, k := range serviceAttachmentOutputOnly { + delete(body, k) + } + + err := store.InsertGCPServiceAttachment(r.Context(), lbdriver.GCPResource{ + Scope: rp.ScopeName, + Name: name, + ID: numericID(resourceServiceAttachments + "/" + rp.ScopeName + "/" + name), + CreationTimestamp: time.Now().UTC().Format(time.RFC3339), + Body: body, + }) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeServiceAttachmentOp(w, r, rp, name, opInsert) +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) patchServiceAttachment(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPServiceAttachmentStore, +) { + var patch map[string]any + if !gcprest.DecodeJSON(w, r, &patch) { + return + } + + for _, k := range serviceAttachmentOutputOnly { + delete(patch, k) + } + + err := store.UpdateGCPServiceAttachment(r.Context(), rp.ScopeName, rp.ResourceName, + func(res *lbdriver.GCPResource) error { + mergePatch(res.Body, patch) + res.Body["name"] = res.Name + + return nil + }) + if err != nil { + gcprest.WriteCErr(w, err) + return + } + + h.writeServiceAttachmentOp(w, r, rp, rp.ResourceName, opPatch) +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) writeServiceAttachmentOp(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, name, opType string) { + gcprest.WriteJSON(w, http.StatusOK, h.ops.RecordDone(hostOf(r), rp.Project, rp.Scope, rp.ScopeName, + resourceServiceAttachments, name, opType)) +} + +// attachmentRef splits a ".../regions/{r}/serviceAttachments/{n}" target into +// its region and name. +func attachmentRef(ref string) (region, name string, ok bool) { + i := strings.Index(ref, pscServiceAttachmentsSegment) + if i < 0 { + return "", "", false + } + + name = ref[i+len(pscServiceAttachmentsSegment):] + + const regionsMarker = "regions/" + + j := strings.LastIndex(ref[:i], regionsMarker) + if j < 0 || name == "" || strings.Contains(name, "/") { + return "", "", false + } + + return ref[j+len(regionsMarker) : i], name, true +} + +// validateAttachmentTarget checks a PSC consumer rule's service-attachment +// target: the rule must be regional, in the attachment's region, and the +// attachment must exist — GCP refuses an endpoint for an attachment it cannot +// find. +// +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) validateAttachmentTarget(ctx context.Context, rp gcprest.ResourcePath, target string) error { + store, ok := h.serviceAttachmentStore() + if !ok || !strings.Contains(target, pscServiceAttachmentsSegment) { + return nil + } + + region, name, parsed := attachmentRef(target) + if !parsed || rp.Scope != gcprest.ScopeRegions || region != rp.ScopeName { + return cerrors.Newf(cerrors.InvalidArgument, + "Invalid value for field 'resource.target': '%s'. A service attachment target must be in the forwarding rule's region.", + target) + } + + if _, err := store.GetGCPServiceAttachment(ctx, region, name); err != nil { + if cerrors.IsNotFound(err) { + return invalidRefErr("target", target, "serviceAttachment") + } + + return err + } + + return nil +} diff --git a/server/gcp/loadbalancer/serviceattachments_sdk_test.go b/server/gcp/loadbalancer/serviceattachments_sdk_test.go new file mode 100644 index 000000000..747c0cc53 --- /dev/null +++ b/server/gcp/loadbalancer/serviceattachments_sdk_test.go @@ -0,0 +1,289 @@ +package loadbalancer_test + +import ( + "context" + "net/http/httptest" + "testing" + + gcpcompute "cloud.google.com/go/compute/apiv1" + computepb "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/option" +) + +const ( + saProducerRule = "projects/" + testProject + "/regions/" + testRegion + "/forwardingRules/producer-ilb" + saNatSubnet = "projects/" + testProject + "/regions/" + testRegion + "/subnetworks/psc-nat" +) + +func newServiceAttachmentsClient(t *testing.T, ts *httptest.Server) *gcpcompute.ServiceAttachmentsClient { + t.Helper() + + c, err := gcpcompute.NewServiceAttachmentsRESTClient(context.Background(), clientOpts(ts)...) + if err != nil { + t.Fatalf("NewServiceAttachmentsRESTClient: %v", err) + } + + t.Cleanup(func() { _ = c.Close() }) + + return c +} + +// attachment builds a valid service attachment named name. +func attachment(name, preference string) *computepb.ServiceAttachment { + return &computepb.ServiceAttachment{ + Name: ptrStr(name), + TargetService: ptrStr(saProducerRule), + NatSubnets: []string{saNatSubnet}, + ConnectionPreference: ptrStr(preference), + } +} + +// insertAttachment inserts sa in testRegion, failing the test on error. +func insertAttachment(ctx context.Context, t *testing.T, ts *httptest.Server, sa *computepb.ServiceAttachment) { + t.Helper() + + c := newServiceAttachmentsClient(t, ts) + + waitOp(ctx, t, "ServiceAttachment Insert "+sa.GetName(), func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertServiceAttachmentRequest{ + Project: testProject, Region: testRegion, ServiceAttachmentResource: sa, + }) + }) +} + +func getAttachment(ctx context.Context, t *testing.T, c *gcpcompute.ServiceAttachmentsClient, name string) *computepb.ServiceAttachment { + t.Helper() + + got, err := c.Get(ctx, &computepb.GetServiceAttachmentRequest{Project: testProject, Region: testRegion, ServiceAttachment: name}) + if err != nil { + t.Fatalf("ServiceAttachment Get %s: %v", name, err) + } + + return got +} + +// TestSDKGCPServiceAttachmentLifecycle drives compute.serviceAttachments +// (501 before) through the real ServiceAttachmentsClient: insert, get, list, +// patch, delete, and the 400/404/409 refusals. +func TestSDKGCPServiceAttachmentLifecycle(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + c := newServiceAttachmentsClient(t, ts) + + sa := attachment("producer-sa", "ACCEPT_MANUAL") + sa.ConsumerAcceptLists = []*computepb.ServiceAttachmentConsumerProjectLimit{ + {ProjectIdOrNum: ptrStr("consumer-a"), ConnectionLimit: func() *uint32 { v := uint32(5); return &v }()}, + } + sa.ConsumerRejectLists = []string{"consumer-bad"} + sa.EnableProxyProtocol = ptrBool(true) + insertAttachment(ctx, t, ts, sa) + + got := getAttachment(ctx, t, c, "producer-sa") + if got.GetKind() != "compute#serviceAttachment" || got.GetId() == 0 || got.GetTargetService() != saProducerRule || + got.GetConnectionPreference() != "ACCEPT_MANUAL" || len(got.GetNatSubnets()) != 1 || !got.GetEnableProxyProtocol() || + len(got.GetConsumerAcceptLists()) != 1 || got.GetConsumerAcceptLists()[0].GetConnectionLimit() != 5 || + len(got.GetConsumerRejectLists()) != 1 || got.GetRegion() == "" || got.GetSelfLink() == "" { + t.Fatalf("Get = %+v, want the inserted attachment", got) + } + + insert := func(sa *computepb.ServiceAttachment) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Insert(ctx, &computepb.InsertServiceAttachmentRequest{ + Project: testProject, Region: testRegion, ServiceAttachmentResource: sa, + }) + }) + } + + assertHTTPCode(t, insert(attachment("producer-sa", "ACCEPT_AUTOMATIC")), 409) + assertHTTPCode(t, insert(attachment("Bad_Name", "ACCEPT_AUTOMATIC")), 400) + assertHTTPCode(t, insert(attachment("bad-pref", "ACCEPT_SOMETIMES")), 400) + + noNat := attachment("no-nat", "ACCEPT_AUTOMATIC") + noNat.NatSubnets = nil + assertHTTPCode(t, insert(noNat), 400) + + noTarget := attachment("no-target", "ACCEPT_AUTOMATIC") + noTarget.TargetService = nil + assertHTTPCode(t, insert(noTarget), 400) + + emptyEntry := attachment("empty-entry", "ACCEPT_MANUAL") + emptyEntry.ConsumerAcceptLists = []*computepb.ServiceAttachmentConsumerProjectLimit{{}} + assertHTTPCode(t, insert(emptyEntry), 400) + + insertAttachment(ctx, t, ts, attachment("second-sa", "ACCEPT_AUTOMATIC")) + + it := c.List(ctx, &computepb.ListServiceAttachmentsRequest{Project: testProject, Region: testRegion}) + + var names []string + + for { + sa, err := it.Next() + if err != nil { + break + } + + names = append(names, sa.GetName()) + } + + if len(names) != 2 || names[0] != "producer-sa" || names[1] != "second-sa" { + t.Fatalf("List = %v, want [producer-sa second-sa]", names) + } + + patch := func(name string, sa *computepb.ServiceAttachment) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Patch(ctx, &computepb.PatchServiceAttachmentRequest{ + Project: testProject, Region: testRegion, ServiceAttachment: name, ServiceAttachmentResource: sa, + }) + }) + } + + if err := patch("producer-sa", &computepb.ServiceAttachment{ConnectionPreference: ptrStr("ACCEPT_AUTOMATIC")}); err != nil { + t.Fatalf("Patch: %v", err) + } + + if got := getAttachment(ctx, t, c, "producer-sa"); got.GetConnectionPreference() != "ACCEPT_AUTOMATIC" || + got.GetTargetService() != saProducerRule { + t.Fatalf("after Patch: preference=%q targetService=%q", got.GetConnectionPreference(), got.GetTargetService()) + } + + assertHTTPCode(t, patch("producer-sa", &computepb.ServiceAttachment{ConnectionPreference: ptrStr("NOPE")}), 400) + assertHTTPCode(t, patch("ghost-sa", &computepb.ServiceAttachment{Description: ptrStr("x")}), 404) + + if err := callOp(ctx, func() (*gcpcompute.Operation, error) { + return c.Delete(ctx, &computepb.DeleteServiceAttachmentRequest{Project: testProject, Region: testRegion, ServiceAttachment: "second-sa"}) + }); err != nil { + t.Fatalf("Delete: %v", err) + } + + _, err := c.Get(ctx, &computepb.GetServiceAttachmentRequest{Project: testProject, Region: testRegion, ServiceAttachment: "second-sa"}) + assertHTTPCode(t, err, 404) +} + +// TestSDKGCPPSCStatusFromServiceAttachment: a consumer rule's +// pscConnectionStatus comes from the attachment it targets. Under +// ACCEPT_MANUAL an accept-listed project is ACCEPTED up to its +// connectionLimit and PENDING beyond it, a reject-listed one is REJECTED, an +// unlisted one is PENDING; statuses re-evaluate when the lists change, the +// attachment lists its connectedEndpoints, and deleting the attachment leaves +// its consumers CLOSED. +func TestSDKGCPPSCStatusFromServiceAttachment(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + sac := newServiceAttachmentsClient(t, ts) + frc := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + one := uint32(1) + sa := attachment("manual-sa", "ACCEPT_MANUAL") + sa.ConsumerAcceptLists = []*computepb.ServiceAttachmentConsumerProjectLimit{ + {ProjectIdOrNum: ptrStr(testProject), ConnectionLimit: &one}, + } + insertAttachment(ctx, t, ts, sa) + insertAttachment(ctx, t, ts, attachment("unlisted-sa", "ACCEPT_MANUAL")) + + target := "projects/" + testProject + "/regions/" + testRegion + "/serviceAttachments/" + + connect := func(name, attachment string) { + t.Helper() + + waitOp(ctx, t, "ForwardingRule Insert "+name, func() (*gcpcompute.Operation, error) { + return frc.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr(name), Network: ptrStr(pscNetwork), Target: ptrStr(target + attachment), + }, + }) + }) + } + + status := func(name string) string { + t.Helper() + + got, err := frc.Get(ctx, &computepb.GetForwardingRuleRequest{Project: testProject, Region: testRegion, ForwardingRule: name}) + if err != nil { + t.Fatalf("Get %s: %v", name, err) + } + + return got.GetPscConnectionStatus() + } + + want := func(what string, pairs ...string) { + t.Helper() + + for i := 0; i < len(pairs); i += 2 { + if got := status(pairs[i]); got != pairs[i+1] { + t.Errorf("%s: %s pscConnectionStatus = %q, want %s", what, pairs[i], got, pairs[i+1]) + } + } + } + + connect("ep-1", "manual-sa") + connect("ep-2", "manual-sa") + connect("ep-unlisted", "unlisted-sa") + want("limit 1", "ep-1", "ACCEPTED", "ep-2", "PENDING", "ep-unlisted", "PENDING") + + if eps := getAttachment(ctx, t, sac, "manual-sa").GetConnectedEndpoints(); len(eps) != 2 || + eps[0].GetStatus() != "ACCEPTED" || eps[1].GetStatus() != "PENDING" || eps[0].GetPscConnectionId() == 0 || + eps[0].GetConsumerNetwork() != pscNetwork { + t.Fatalf("connectedEndpoints = %v, want ep-1 ACCEPTED then ep-2 PENDING", eps) + } + + patch := func(sa *computepb.ServiceAttachment) { + t.Helper() + + waitOp(ctx, t, "ServiceAttachment Patch", func() (*gcpcompute.Operation, error) { + return sac.Patch(ctx, &computepb.PatchServiceAttachmentRequest{ + Project: testProject, Region: testRegion, ServiceAttachment: "manual-sa", ServiceAttachmentResource: sa, + }) + }) + } + + two := uint32(2) + patch(&computepb.ServiceAttachment{ConsumerAcceptLists: []*computepb.ServiceAttachmentConsumerProjectLimit{ + {ProjectIdOrNum: ptrStr(testProject), ConnectionLimit: &two}, + }}) + want("limit raised to 2", "ep-1", "ACCEPTED", "ep-2", "ACCEPTED") + + patch(&computepb.ServiceAttachment{ConsumerRejectLists: []string{testProject}}) + want("project rejected", "ep-1", "REJECTED", "ep-2", "REJECTED") + + waitOp(ctx, t, "ForwardingRule Delete", func() (*gcpcompute.Operation, error) { + return frc.Delete(ctx, &computepb.DeleteForwardingRuleRequest{Project: testProject, Region: testRegion, ForwardingRule: "ep-2"}) + }) + + if eps := getAttachment(ctx, t, sac, "manual-sa").GetConnectedEndpoints(); len(eps) != 1 { + t.Fatalf("connectedEndpoints after deleting ep-2 = %v, want 1", eps) + } + + waitOp(ctx, t, "ServiceAttachment Delete", func() (*gcpcompute.Operation, error) { + return sac.Delete(ctx, &computepb.DeleteServiceAttachmentRequest{Project: testProject, Region: testRegion, ServiceAttachment: "manual-sa"}) + }) + want("attachment deleted", "ep-1", "CLOSED") +} + +// TestSDKGCPPSCAttachmentTargetMustExist: GCP refuses a PSC endpoint for a +// service attachment it cannot find, or one in another region. +func TestSDKGCPPSCAttachmentTargetMustExist(t *testing.T) { + ts := newGCPLBServer(t) + ctx := context.Background() + frc := newRegionalForwardingRulesClient(t, ts.URL, option.WithHTTPClient(ts.Client())) + + insertAttachment(ctx, t, ts, attachment("real-sa", "ACCEPT_AUTOMATIC")) + + insert := func(target string) error { + return callOp(ctx, func() (*gcpcompute.Operation, error) { + return frc.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: testProject, Region: testRegion, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: ptrStr("ep"), Network: ptrStr(pscNetwork), Target: ptrStr(target), + }, + }) + }) + } + + wantBadRequest(t, insert("projects/"+testProject+"/regions/"+testRegion+"/serviceAttachments/nosuch"), "target") + wantBadRequest(t, insert("projects/"+testProject+"/regions/europe-west1/serviceAttachments/real-sa"), "target") + + if err := insert("projects/" + testProject + "/regions/" + testRegion + "/serviceAttachments/real-sa"); err != nil { + t.Fatalf("Insert against an existing attachment: %v", err) + } +} diff --git a/services/loadbalancer/driver/gcp_service_attachments.go b/services/loadbalancer/driver/gcp_service_attachments.go new file mode 100644 index 000000000..c7e810f47 --- /dev/null +++ b/services/loadbalancer/driver/gcp_service_attachments.go @@ -0,0 +1,80 @@ +package driver + +import "context" + +// GCPServiceAttachmentCollection is the Collection a Private Service Connect +// service attachment is stored under. +const GCPServiceAttachmentCollection = "serviceAttachments" + +// Private Service Connect connection statuses, as reported on a consumer +// forwarding rule's pscConnectionStatus and on the producer attachment's +// connectedEndpoints[].status. +const ( + PSCStatusAccepted = "ACCEPTED" + PSCStatusPending = "PENDING" + PSCStatusRejected = "REJECTED" + // PSCStatusClosed is reported for a consumer rule whose attachment has + // been deleted. + PSCStatusClosed = "CLOSED" +) + +// Connection preferences a service attachment accepts. +const ( + PSCAcceptAutomatic = "ACCEPT_AUTOMATIC" + PSCAcceptManual = "ACCEPT_MANUAL" +) + +// GCPPSCEndpoint is a consumer PSC forwarding rule connecting to a service +// attachment. +type GCPPSCEndpoint struct { + // Endpoint is the consumer forwarding rule's URL; its projects/{p} + // segment is the consumer project the accept/reject lists match. + Endpoint string + // PscConnectionID is the consumer rule's pscConnectionId. + PscConnectionID string + // ConsumerNetwork is the consumer rule's network. + ConsumerNetwork string +} + +// GCPServiceAttachmentStore is an OPTIONAL, type-asserted capability +// implemented only by the GCP load-balancer provider. It persists regional +// compute.serviceAttachments (the producer side of Private Service Connect) as +// GCPResource values (Collection GCPServiceAttachmentCollection, Scope = the +// region) alongside the other opaque GCP resources, so they snapshot and +// restore with them, and it owns the connection decisions for the consumer +// rules that target them. Non-GCP providers do not implement it. +// +// A connection's status follows the attachment's connectionPreference: +// ACCEPT_AUTOMATIC accepts every consumer; ACCEPT_MANUAL rejects a consumer +// whose project or network is in consumerRejectLists, accepts one matched by a +// consumerAcceptLists entry while that entry's connectionLimit has room, and +// leaves every other one PENDING. Statuses are re-evaluated, in connection +// order, whenever the attachment changes. +type GCPServiceAttachmentStore interface { + // InsertGCPServiceAttachment validates and stores a new attachment, + // returning AlreadyExists when (region, name) is taken. + InsertGCPServiceAttachment(ctx context.Context, res GCPResource) error + // GetGCPServiceAttachment returns the attachment, or NotFound. + GetGCPServiceAttachment(ctx context.Context, region, name string) (*GCPResource, error) + // ListGCPServiceAttachments returns every attachment in a region. + ListGCPServiceAttachments(ctx context.Context, region string) ([]GCPResource, error) + // UpdateGCPServiceAttachment applies mutate under the store lock, keeps the + // attachment's connectedEndpoints, validates the result and re-evaluates + // every connection. A mutate or validation error leaves the record + // unchanged. Returns NotFound when absent. + UpdateGCPServiceAttachment(ctx context.Context, region, name string, mutate func(*GCPResource) error) error + // DeleteGCPServiceAttachment removes the attachment, or returns NotFound. + // Its consumer rules then report PSCStatusClosed. + DeleteGCPServiceAttachment(ctx context.Context, region, name string) error + // ConnectGCPServiceAttachment records a consumer endpoint on the attachment + // and returns the status it was given. Returns NotFound when the + // attachment does not exist. + ConnectGCPServiceAttachment(ctx context.Context, region, name string, ep GCPPSCEndpoint) (string, error) + // DisconnectGCPServiceAttachment removes a consumer endpoint (by + // pscConnectionId) and re-evaluates the rest; an absent attachment or + // endpoint is not an error. + DisconnectGCPServiceAttachment(ctx context.Context, region, name, pscConnectionID string) error + // GCPPSCConnectionStatus returns the current status of a consumer + // endpoint, or PSCStatusClosed when the attachment or endpoint is gone. + GCPPSCConnectionStatus(ctx context.Context, region, name, pscConnectionID string) string +} From 5ba843dd5a4f737c3538b91216844eee69b79195 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:40:49 +0530 Subject: [PATCH 20/24] test(gcp-vpc): cover address delete through the provider store The snapshot round-trip test now also deletes an address on the restored emulator and checks the 404s, the only path of the provider-backed address store the vpc package tests did not reach. --- server/gcp/vpc/address_snapshot_sdk_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/gcp/vpc/address_snapshot_sdk_test.go b/server/gcp/vpc/address_snapshot_sdk_test.go index 5e546f194..75f337454 100644 --- a/server/gcp/vpc/address_snapshot_sdk_test.go +++ b/server/gcp/vpc/address_snapshot_sdk_test.go @@ -102,4 +102,14 @@ func TestSDKAddressLabelsSurviveSnapshotRestore(t *testing.T) { if next.Address == after.Address { t.Errorf("new address got IP %s, already held by the restored address", next.Address) } + + if _, err := restored.GlobalAddresses.Delete(testProject, "next-range").Context(ctx).Do(); err != nil { + t.Fatalf("Delete: %v", err) + } + + _, err = restored.GlobalAddresses.Get(testProject, "next-range").Context(ctx).Do() + wantStatus(t, "Get after Delete", err, 404) + + _, err = restored.GlobalAddresses.Delete(testProject, "next-range").Context(ctx).Do() + wantStatus(t, "Delete twice", err, 404) } From cefe314d5f040188fb8515885b135b89a1fb7fc9 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:49:01 +0530 Subject: [PATCH 21/24] refactor(gcp-lb): split service-attachment routing and validation for lint Split routeServiceAttachments into collection and item routers and the consumerAcceptLists checks out of validateServiceAttachment (gocyclo), wrap a long PSC error string (lll), and drop two nolint directives that suppressed nothing (nolintlint). No behaviour change. --- .../gcp/loadbalancer/service_attachments.go | 8 +++-- providers/gcp/vpc/addresses.go | 2 -- .../gcp/loadbalancer/forwardingrules_psc.go | 3 +- server/gcp/loadbalancer/serviceattachments.go | 34 ++++++++++++++++--- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/providers/gcp/loadbalancer/service_attachments.go b/providers/gcp/loadbalancer/service_attachments.go index d92f8398b..e4639078d 100644 --- a/providers/gcp/loadbalancer/service_attachments.go +++ b/providers/gcp/loadbalancer/service_attachments.go @@ -121,8 +121,6 @@ func (m *Mock) DeleteGCPServiceAttachment(_ context.Context, region, name string // ConnectGCPServiceAttachment appends a consumer endpoint and returns the // status the evaluation gave it. -// -//nolint:gocritic // hugeParam: interface method signature is fixed. func (m *Mock) ConnectGCPServiceAttachment(_ context.Context, region, name string, ep driver.GCPPSCEndpoint, ) (string, error) { @@ -245,6 +243,12 @@ func validateServiceAttachment(body map[string]any) error { "Invalid value for field 'resource.natSubnets': ''. At least one NAT subnetwork must be specified.") } + return validateAcceptLists(body) +} + +// validateAcceptLists checks each consumerAcceptLists entry names a consumer +// and carries a non-negative connectionLimit. +func validateAcceptLists(body map[string]any) error { accept, _ := body[saAcceptLists].([]any) for i, raw := range accept { entry, _ := raw.(map[string]any) diff --git a/providers/gcp/vpc/addresses.go b/providers/gcp/vpc/addresses.go index fe81f0aac..517c27a19 100644 --- a/providers/gcp/vpc/addresses.go +++ b/providers/gcp/vpc/addresses.go @@ -42,8 +42,6 @@ func cloneAddress(a *driver.GCPAddress) driver.GCPAddress { // InsertGCPAddress stores a new reserved address and stamps its // labelFingerprint from the labels it was created with. -// -//nolint:gocritic // hugeParam: interface method signature is fixed. func (m *Mock) InsertGCPAddress(_ context.Context, addr driver.GCPAddress) error { obj, err := addressObject(addr.Body) if err != nil { diff --git a/server/gcp/loadbalancer/forwardingrules_psc.go b/server/gcp/loadbalancer/forwardingrules_psc.go index d7ddb8b13..8af11ecd0 100644 --- a/server/gcp/loadbalancer/forwardingrules_psc.go +++ b/server/gcp/loadbalancer/forwardingrules_psc.go @@ -69,7 +69,8 @@ func validatePSCTarget(rp gcprest.ResourcePath, req *forwardingRuleRequest) erro if req.LoadBalancingScheme != "" { return cerrors.Newf(cerrors.InvalidArgument, - "Invalid value for field 'resource.loadBalancingScheme': '%s'. The load balancing scheme must be empty for a Private Service Connect forwarding rule.", + "Invalid value for field 'resource.loadBalancingScheme': '%s'. "+ + "The load balancing scheme must be empty for a Private Service Connect forwarding rule.", req.LoadBalancingScheme) } diff --git a/server/gcp/loadbalancer/serviceattachments.go b/server/gcp/loadbalancer/serviceattachments.go index 2d7c81ac2..a28c0509e 100644 --- a/server/gcp/loadbalancer/serviceattachments.go +++ b/server/gcp/loadbalancer/serviceattachments.go @@ -55,9 +55,23 @@ func (h *Handler) routeServiceAttachments(w http.ResponseWriter, r *http.Request } switch { - case rp.ResourceName == "" && r.Method == http.MethodPost: + case rp.ResourceName == "": + h.routeServiceAttachmentCollection(w, r, rp, store) + case rp.Action == "": + h.routeServiceAttachmentItem(w, r, rp, store) + default: + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeServiceAttachmentCollection(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPServiceAttachmentStore, +) { + switch r.Method { + case http.MethodPost: h.insertServiceAttachment(w, r, rp, store) - case rp.ResourceName == "" && r.Method == http.MethodGet: + case http.MethodGet: items, err := store.ListGCPServiceAttachments(r.Context(), rp.ScopeName) if err != nil { gcprest.WriteCErr(w, err) @@ -65,7 +79,17 @@ func (h *Handler) routeServiceAttachments(w http.ResponseWriter, r *http.Request } writeGCPResourceList(w, r, rp, items) - case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodGet: + default: + gcprest.WriteError(w, http.StatusMethodNotAllowed, "methodNotAllowed", "method not allowed") + } +} + +//nolint:gocritic // rp is a request-scoped value +func (h *Handler) routeServiceAttachmentItem(w http.ResponseWriter, r *http.Request, rp gcprest.ResourcePath, + store lbdriver.GCPServiceAttachmentStore, +) { + switch r.Method { + case http.MethodGet: res, err := store.GetGCPServiceAttachment(r.Context(), rp.ScopeName, rp.ResourceName) if err != nil { gcprest.WriteCErr(w, err) @@ -73,9 +97,9 @@ func (h *Handler) routeServiceAttachments(w http.ResponseWriter, r *http.Request } gcprest.WriteJSON(w, http.StatusOK, gcpResourceJSON(res, rp, hostOf(r))) - case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodPatch: + case http.MethodPatch: h.patchServiceAttachment(w, r, rp, store) - case rp.ResourceName != "" && rp.Action == "" && r.Method == http.MethodDelete: + case http.MethodDelete: if err := store.DeleteGCPServiceAttachment(r.Context(), rp.ScopeName, rp.ResourceName); err != nil { gcprest.WriteCErr(w, err) return From db12f2ffdac80b277e7c6cdf3f6669384278834b Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 14:50:34 +0530 Subject: [PATCH 22/24] fix(azure-apim): serve azurerm's create/refresh/destroy surface, rotate etags, enforce tier and name rules Review of #1354 found that azurerm_api_management could not be created and that several ARM behaviours of Microsoft.ApiManagement/service were wrong. Terraform surface. azurerm v4 (api_management_resource.go) GETs locations/{l}/deletedservices/{name} before every create (501 here), lists and deletes the sample Echo API and the Starter/Unlimited products after create, PUTs portalsettings signin/signup, reads policies/policy, portalsettings signin/signup/delegation (+ delegation/listSecrets) and tenant/access/listSecrets on refresh, and on destroy (purge_soft_delete_on_ destroy defaults to true) GETs the soft-deleted service and purges it. The provider now models soft delete as Azure does since 2020-06-01-preview: a delete (or a resource-group delete) keeps the service for 48 hours under deletedservices, where it can be read, listed, purged or recovered by a PUT with properties.restore. A new non-Consumption service is born with echo-api and the starter/unlimited products; the service policy, portal settings and tenant access (get/patch/listSecrets) are served. The sequence is replayed through the official armapimanagement clients; no terraform binary was run, so the package doc now claims only what that proves. Etag. The etag was minted once and never changed. It now changes on every PUT/PATCH (and on every child write), and a non-wildcard If-Match that does not match is 412 PreconditionFailed on the service and on the children. Tier rules. Capacity has per-tier ceilings (Developer 1, Basic 2, Standard 4, Premium 12, BasicV2/StandardV2 10) and zones are Premium-only, on create and on PATCH. Name and location. The service name is a global *.azure-api.net label: a second service of that name in any group or subscription is 409, a name held by a soft-deleted service is 409 until it is purged or recovered, and POST .../checkNameAvailability answers instead of 501. A PUT naming another location on an existing service is 409 InvalidResourceLocation instead of a 200 that silently kept the old one. Architecture. The defaults (virtualNetworkType, publicNetworkAccess, notificationSenderEmail, disableGateway, customProperties) and the computed properties (provisioningState, createdAtUtc, platformVersion, the gateway, regional gateway, portal, management and SCM URLs, the Consumption endpoint rules) moved from the server layer into the provider, so the Go library and serve return the same resource. Snapshots restore through the same path. Discovery sets Attrs.SKU/SKUCapacity/Zones so the Resource Graph row has a top-level sku{name,capacity}. Validation errors are APIM's 400 ValidationError. Lists page with $skip/$top and a nextLink via a new shared azurearm.Paginate. The server's unmodeled-property echo no longer reflects a delegation validationKey on GET (Azure serves it only via listSecrets). --- docs/coverage/README.md | 2 +- docs/coverage/azure/README.md | 2 +- docs/coverage/azure/apimanagement.md | 28 +- docs/coverage/coverage.json | 60 ++ docs/coverage/nongoals/apimanagement.md | 5 + docs/sdk-server.md | 2 +- .../azure/apimanagement/apimanagement.go | 249 +++++- .../azure/apimanagement/apimanagement_test.go | 12 +- providers/azure/apimanagement/children.go | 708 ++++++++++++++++++ providers/azure/apimanagement/deleted.go | 189 +++++ providers/azure/apimanagement/properties.go | 137 ++++ .../azure/apimanagement/review_fixes_test.go | 463 ++++++++++++ providers/azure/apimanagement/snapshot.go | 53 +- providers/azure/apimanagement/validate.go | 115 ++- providers/azure/apimanagement_discovery.go | 9 +- .../apimanagement/apimanagement_sdk_test.go | 29 +- server/azure/apimanagement/children.go | 370 +++++++++ server/azure/apimanagement/handler.go | 229 +++++- .../apimanagement/review_fixes_sdk_test.go | 556 ++++++++++++++ .../apimanagement/terraform_flow_sdk_test.go | 276 +++++++ server/azure/apimanagement/types.go | 207 ++--- .../apimanagement/wire_errors_sdk_test.go | 106 +++ server/azure/azure.go | 8 +- server/azure/echo_properties.go | 10 +- server/wire/azurearm/paging.go | 70 ++ server/wire/azurearm/paging_test.go | 51 ++ 26 files changed, 3728 insertions(+), 218 deletions(-) create mode 100644 docs/coverage/nongoals/apimanagement.md create mode 100644 providers/azure/apimanagement/children.go create mode 100644 providers/azure/apimanagement/deleted.go create mode 100644 providers/azure/apimanagement/properties.go create mode 100644 providers/azure/apimanagement/review_fixes_test.go create mode 100644 server/azure/apimanagement/children.go create mode 100644 server/azure/apimanagement/review_fixes_sdk_test.go create mode 100644 server/azure/apimanagement/terraform_flow_sdk_test.go create mode 100644 server/azure/apimanagement/wire_errors_sdk_test.go create mode 100644 server/wire/azurearm/paging.go create mode 100644 server/wire/azurearm/paging_test.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index 674f7c36a..abd86e91d 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -18,7 +18,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `apigateway` | [APIGateway](./aws/apigateway.md) | - | - | - | 29 | | `apigatewaygcp` | - | - | [APIGateway](./gcp/apigateway.md) | - | 16 | | `apigatewayv2` | [APIGatewayV2](./aws/apigatewayv2.md) | - | - | - | 20 | -| `apimanagement` | - | [APIManagement](./azure/apimanagement.md) | - | - | 10 | +| `apimanagement` | - | [APIManagement](./azure/apimanagement.md) | - | - | 30 | | `appconfiguration` | - | [AppConfiguration](./azure/appconfiguration.md) | - | - | 9 | | `appflow` | [AppFlow](./aws/appflow.md) | - | - | - | 14 | | `appinsights` | - | [Appinsights](./azure/appinsights.md) | - | - | 6 | diff --git a/docs/coverage/azure/README.md b/docs/coverage/azure/README.md index 62735d62d..cbd09dbfe 100644 --- a/docs/coverage/azure/README.md +++ b/docs/coverage/azure/README.md @@ -8,7 +8,7 @@ Services cloudemu emulates for Azure, by native name. Back to the [cross-provide | [ACR](./acr.md) | `containerregistry` | 15 | | [AI](./ai.md) | `azureai` | 92 | | [AKS](./aks.md) | (provider-native) | 18 | -| [APIManagement](./apimanagement.md) | (provider-native) | 10 | +| [APIManagement](./apimanagement.md) | (provider-native) | 30 | | [Aad](./aad.md) | (provider-native) | 2 | | [AppConfiguration](./appconfiguration.md) | (provider-native) | 9 | | [AppGateway](./appgateway.md) | `applicationgateway` | 4 | diff --git a/docs/coverage/azure/apimanagement.md b/docs/coverage/azure/apimanagement.md index 2fb1c9c00..381fa2554 100644 --- a/docs/coverage/azure/apimanagement.md +++ b/docs/coverage/azure/apimanagement.md @@ -3,21 +3,45 @@ provider-native `apimanagement` wire service (Azure-only) · no portable driver · [Azure index](./README.md) -## Operations (10) +## Operations (30) | Operation | Description | | --- | --- | | `ARMID` | | +| `CheckNameAvailability` | | | `CreateOrUpdateService` | | +| `DelegationValidationKey` | | +| `DeleteAPI` | | +| `DeletePolicy` | | +| `DeleteProduct` | | | `DeleteService` | | +| `DeleteServiceIfMatch` | | | `DiscoverServices` | | | `Endpoints` | | +| `GetAPI` | | +| `GetDeletedService` | | +| `GetPolicy` | | +| `GetPortalSetting` | | +| `GetProduct` | | | `GetService` | | +| `GetTenantAccess` | | +| `ListAPIs` | | +| `ListDeletedServices` | | +| `ListProducts` | | | `ListServicesByResourceGroup` | | | `ListServicesBySubscription` | | +| `PurgeDeletedService` | | | `PurgeResourceGroup` | | +| `PutPolicy` | | +| `PutPortalSetting` | | +| `RegionalGatewayURL` | | | `UpdateService` | | +| `UpdateTenantAccess` | | ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- Only the child resources a service's create, refresh and destroy touch are modelled: the sample Echo API and the Starter/Unlimited products (list, get, delete), the service-level policy, the developer portal sign-in/sign-up/delegation settings and tenant access. Creating APIs, operations, products, subscriptions, backends, named values or loggers returns 404 `InvalidResourceType`. +- The gateway data plane (proxying traffic), the developer portal, backup/restore and network-configuration updates are not emulated. +- Policies are stored inline only (`xml`, `rawxml`); the `-link` formats, which make Azure fetch the document, are rejected. +- Terraform `azurerm_api_management`: the provider's create/refresh/destroy request sequence is replayed in tests through the official SDK clients; an empty plan after apply has not been checked with a real terraform binary. +- A purged name is immediately reusable from any subscription; Azure reserves it to the original subscription's tenant for several days. diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index 6c75ba993..5e0420f5b 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -501,32 +501,92 @@ { "name": "ARMID" }, + { + "name": "CheckNameAvailability" + }, { "name": "CreateOrUpdateService" }, + { + "name": "DelegationValidationKey" + }, + { + "name": "DeleteAPI" + }, + { + "name": "DeletePolicy" + }, + { + "name": "DeleteProduct" + }, { "name": "DeleteService" }, + { + "name": "DeleteServiceIfMatch" + }, { "name": "DiscoverServices" }, { "name": "Endpoints" }, + { + "name": "GetAPI" + }, + { + "name": "GetDeletedService" + }, + { + "name": "GetPolicy" + }, + { + "name": "GetPortalSetting" + }, + { + "name": "GetProduct" + }, { "name": "GetService" }, + { + "name": "GetTenantAccess" + }, + { + "name": "ListAPIs" + }, + { + "name": "ListDeletedServices" + }, + { + "name": "ListProducts" + }, { "name": "ListServicesByResourceGroup" }, { "name": "ListServicesBySubscription" }, + { + "name": "PurgeDeletedService" + }, { "name": "PurgeResourceGroup" }, + { + "name": "PutPolicy" + }, + { + "name": "PutPortalSetting" + }, + { + "name": "RegionalGatewayURL" + }, { "name": "UpdateService" + }, + { + "name": "UpdateTenantAccess" } ], "providers": { diff --git a/docs/coverage/nongoals/apimanagement.md b/docs/coverage/nongoals/apimanagement.md new file mode 100644 index 000000000..856241537 --- /dev/null +++ b/docs/coverage/nongoals/apimanagement.md @@ -0,0 +1,5 @@ +- Only the child resources a service's create, refresh and destroy touch are modelled: the sample Echo API and the Starter/Unlimited products (list, get, delete), the service-level policy, the developer portal sign-in/sign-up/delegation settings and tenant access. Creating APIs, operations, products, subscriptions, backends, named values or loggers returns 404 `InvalidResourceType`. +- The gateway data plane (proxying traffic), the developer portal, backup/restore and network-configuration updates are not emulated. +- Policies are stored inline only (`xml`, `rawxml`); the `-link` formats, which make Azure fetch the document, are rejected. +- Terraform `azurerm_api_management`: the provider's create/refresh/destroy request sequence is replayed in tests through the official SDK clients; an empty plan after apply has not been checked with a real terraform binary. +- A purged name is immediately reusable from any subscription; Azure reserves it to the original subscription's tenant for several days. diff --git a/docs/sdk-server.md b/docs/sdk-server.md index b8a8153e7..8e3a9fd85 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -208,7 +208,7 @@ All handlers speak ARM JSON over HTTPS unless noted. | **IAM (armauthorization)** | `Microsoft.Authorization`: RoleDefinitions (CreateOrUpdate, Get, List, Delete) and RoleAssignments (Create, Get, ListForScope, Delete) at any scope (subscription, resource group, resource, management group). Real `armauthorization` SDK clients round-trip end-to-end. Microsoft Graph (users/groups) is not implemented yet. | | **Resource Graph** | `Microsoft.ResourceGraph`: `POST /providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01` with a KQL-shaped query over the cross-service inventory; supports `subscriptions[]` scoping and `$top`/`$skipToken` pagination. Rows carry the fixed columns (`id` [ARM-shaped], `name`, `type`, `location`, `resourceGroup`, `subscriptionId`, `tags`) plus resource-shape columns emitted when present; `sku.name`, `properties`, `managedBy`, `kind`, `zones`; so SKU/tier/size-sensitive consumers (e.g. a discovery + cost engine) can read a VM's size, a managed disk's tier/`diskSizeGB`/owning VM, or a flexible server's compute SKU. `project`/`summarize`/`join` are tolerated but ignored (the full row is always returned). | | **Databricks (ARM control plane)** | `Microsoft.Databricks/workspaces`: CreateOrUpdate, Get, Delete, UpdateTags, List / ListByResourceGroup. Real `armdatabricks` SDK clients round-trip end-to-end. | -| **API Management** | `Microsoft.ApiManagement/service`: CreateOrUpdate, Get, Update (PATCH: properties merged, tags replaced when sent), Delete, ListByResourceGroup, List. SKU (Developer/Basic/Standard/Premium/Consumption/Isolated/BasicV2/StandardV2) + capacity validated (Consumption = 0, others >= 1), `publisherEmail`/`publisherName` required, service name 1-50 chars; `gatewayUrl`/`portalUrl`/`developerPortalUrl`/`managementApiUrl`/`scmUrl` derived from the name, stable `createdAtUtc`/etag/system-assigned identity. LROs complete synchronously. Real `armapimanagement/v3` `ServiceClient` round-trips end-to-end. Child resources (apis, products, policies, ...) and backup/restore are out of scope. | +| **API Management** | `Microsoft.ApiManagement/service`: CreateOrUpdate, Get, Update (PATCH: properties merged, tags replaced when sent), Delete (soft delete, kept 48 h), ListByResourceGroup, List (paged with `nextLink`); `deletedservices` GetByName / Purge / ListBySubscription and recovery via `properties.restore`; `checkNameAvailability` (the name is global, so a second service of that name anywhere is 409). SKU name + capacity validated per tier (Consumption = 0; Developer 1, Basic 2, Standard 4, Premium 12, BasicV2/StandardV2 10), zones Premium-only, location immutable (409 `InvalidResourceLocation`), `publisherEmail`/`publisherName` required, validation errors are 400 `ValidationError`. Defaults and computed properties (endpoint URLs, platformVersion, ...) come from the provider, so the Go library and the server return the same resource; the etag changes on every write and a stale If-Match is 412. Child resources around create/refresh/destroy: the sample Echo API and Starter/Unlimited products (list/get/delete), `policies/policy`, `portalsettings` signin/signup/delegation, `tenant` access. LROs complete synchronously. Real `armapimanagement/v3` clients round-trip end-to-end. Creating APIs/products/operations, backends, named values and backup/restore are out of scope. | | **Databricks (workspace data plane)** *(`databricks-sdk-go`, `/api/2.x`)* | Point the real `WorkspaceClient` at `Config.Host`. Clusters (create/edit/start/restart/resize/pin/unpin/delete + list-node-types / spark-versions / zones), instance pools, jobs + runs (submit / run-now / get / list / cancel / cancel-all / repair / output / delete), cluster policies, libraries (install / uninstall / status), and object permissions. Self-contained families: secrets (scopes / secrets / ACLs), tokens, git credentials, repos, DBFS (incl. block upload), workspace notebooks/directories, SQL warehouses, pipelines, serving endpoints, SCIM identity (users / groups / service principals), and Unity Catalog (catalogs / schemas / tables + metastores / external locations / storage credentials / volumes). Also serves `GET /.well-known/databricks-config` so the SDK's host-metadata resolution succeeds (workspace-host stub) instead of logging a warning. | ### GCP (`server/gcp/`) diff --git a/providers/azure/apimanagement/apimanagement.go b/providers/azure/apimanagement/apimanagement.go index a85130eeb..450151e44 100644 --- a/providers/azure/apimanagement/apimanagement.go +++ b/providers/azure/apimanagement/apimanagement.go @@ -1,25 +1,38 @@ // Package apimanagement provides an in-memory mock of Azure API Management -// (Microsoft.ApiManagement/service), the ARM control plane only. It manages the +// (Microsoft.ApiManagement), the ARM control plane only. It manages the // service lifecycle (create-or-update, get, patch, delete, list-by-group, -// list-by-subscription), the service SKU (name + capacity), availability zones -// and the system/user-assigned managed identity. +// list-by-subscription), the service SKU (name + capacity, bounded per tier), +// availability zones (Premium only), the system/user-assigned managed identity, +// the global name reservation (checkNameAvailability), and soft delete: a +// deleted service is kept for 48 hours under +// locations/{location}/deletedservices/{name}, where it can be read, purged, +// or recovered by a PUT with properties.restore = true. // -// The API Management data plane (the gateway that proxies traffic, the -// developer portal) and the service's child resources (apis, products, -// subscriptions, policies, named values, backends, loggers, ...) are out of -// scope, as are backup/restore, network-configuration updates and the -// soft-deleted services (deletedservices) surface. +// A service also carries the child resources an infrastructure-as-code tool +// touches around create, refresh and destroy: the sample Echo API and the +// Starter/Unlimited products every non-Consumption service is born with (list, +// get, delete), the service-level policy (get, put, delete), the developer +// portal sign-in/sign-up/delegation settings (get, put) and the tenant access +// information (get, patch, listSecrets). Creating APIs, products, operations, +// subscriptions, backends, named values and loggers is out of scope, as are the +// gateway data plane, backup/restore and network-configuration updates. // -// Every service-minted field stays stable for the lifetime of the resource so -// infrastructure-as-code tools (Terraform's azurerm_api_management) see no drift -// on re-plan: id/name, provisioningState ("Succeeded"), createdAtUtc, etag and -// the system-assigned identity's principalId/tenantId are minted once at create -// and byte-stable across every read and patch. The endpoint host names -// (gateway, portal, developer portal, management, scm) derive from the service -// name, exactly as Azure derives them. +// The provider owns the whole resource: the properties block it stores and +// returns already holds Azure's defaults for unset writable fields and the +// computed read-only fields (provisioningState, createdAtUtc, platformVersion, +// the endpoint URLs derived from the name and location), so the Go library and +// the HTTP server return the same resource. id/name, createdAtUtc and the +// system-assigned identity's principalId/tenantId are minted once and stay +// stable; the etag changes on every write and a non-wildcard If-Match that no +// longer matches is rejected with FailedPrecondition. // -// The writable properties block (publisherEmail, publisherName and every other -// caller-set property) is stored as raw JSON and round-trips verbatim. +// Terraform: the requests terraform-provider-azurerm v4 makes for +// azurerm_api_management create, refresh and destroy (with its default +// recover_soft_deleted and purge_soft_delete_on_destroy features) are served, +// and two refreshes read back identical state. That was verified by replaying +// the provider's request sequence through the official SDK clients +// (server/azure/apimanagement TestSDKTerraformCreateReadDestroy), not by running +// a terraform binary, so an empty plan after apply is not yet proven. package apimanagement import ( @@ -53,6 +66,9 @@ const ( // hostSuffix is the DNS suffix every API Management endpoint lives under. hostSuffix = ".azure-api.net" + + // wildcardETag is the If-Match value that matches any current version. + wildcardETag = "*" ) // ManagedIdentity is a service's top-level managed identity. For a @@ -67,9 +83,9 @@ type ManagedIdentity struct { } // Service is a stored Microsoft.ApiManagement/service resource. Subscription, -// ResourceGroup and Name preserve the caller's casing; the computed fields are -// minted at create and never regenerated on a read. Properties holds the -// writable properties block and round-trips verbatim. +// ResourceGroup and Name preserve the caller's casing. Properties is the full +// properties block as Azure returns it: the caller's writable properties, +// Azure's defaults for the unset ones and the computed read-only fields. type Service struct { Subscription string `json:"subscription"` ResourceGroup string `json:"resourceGroup"` @@ -84,7 +100,7 @@ type Service struct { Properties json.RawMessage `json:"properties,omitempty"` - // Computed, stable fields. + // Computed fields. Etag changes on every write; the rest are stable. ProvisioningState string `json:"provisioningState"` Etag string `json:"etag"` CreatedAt time.Time `json:"createdAt"` @@ -123,7 +139,8 @@ func (s *Service) endpoint(infix string) string { // ServiceInput carries the mutable fields of a service create/update request. // A nil pointer/map/slice means "not supplied": on a PATCH the stored value is -// preserved, so the request overlays only what it names. +// preserved, so the request overlays only what it names. IfMatch, when set to +// anything but "*", makes the write conditional on the stored etag. type ServiceInput struct { Tags map[string]string Zones []string @@ -131,6 +148,7 @@ type ServiceInput struct { SkuCapacity *int32 Identity *ManagedIdentity Properties json.RawMessage + IfMatch string } // Mock is the in-memory backend for API Management services. @@ -138,6 +156,8 @@ type Mock struct { mu sync.RWMutex clock config.Clock services *memstore.Store[*Service] + children *memstore.Store[*Children] + deleted *memstore.Store[*DeletedService] } // New creates an empty API Management mock. It falls back to the real clock @@ -148,7 +168,12 @@ func New(opts *config.Options) *Mock { clock = opts.Clock } - return &Mock{clock: clock, services: memstore.New[*Service]()} + return &Mock{ + clock: clock, + services: memstore.New[*Service](), + children: memstore.New[*Children](), + deleted: memstore.New[*DeletedService](), + } } // serviceKey is the case-insensitive store key for a service. @@ -158,13 +183,20 @@ func serviceKey(sub, rg, name string) string { // CreateOrUpdateService creates a new service or replaces an existing one (ARM // PUT semantics: tags, zones, identity and the properties block are replaced -// wholesale). The computed fields (provisioningState, etag, createdAtUtc) are -// minted once at create and preserved across updates; location is immutable in -// real Azure and is preserved on update. It returns the stored service and -// whether it was newly created. +// wholesale). The service name is a global DNS label, so a name already held by +// another live service (in any subscription or group) is ErrNameNotAvailable +// and one held by a soft-deleted service is ErrSoftDeleted, unless the request +// sets properties.restore to recover it. Location is immutable: a replace that +// names another location is ErrLocationMismatch. createdAtUtc is minted once; +// the etag changes on every write. It returns the stored service and whether it +// was newly created. func (m *Mock) CreateOrUpdateService( _ context.Context, sub, rg, name, location string, in *ServiceInput, ) (Service, bool, error) { + if restoreRequested(in.Properties) { + return m.restoreService(sub, rg, name, location, in.IfMatch) + } + if err := validateCreate(sub, rg, name, location, in); err != nil { return Service{}, false, err } @@ -175,11 +207,25 @@ func (m *Mock) CreateOrUpdateService( k := serviceKey(sub, rg, name) existing, existed := m.services.Get(k) + if err := checkIfMatch(existed, existing, in.IfMatch, name); err != nil { + return Service{}, false, err + } var s Service + if existed { + if normalizeLocation(existing.Location) != normalizeLocation(location) { + return Service{}, false, coded(ErrLocationMismatch, cerrors.Newf(cerrors.AlreadyExists, + "the resource %q already exists in location %q; a resource cannot be moved to %q", + name, existing.Location, location)) + } + s = *existing } else { + if err := m.nameTakenLocked(name); err != nil { + return Service{}, false, err + } + s = m.newService(sub, rg, name, location) } @@ -189,17 +235,83 @@ func (m *Mock) CreateOrUpdateService( s.SkuCapacity = *in.SkuCapacity s.Identity = resolveIdentity(in.Identity, sub, rg, name) s.Properties = append(json.RawMessage(nil), in.Properties...) + m.commitLocked(k, &s) - m.services.Set(k, &s) + if !existed { + m.children.Set(k, seedChildren(&s)) + } return cloneService(&s), !existed, nil } +// commitLocked re-materializes the full properties block, rotates the etag and +// stores the service. The caller holds m.mu. +func (m *Mock) commitLocked(k string, s *Service) { + s.materializeProperties() + s.Etag = nextETag(k, s) + m.services.Set(k, s) +} + +// nextETag derives a new etag from the resource key, its creation time and the +// previous etag, so every write yields a different value (If-Match optimistic +// concurrency works) while the sequence stays deterministic and survives a +// snapshot round trip. Seeding with the creation time keeps a re-created +// service from reusing the etags of an earlier one with the same name. +func nextETag(k string, s *Service) string { + return idgen.SyntheticGUID("apimanagement/etag/" + k + "/" + s.CreatedAt.String() + "/" + s.Etag) +} + +// checkIfMatch enforces a conditional write: an If-Match other than "*" must +// equal the stored etag, and a conditional write on a missing resource fails. +func checkIfMatch(existed bool, s *Service, ifMatch, name string) error { + if ifMatch == "" || ifMatch == wildcardETag { + return nil + } + + if !existed || !etagMatches(s.Etag, ifMatch) { + return cerrors.Newf(cerrors.FailedPrecondition, + "the If-Match etag %s does not match the current state of API Management service %q", ifMatch, name) + } + + return nil +} + +// etagMatches compares two etags ignoring the weak-validator prefix and quotes, +// which clients add or strip inconsistently. +func etagMatches(stored, given string) bool { + norm := func(e string) string { + return strings.Trim(strings.TrimPrefix(strings.TrimSpace(e), "W/"), `"`) + } + + return norm(stored) == norm(given) +} + +// nameTakenLocked reports ErrNameNotAvailable when a live service anywhere +// already holds name, or ErrSoftDeleted when a soft-deleted one does. Service +// names are global DNS labels (.azure-api.net). The caller holds m.mu. +func (m *Mock) nameTakenLocked(name string) error { + for _, s := range m.services.All() { + if strings.EqualFold(s.Name, name) { + return coded(ErrNameNotAvailable, cerrors.Newf(cerrors.AlreadyExists, + "API Management service name %q is already in use: %s%s is taken", name, strings.ToLower(name), hostSuffix)) + } + } + + if d := m.deletedByNameLocked(name); d != nil { + return coded(ErrSoftDeleted, cerrors.Newf(cerrors.AlreadyExists, + "API Management service %q is soft-deleted in location %q; recover it (properties.restore = true) "+ + "or purge it before reusing the name", name, d.Service.Location)) + } + + return nil +} + // UpdateService applies an ARM PATCH: tags and zones are replaced wholesale // when supplied, sku/identity are re-resolved only when supplied, and the // properties block is merged key-by-key onto the stored block. The merged // result is re-validated, so a PATCH cannot blank the publisher fields or leave -// an invalid SKU/capacity pair. A PATCH on a missing service is a NotFound. +// an invalid SKU/capacity/zones combination. A PATCH on a missing service is a +// NotFound. func (m *Mock) UpdateService(_ context.Context, sub, rg, name string, in *ServiceInput) (Service, error) { m.mu.Lock() defer m.mu.Unlock() @@ -211,6 +323,10 @@ func (m *Mock) UpdateService(_ context.Context, sub, rg, name string, in *Servic return Service{}, notFound(name) } + if err := checkIfMatch(true, existing, in.IfMatch, name); err != nil { + return Service{}, err + } + s := *existing applyPatch(&s, in, sub, rg, name) @@ -218,11 +334,15 @@ func (m *Mock) UpdateService(_ context.Context, sub, rg, name string, in *Servic return Service{}, err } + if err := validateZones(s.SkuName, s.Zones); err != nil { + return Service{}, err + } + if err := validatePublisher(s.Properties); err != nil { return Service{}, err } - m.services.Set(k, &s) + m.commitLocked(k, &s) return cloneService(&s), nil } @@ -256,8 +376,7 @@ func applyPatch(s *Service, in *ServiceInput, sub, rg, name string) { } // newService seeds a fresh service with its immutable identity and its -// computed, stable fields. The etag derives deterministically from the resource -// id so it is stable yet distinct per service. +// computed, stable fields. func (m *Mock) newService(sub, rg, name, location string) Service { return Service{ Subscription: sub, @@ -265,7 +384,6 @@ func (m *Mock) newService(sub, rg, name, location string) Service { Name: name, Location: location, ProvisioningState: stateSucceeded, - Etag: idgen.SyntheticGUID("apimanagement/etag/" + serviceKey(sub, rg, name)), CreatedAt: m.clock.Now().UTC().Truncate(time.Second), } } @@ -283,12 +401,39 @@ func (m *Mock) GetService(_ context.Context, sub, rg, name string) (Service, err return cloneService(s), nil } -// DeleteService removes the service, reporting whether it existed. -func (m *Mock) DeleteService(_ context.Context, sub, rg, name string) (bool, error) { +// DeleteService soft-deletes the service, reporting whether it existed. It is +// DeleteServiceIfMatch with no precondition. +func (m *Mock) DeleteService(ctx context.Context, sub, rg, name string) (bool, error) { + return m.DeleteServiceIfMatch(ctx, sub, rg, name, "") +} + +// DeleteServiceIfMatch soft-deletes the service, as Azure does for every delete +// since API version 2020-06-01-preview: the service leaves the live store and is +// kept, with its child resources, under +// locations/{location}/deletedservices/{name} until it is purged, recovered or +// its 48-hour retention lapses. It reports whether the service existed; a +// non-wildcard ifMatch that does not equal the stored etag is a +// FailedPrecondition. +func (m *Mock) DeleteServiceIfMatch(_ context.Context, sub, rg, name, ifMatch string) (bool, error) { m.mu.Lock() defer m.mu.Unlock() - return m.services.Delete(serviceKey(sub, rg, name)), nil + k := serviceKey(sub, rg, name) + + s, ok := m.services.Get(k) + if ifMatch != "" && ifMatch != wildcardETag { + if err := checkIfMatch(ok, s, ifMatch, name); err != nil { + return false, err + } + } + + if !ok { + return false, nil + } + + m.softDeleteLocked(k, s) + + return true, nil } // ListServicesByResourceGroup returns every service in the group, sorted by @@ -312,21 +457,47 @@ func (m *Mock) DiscoverServices(_ context.Context) ([]Service, error) { return m.filterServices(func(*Service) bool { return true }), nil } -// PurgeResourceGroup deletes every service under sub/rg, so a resource-group -// delete cascades into its API Management services. +// PurgeResourceGroup soft-deletes every service under sub/rg, so a +// resource-group delete cascades into its API Management services exactly as a +// service delete does. func (m *Mock) PurgeResourceGroup(_ context.Context, sub, rg string) error { m.mu.Lock() defer m.mu.Unlock() for k, s := range m.services.All() { if strings.EqualFold(s.Subscription, sub) && strings.EqualFold(s.ResourceGroup, rg) { - m.services.Delete(k) + m.softDeleteLocked(k, s) } } return nil } +// NameAvailability is the checkNameAvailability verdict. +type NameAvailability struct { + Available bool + Reason string // "Valid", "Invalid" or "AlreadyExists" + Message string +} + +// CheckNameAvailability reports whether name can be used for a new service. The +// name is a global DNS label, so any live or soft-deleted service holding it, +// in any subscription, makes it unavailable. +func (m *Mock) CheckNameAvailability(_ context.Context, name string) NameAvailability { + if !validName(name) { + return NameAvailability{Reason: "Invalid", Message: cerrors.Message(validateName(name))} + } + + m.mu.Lock() + defer m.mu.Unlock() + + if err := m.nameTakenLocked(name); err != nil { + return NameAvailability{Reason: "AlreadyExists", Message: cerrors.Message(err)} + } + + return NameAvailability{Available: true, Reason: "Valid"} +} + // filterServices returns the services matching pred, sorted by name. func (m *Mock) filterServices(pred func(*Service) bool) []Service { m.mu.RLock() diff --git a/providers/azure/apimanagement/apimanagement_test.go b/providers/azure/apimanagement/apimanagement_test.go index 1df498b96..c8d28e501 100644 --- a/providers/azure/apimanagement/apimanagement_test.go +++ b/providers/azure/apimanagement/apimanagement_test.go @@ -80,7 +80,9 @@ func TestCreateComputedFields(t *testing.T) { } } -func TestReplaceKeepsComputedFieldsAndLocation(t *testing.T) { +// TestReplaceKeepsStableFieldsAndRotatesEtag: a replace keeps createdAt, location +// and the principal, but mints a new etag (every write changes it). +func TestReplaceKeepsStableFieldsAndRotatesEtag(t *testing.T) { m, clk := newMock() first := create(t, m, "apim1") @@ -91,15 +93,19 @@ func TestReplaceKeepsComputedFieldsAndLocation(t *testing.T) { in.SkuName = sptr("premium") in.SkuCapacity = i32(2) - got, created, err := m.CreateOrUpdateService(context.Background(), sub, rg, "apim1", "West Europe", in) + got, created, err := m.CreateOrUpdateService(context.Background(), sub, rg, "apim1", "eastus", in) if err != nil || created { t.Fatalf("replace: err=%v created=%v", err, created) } - if !got.CreatedAt.Equal(first.CreatedAt) || got.Etag != first.Etag || got.Location != "East US" { + if !got.CreatedAt.Equal(first.CreatedAt) || got.Location != "East US" { t.Errorf("replace changed stable fields: %+v", got) } + if got.Etag == first.Etag || got.Etag == "" { + t.Errorf("replace must rotate the etag, got %q (was %q)", got.Etag, first.Etag) + } + if got.SkuName != "Premium" || got.SkuCapacity != 2 { t.Errorf("sku = %s/%d, want Premium/2", got.SkuName, got.SkuCapacity) } diff --git a/providers/azure/apimanagement/children.go b/providers/azure/apimanagement/children.go new file mode 100644 index 000000000..9c273b39c --- /dev/null +++ b/providers/azure/apimanagement/children.go @@ -0,0 +1,708 @@ +package apimanagement + +import ( + "context" + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "io" + "maps" + "sort" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" +) + +// Portal setting names (…/portalsettings/{name}). +const ( + PortalSignIn = "signin" + PortalSignUp = "signup" + PortalDelegation = "delegation" +) + +// Tenant access names (…/tenant/{name}). +const ( + TenantAccessName = "access" + TenantGitAccessName = "gitAccess" +) + +// Policy formats the service-level policy accepts and stores inline. +const ( + PolicyFormatXML = "xml" + PolicyFormatRawXML = "rawxml" +) + +// delegationValidationKey is the secret key of the delegation settings; Azure +// returns it only from listSecrets, never from a GET. +const delegationValidationKey = "validationKey" + +// ChildResource is a stored child resource of a service (an API, a product, +// the service policy or a portal setting): its name, its properties block as +// Azure returns it, and its etag, which changes on every write. +type ChildResource struct { + Name string `json:"name"` + Properties json.RawMessage `json:"properties"` + Etag string `json:"etag"` +} + +// TenantAccess is a tenant access information entity (access or gitAccess): +// whether direct management / git access is enabled, and its principal and +// keys, which only listSecrets reveals. +type TenantAccess struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + PrincipalID string `json:"principalId"` + PrimaryKey string `json:"primaryKey"` + SecondaryKey string `json:"secondaryKey"` + Etag string `json:"etag"` +} + +// Children are the child resources held for one service. +type Children struct { + APIs map[string]*ChildResource `json:"apis,omitempty"` + Products map[string]*ChildResource `json:"products,omitempty"` + Policy *ChildResource `json:"policy,omitempty"` + Portal map[string]*ChildResource `json:"portalSettings,omitempty"` + Tenant map[string]*TenantAccess `json:"tenantAccess,omitempty"` +} + +// ErrTierNotSupported reports a child resource that does not exist in the +// service's tier (the developer portal settings and tenant access on +// Consumption and v2). +var ErrTierNotSupported = errors.New("not supported in this API Management tier") + +// hasDeveloperPortal reports whether the tier has the classic developer portal +// and tenant access surface: every tier but Consumption and the v2 tiers. +func hasDeveloperPortal(sku string) bool { + return sku != skuConsumption && !strings.HasSuffix(strings.ToUpper(sku), "V2") +} + +// productPublished is the state of a product visible on the developer portal. +const productPublished = "published" + +// enabledFlag is the {"enabled": bool} block the portal settings share. +type enabledFlag struct { + Enabled bool `json:"enabled"` +} + +// apiSeed is the sample Echo API's properties block. +type apiSeed struct { + DisplayName string `json:"displayName"` + APIRevision string `json:"apiRevision"` + Description string `json:"description"` + SubscriptionRequired bool `json:"subscriptionRequired"` + ServiceURL string `json:"serviceUrl"` + Path string `json:"path"` + Protocols []string `json:"protocols"` + IsCurrent bool `json:"isCurrent"` + KeyParameterNames map[string]string `json:"subscriptionKeyParameterNames"` +} + +// productSeed is a sample product's properties block. +type productSeed struct { + DisplayName string `json:"displayName"` + Description string `json:"description"` + SubscriptionRequired bool `json:"subscriptionRequired"` + ApprovalRequired bool `json:"approvalRequired"` + SubscriptionsLimit int `json:"subscriptionsLimit"` + State string `json:"state"` +} + +// signUpSeed / delegationSeed are the default sign-up and delegation settings. +type signUpSeed struct { + Enabled bool `json:"enabled"` + TermsOfService struct { + Enabled bool `json:"enabled"` + ConsentRequired bool `json:"consentRequired"` + Text string `json:"text"` + } `json:"termsOfService"` +} + +type delegationSeed struct { + URL string `json:"url"` + Subscriptions enabledFlag `json:"subscriptions"` + UserRegistration enabledFlag `json:"userRegistration"` +} + +// seedChildren is what Azure provisions with a new service: every tier but +// Consumption gets the sample Echo API and the Starter and Unlimited products, +// and the tiers with a developer portal get its default settings and the tenant +// access entities. +func seedChildren(s *Service) *Children { + c := defaultSettings(s) + + if s.SkuName == skuConsumption { + return c + } + + c.APIs = map[string]*ChildResource{ + "echo-api": newChild(s, "apis", "echo-api", apiSeed{ + DisplayName: "Echo API", + APIRevision: "1", + SubscriptionRequired: true, + ServiceURL: "http://echoapi.cloudapp.net/api", + Path: "echo", + Protocols: []string{"https"}, + IsCurrent: true, + KeyParameterNames: map[string]string{"header": "Ocp-Apim-Subscription-Key", "query": "subscription-key"}, + }), + } + + c.Products = map[string]*ChildResource{ + "starter": newChild(s, "products", "starter", productSeed{ + DisplayName: "Starter", + Description: "Subscribers will be able to run 5 calls/minute up to a maximum of 100 calls/week.", + SubscriptionRequired: true, + SubscriptionsLimit: 1, + State: productPublished, + }), + "unlimited": newChild(s, "products", "unlimited", productSeed{ + DisplayName: "Unlimited", + Description: "Subscribers have completely unlimited access to the API. Administrator approval is required.", + SubscriptionRequired: true, + ApprovalRequired: true, + SubscriptionsLimit: 1, + State: productPublished, + }), + } + + return c +} + +// defaultSettings is the developer portal settings and tenant access a service +// starts with (none on the tiers without a developer portal). +func defaultSettings(s *Service) *Children { + c := &Children{} + if !hasDeveloperPortal(s.SkuName) { + return c + } + + c.Portal = map[string]*ChildResource{ + PortalSignIn: newChild(s, "portalsettings", PortalSignIn, enabledFlag{}), + PortalSignUp: newChild(s, "portalsettings", PortalSignUp, signUpSeed{Enabled: true}), + PortalDelegation: newChild(s, "portalsettings", PortalDelegation, delegationSeed{}), + } + + principals := map[string]string{TenantAccessName: "integration", TenantGitAccessName: "git"} + c.Tenant = map[string]*TenantAccess{} + + for n, principal := range principals { + seed := "apimanagement/tenant/" + serviceKey(s.Subscription, s.ResourceGroup, s.Name) + "/" + n + c.Tenant[strings.ToLower(n)] = &TenantAccess{ + Name: n, + PrincipalID: principal, + PrimaryKey: secretKey(seed + "/primary"), + SecondaryKey: secretKey(seed + "/secondary"), + Etag: idgen.SyntheticGUID(seed + "/etag"), + } + } + + return c +} + +// secretKey renders a deterministic base64 access key. +func secretKey(seed string) string { + return base64.StdEncoding.EncodeToString([]byte(idgen.SyntheticGUID(seed) + idgen.SyntheticGUID(seed+"/2"))) +} + +// newChild builds a seeded child resource with its first etag. +func newChild(s *Service, kind, name string, props any) *ChildResource { + raw, err := json.Marshal(props) + if err != nil { + raw = json.RawMessage(`{}`) + } + + return &ChildResource{ + Name: name, + Properties: raw, + Etag: idgen.SyntheticGUID("apimanagement/" + serviceKey(s.Subscription, s.ResourceGroup, s.Name) + "/" + kind + "/" + name), + } +} + +// rotate gives a child resource a new etag after a write. +func (c *ChildResource) rotate() { + c.Etag = idgen.SyntheticGUID("apimanagement/child/" + c.Name + "/" + c.Etag + "/" + string(c.Properties)) +} + +// cloneChild deep-copies a child resource. +func cloneChild(c *ChildResource) ChildResource { + out := *c + out.Properties = append(json.RawMessage(nil), c.Properties...) + + return out +} + +// childrenLocked returns the live service at sub/rg/name and its children, +// creating the default settings for a service restored from a snapshot taken +// before child resources existed. The caller holds m.mu for writing. +func (m *Mock) childrenLocked(sub, rg, name string) (*Service, *Children, error) { + k := serviceKey(sub, rg, name) + + s, ok := m.services.Get(k) + if !ok { + return nil, nil, notFound(name) + } + + c, ok := m.children.Get(k) + if !ok { + c = defaultSettings(s) + m.children.Set(k, c) + } + + return s, c, nil +} + +// childNotFound is the NotFound error for a missing child resource. +func childNotFound(kind, name string) error { + return cerrors.Newf(cerrors.NotFound, "%s %q not found", kind, name) +} + +// childIfMatch enforces a conditional child write or delete. +func childIfMatch(etag, ifMatch, kind, name string) error { + if ifMatch == "" || ifMatch == wildcardETag || etagMatches(etag, ifMatch) { + return nil + } + + return cerrors.Newf(cerrors.FailedPrecondition, + "the If-Match etag %s does not match the current state of %s %q", ifMatch, kind, name) +} + +// collection selects one of the named child collections. +func collection(c *Children, kind string) map[string]*ChildResource { + if kind == "apis" { + return c.APIs + } + + return c.Products +} + +// listChildren returns a child collection sorted by name. +func (m *Mock) listChildren(sub, rg, svc, kind string) ([]ChildResource, error) { + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return nil, err + } + + out := make([]ChildResource, 0, len(collection(c, kind))) + for _, r := range collection(c, kind) { + out = append(out, cloneChild(r)) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + + return out, nil +} + +// getChild returns one member of a child collection. +func (m *Mock) getChild(sub, rg, svc, kind, name string) (ChildResource, error) { + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return ChildResource{}, err + } + + r, ok := collection(c, kind)[strings.ToLower(name)] + if !ok { + return ChildResource{}, childNotFound(kind, name) + } + + return cloneChild(r), nil +} + +// deleteChild removes one member of a child collection, reporting whether it +// existed. +func (m *Mock) deleteChild(sub, rg, svc, kind, name, ifMatch string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return false, err + } + + col := collection(c, kind) + + r, ok := col[strings.ToLower(name)] + if !ok { + return false, nil + } + + if err := childIfMatch(r.Etag, ifMatch, kind, name); err != nil { + return false, err + } + + delete(col, strings.ToLower(name)) + + return true, nil +} + +// ListAPIs returns the service's APIs, sorted by id. +func (m *Mock) ListAPIs(_ context.Context, sub, rg, svc string) ([]ChildResource, error) { + return m.listChildren(sub, rg, svc, "apis") +} + +// GetAPI returns one API of the service. +func (m *Mock) GetAPI(_ context.Context, sub, rg, svc, apiID string) (ChildResource, error) { + return m.getChild(sub, rg, svc, "apis", apiID) +} + +// DeleteAPI removes an API, reporting whether it existed. +func (m *Mock) DeleteAPI(_ context.Context, sub, rg, svc, apiID, ifMatch string) (bool, error) { + return m.deleteChild(sub, rg, svc, "apis", apiID, ifMatch) +} + +// ListProducts returns the service's products, sorted by id. +func (m *Mock) ListProducts(_ context.Context, sub, rg, svc string) ([]ChildResource, error) { + return m.listChildren(sub, rg, svc, "products") +} + +// GetProduct returns one product of the service. +func (m *Mock) GetProduct(_ context.Context, sub, rg, svc, productID string) (ChildResource, error) { + return m.getChild(sub, rg, svc, "products", productID) +} + +// DeleteProduct removes a product, reporting whether it existed. +func (m *Mock) DeleteProduct(_ context.Context, sub, rg, svc, productID, ifMatch string) (bool, error) { + return m.deleteChild(sub, rg, svc, "products", productID, ifMatch) +} + +// GetPolicy returns the service-level (global) policy, or NotFound when none is +// set. +func (m *Mock) GetPolicy(_ context.Context, sub, rg, svc string) (ChildResource, error) { + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return ChildResource{}, err + } + + if c.Policy == nil { + return ChildResource{}, childNotFound("policy", "policy") + } + + return cloneChild(c.Policy), nil +} + +// PutPolicy sets the service-level policy from an inline XML document (format +// xml or rawxml; the -link formats, which make Azure fetch the document, are +// not supported). The document must be well-formed XML. It reports whether the +// policy was newly created. +func (m *Mock) PutPolicy(_ context.Context, sub, rg, svc, value, format, ifMatch string) (ChildResource, bool, error) { + format, err := validatePolicy(value, format) + if err != nil { + return ChildResource{}, false, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return ChildResource{}, false, err + } + + created := c.Policy == nil + + current := "" + if !created { + current = c.Policy.Etag + } + + // A conditional write on a policy that does not exist yet fails: no etag + // can match it. + if mErr := childIfMatch(current, ifMatch, "policy", "policy"); mErr != nil { + return ChildResource{}, false, mErr + } + + if created { + c.Policy = &ChildResource{Name: "policy"} + } + + raw, err := json.Marshal(map[string]string{"value": value, "format": format}) + if err != nil { + return ChildResource{}, false, cerrors.Newf(cerrors.Internal, "encode policy: %v", err) + } + + c.Policy.Properties = raw + c.Policy.rotate() + + return cloneChild(c.Policy), created, nil +} + +// validatePolicy checks the format (defaulting to xml) and the document, and +// returns the format to store. +func validatePolicy(value, format string) (string, error) { + if format == "" { + format = PolicyFormatXML + } + + if format != PolicyFormatXML && format != PolicyFormatRawXML { + return "", cerrors.Newf(cerrors.InvalidArgument, + "policy format %q is not supported: use %q or %q", format, PolicyFormatXML, PolicyFormatRawXML) + } + + return format, wellFormedXML(value) +} + +// wellFormedXML rejects a policy document that is empty or not well-formed. +func wellFormedXML(doc string) error { + if strings.TrimSpace(doc) == "" { + return cerrors.New(cerrors.InvalidArgument, "policy value is required") + } + + dec := xml.NewDecoder(strings.NewReader(doc)) + + for { + _, err := dec.Token() + if errors.Is(err, io.EOF) { + return nil + } + + if err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "policy value is not well-formed XML: %v", err) + } + } +} + +// DeletePolicy removes the service-level policy, reporting whether one was set. +func (m *Mock) DeletePolicy(_ context.Context, sub, rg, svc, ifMatch string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + _, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return false, err + } + + if c.Policy == nil { + return false, nil + } + + if err := childIfMatch(c.Policy.Etag, ifMatch, "policy", "policy"); err != nil { + return false, err + } + + c.Policy = nil + + return true, nil +} + +// portalLocked returns the named portal setting of a service whose tier has a +// developer portal. The caller holds m.mu. +func (m *Mock) portalLocked(sub, rg, svc, name string) (*ChildResource, error) { + s, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return nil, err + } + + if !hasDeveloperPortal(s.SkuName) { + return nil, coded(ErrTierNotSupported, cerrors.Newf(cerrors.InvalidArgument, + "portal settings are not supported in the %s tier", s.SkuName)) + } + + r, ok := c.Portal[strings.ToLower(name)] + if !ok { + return nil, childNotFound("portal setting", name) + } + + return r, nil +} + +// GetPortalSetting returns a developer portal setting (signin, signup or +// delegation). The delegation validation key is a secret and is left out; read +// it with DelegationValidationKey. +func (m *Mock) GetPortalSetting(_ context.Context, sub, rg, svc, name string) (ChildResource, error) { + m.mu.Lock() + defer m.mu.Unlock() + + r, err := m.portalLocked(sub, rg, svc, name) + if err != nil { + return ChildResource{}, err + } + + out := cloneChild(r) + out.Properties = withoutKey(out.Properties, delegationValidationKey) + + return out, nil +} + +// PutPortalSetting replaces a developer portal setting's properties block. +func (m *Mock) PutPortalSetting( + _ context.Context, sub, rg, svc, name string, props json.RawMessage, ifMatch string, +) (ChildResource, error) { + var obj map[string]json.RawMessage + if err := json.Unmarshal(props, &obj); err != nil || obj == nil { + return ChildResource{}, cerrors.New(cerrors.InvalidArgument, "properties must be a JSON object") + } + + m.mu.Lock() + defer m.mu.Unlock() + + r, err := m.portalLocked(sub, rg, svc, name) + if err != nil { + return ChildResource{}, err + } + + if err := childIfMatch(r.Etag, ifMatch, "portal setting", name); err != nil { + return ChildResource{}, err + } + + r.Properties = append(json.RawMessage(nil), props...) + r.rotate() + + out := cloneChild(r) + out.Properties = withoutKey(out.Properties, delegationValidationKey) + + return out, nil +} + +// DelegationValidationKey returns the delegation settings' validation key (the +// delegation listSecrets action), "" when none is set. +func (m *Mock) DelegationValidationKey(_ context.Context, sub, rg, svc string) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + r, err := m.portalLocked(sub, rg, svc, PortalDelegation) + if err != nil { + return "", err + } + + var p struct { + ValidationKey string `json:"validationKey"` + } + + _ = json.Unmarshal(r.Properties, &p) + + return p.ValidationKey, nil +} + +// withoutKey drops one top-level key from a JSON object. +func withoutKey(raw json.RawMessage, key string) json.RawMessage { + obj := map[string]json.RawMessage{} + if err := json.Unmarshal(raw, &obj); err != nil { + return raw + } + + if _, ok := obj[key]; !ok { + return raw + } + + delete(obj, key) + + out, err := json.Marshal(obj) + if err != nil { + return raw + } + + return out +} + +// tenantLocked returns a tenant access entity of a service whose tier supports +// it. The caller holds m.mu. +func (m *Mock) tenantLocked(sub, rg, svc, name string) (*TenantAccess, error) { + s, c, err := m.childrenLocked(sub, rg, svc) + if err != nil { + return nil, err + } + + if !hasDeveloperPortal(s.SkuName) { + return nil, coded(ErrTierNotSupported, cerrors.Newf(cerrors.InvalidArgument, + "tenant access is not supported in the %s tier", s.SkuName)) + } + + t, ok := c.Tenant[strings.ToLower(name)] + if !ok { + return nil, childNotFound("tenant access", name) + } + + return t, nil +} + +// GetTenantAccess returns a tenant access entity (access or gitAccess), +// including its keys; the HTTP layer reveals them only from listSecrets. +func (m *Mock) GetTenantAccess(_ context.Context, sub, rg, svc, name string) (TenantAccess, error) { + m.mu.Lock() + defer m.mu.Unlock() + + t, err := m.tenantLocked(sub, rg, svc, name) + if err != nil { + return TenantAccess{}, err + } + + return *t, nil +} + +// UpdateTenantAccess enables or disables a tenant access entity. +func (m *Mock) UpdateTenantAccess( + _ context.Context, sub, rg, svc, name string, enabled *bool, ifMatch string, +) (TenantAccess, error) { + m.mu.Lock() + defer m.mu.Unlock() + + t, err := m.tenantLocked(sub, rg, svc, name) + if err != nil { + return TenantAccess{}, err + } + + if err := childIfMatch(t.Etag, ifMatch, "tenant access", name); err != nil { + return TenantAccess{}, err + } + + if enabled != nil { + t.Enabled = *enabled + } + + t.Etag = idgen.SyntheticGUID("apimanagement/tenant/" + t.Name + "/" + t.Etag) + + return *t, nil +} + +// cloneChildren deep-copies a children set (for snapshots and soft delete). +func cloneChildren(c *Children) *Children { + if c == nil { + return nil + } + + out := &Children{ + APIs: cloneChildMap(c.APIs), + Products: cloneChildMap(c.Products), + Portal: cloneChildMap(c.Portal), + Tenant: maps.Clone(c.Tenant), + } + + if c.Policy != nil { + p := cloneChild(c.Policy) + out.Policy = &p + } + + for k, t := range out.Tenant { + cp := *t + out.Tenant[k] = &cp + } + + return out +} + +// cloneChildMap deep-copies a child collection. +func cloneChildMap(in map[string]*ChildResource) map[string]*ChildResource { + if in == nil { + return nil + } + + out := make(map[string]*ChildResource, len(in)) + + for k, v := range in { + c := cloneChild(v) + out[k] = &c + } + + return out +} diff --git a/providers/azure/apimanagement/deleted.go b/providers/azure/apimanagement/deleted.go new file mode 100644 index 000000000..67d656b10 --- /dev/null +++ b/providers/azure/apimanagement/deleted.go @@ -0,0 +1,189 @@ +package apimanagement + +import ( + "context" + "sort" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +// softDeleteRetention is how long Azure keeps a soft-deleted service before it +// is purged for good. +const softDeleteRetention = 48 * time.Hour + +// DeletedService is a soft-deleted service: the service as it was when deleted +// (with its child resources), when it was deleted and when it will be purged. +type DeletedService struct { + Service Service `json:"service"` + Children *Children `json:"children,omitempty"` + DeletionDate time.Time `json:"deletionDate"` + ScheduledPurgeDate time.Time `json:"scheduledPurgeDate"` +} + +// ARMID is the deleted service's own resource id, +// /subscriptions/{s}/providers/Microsoft.ApiManagement/locations/{l}/deletedservices/{name}. +func (d *DeletedService) ARMID() string { + return "/subscriptions/" + d.Service.Subscription + "/providers/" + providerNamespace + + "/locations/" + normalizeLocation(d.Service.Location) + "/deletedservices/" + d.Service.Name +} + +// deletedKey is the case-insensitive store key of a soft-deleted service. +func deletedKey(sub, location, name string) string { + return strings.ToLower(sub) + "/" + normalizeLocation(location) + "/" + strings.ToLower(name) +} + +// softDeleteLocked moves the live service at k (and its children) into the +// soft-deleted store. The caller holds m.mu. +func (m *Mock) softDeleteLocked(k string, s *Service) { + children, _ := m.children.Get(k) + now := m.clock.Now().UTC() + + m.deleted.Set(deletedKey(s.Subscription, s.Location, s.Name), &DeletedService{ + Service: cloneService(s), + Children: cloneChildren(children), + DeletionDate: now, + ScheduledPurgeDate: now.Add(softDeleteRetention), + }) + + m.services.Delete(k) + m.children.Delete(k) +} + +// pruneExpiredLocked drops soft-deleted services whose retention has lapsed. +// The caller holds m.mu. +func (m *Mock) pruneExpiredLocked() { + now := m.clock.Now() + + for k, d := range m.deleted.All() { + if !now.Before(d.ScheduledPurgeDate) { + m.deleted.Delete(k) + } + } +} + +// deletedByNameLocked returns the soft-deleted service holding name in any +// subscription or location, or nil. The caller holds m.mu. +func (m *Mock) deletedByNameLocked(name string) *DeletedService { + m.pruneExpiredLocked() + + for _, d := range m.deleted.All() { + if strings.EqualFold(d.Service.Name, name) { + return d + } + } + + return nil +} + +// GetDeletedService returns the soft-deleted service name in sub/location, or a +// NotFound error when nothing by that name is soft-deleted there. +func (m *Mock) GetDeletedService(_ context.Context, sub, location, name string) (DeletedService, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.pruneExpiredLocked() + + d, ok := m.deleted.Get(deletedKey(sub, location, name)) + if !ok { + return DeletedService{}, deletedNotFound(name, location) + } + + return cloneDeleted(d), nil +} + +// ListDeletedServices returns every soft-deleted service in the subscription, +// sorted by name. +func (m *Mock) ListDeletedServices(_ context.Context, sub string) ([]DeletedService, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.pruneExpiredLocked() + + var out []DeletedService + + for _, d := range m.deleted.All() { + if strings.EqualFold(d.Service.Subscription, sub) { + out = append(out, cloneDeleted(d)) + } + } + + sort.Slice(out, func(i, j int) bool { return out[i].Service.Name < out[j].Service.Name }) + + return out, nil +} + +// PurgeDeletedService permanently removes a soft-deleted service, freeing its +// name. It is a NotFound error when nothing by that name is soft-deleted there. +func (m *Mock) PurgeDeletedService(_ context.Context, sub, location, name string) (DeletedService, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.pruneExpiredLocked() + + k := deletedKey(sub, location, name) + + d, ok := m.deleted.Get(k) + if !ok { + return DeletedService{}, deletedNotFound(name, location) + } + + m.deleted.Delete(k) + + return cloneDeleted(d), nil +} + +// restoreService recovers a soft-deleted service (a PUT whose properties set +// restore = true). As in Azure, every other property of the request is ignored: +// the service comes back as it was, with its child resources, under the +// request's resource id. It is a NotFound error when no service by that name is +// soft-deleted in sub/location. +func (m *Mock) restoreService(sub, rg, name, location, ifMatch string) (Service, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.pruneExpiredLocked() + + dk := deletedKey(sub, location, name) + + d, ok := m.deleted.Get(dk) + if !ok { + return Service{}, false, deletedNotFound(name, location) + } + + if err := checkIfMatch(false, nil, ifMatch, name); err != nil { + return Service{}, false, err + } + + k := serviceKey(sub, rg, name) + s := cloneService(&d.Service) + s.Subscription, s.ResourceGroup, s.Name = sub, rg, name + s.Identity = resolveIdentity(s.Identity, sub, rg, name) + m.commitLocked(k, &s) + + children := d.Children + if children == nil { + children = seedChildren(&s) + } + + m.children.Set(k, children) + m.deleted.Delete(dk) + + return cloneService(&s), true, nil +} + +// deletedNotFound is the NotFound error for a missing soft-deleted service. +func deletedNotFound(name, location string) error { + return cerrors.Newf(cerrors.NotFound, + "no soft-deleted API Management service %q in location %q", name, location) +} + +// cloneDeleted deep-copies a soft-deleted record for a caller. +func cloneDeleted(d *DeletedService) DeletedService { + out := *d + out.Service = cloneService(&d.Service) + out.Children = nil + + return out +} diff --git a/providers/azure/apimanagement/properties.go b/providers/azure/apimanagement/properties.go new file mode 100644 index 000000000..10c72a7bd --- /dev/null +++ b/providers/azure/apimanagement/properties.go @@ -0,0 +1,137 @@ +package apimanagement + +import ( + "encoding/json" + "strings" + "time" +) + +const ( + // platformDedicated / platformConsumption are the computePlatform versions + // Azure reports for the dedicated tiers and for the Consumption tier. + platformDedicated = "stv2" + platformConsumption = "mtv1" + + // notificationSenderDefault is the sender address Azure assigns when the + // caller sets none. + notificationSenderDefault = "apimgmt-noreply@mail.windowsazure.com" +) + +// Computed property keys. +const ( + propProvisioningState = "provisioningState" + propTargetProvisioningState = "targetProvisioningState" + propCreatedAt = "createdAtUtc" + propGatewayURL = "gatewayUrl" + propPublicIPs = "publicIPAddresses" + propPlatformVersion = "platformVersion" + propRegionalGatewayURL = "gatewayRegionalUrl" + propPortalURL = "portalUrl" + propDeveloperPortalURL = "developerPortalUrl" + propManagementAPIURL = "managementApiUrl" + propScmURL = "scmUrl" +) + +// defaultProperties are the writable properties Azure fills in when the caller +// leaves them unset. A caller-supplied value always wins. +func defaultProperties() map[string]any { + return map[string]any{ + "virtualNetworkType": "None", + "publicNetworkAccess": "Enabled", + "notificationSenderEmail": notificationSenderDefault, + "disableGateway": false, + "customProperties": map[string]any{}, + } +} + +// computedProperties returns the read-only properties Azure mints: the +// provisioning state, creation time, platform version and the endpoint URLs. +// The Consumption tier has only a gateway, so its regional gateway, portal, +// management and SCM endpoints are absent. A computed value always overwrites +// whatever the caller sent for that key. +func (s *Service) computedProperties() map[string]any { + out := map[string]any{ + propProvisioningState: s.ProvisioningState, + propTargetProvisioningState: "", + propCreatedAt: s.CreatedAt.UTC().Format(time.RFC3339), + propGatewayURL: s.Endpoints().Gateway, + propPublicIPs: []string{}, + propPlatformVersion: platformDedicated, + } + + if s.SkuName == skuConsumption { + out[propPlatformVersion] = platformConsumption + + return out + } + + ep := s.Endpoints() + out[propRegionalGatewayURL] = s.RegionalGatewayURL() + out[propPortalURL] = ep.Portal + out[propDeveloperPortalURL] = ep.DeveloperPortal + out[propManagementAPIURL] = ep.ManagementAPI + out[propScmURL] = ep.Scm + + return out +} + +// computedKeys lists every property key computedProperties can emit, so a +// re-materialization after a SKU change drops the endpoints the new tier does +// not have (e.g. portalUrl after a move to Consumption). +// +//nolint:gochecknoglobals // static key list +var computedKeys = []string{ + propProvisioningState, propTargetProvisioningState, propCreatedAt, propGatewayURL, propPublicIPs, + propPlatformVersion, propRegionalGatewayURL, propPortalURL, propDeveloperPortalURL, + propManagementAPIURL, propScmURL, +} + +// RegionalGatewayURL renders the primary region's gateway endpoint, +// https://--01.regional.azure-api.net. +func (s *Service) RegionalGatewayURL() string { + return "https://" + strings.ToLower(s.Name) + "-" + normalizeLocation(s.Location) + "-01.regional.azure-api.net" +} + +// materializeProperties rebuilds s.Properties as the full block Azure returns: +// the caller's writable properties, Azure's defaults for the writable fields +// the caller left unset, and the computed read-only fields. It runs on every +// write (and on restore), so the Go library and the HTTP server hand back the +// same resource. The restore flag is a request-only switch: it always +// reads back false, as in Azure. +func (s *Service) materializeProperties() { + obj := map[string]any{} + if len(s.Properties) > 0 { + if err := json.Unmarshal(s.Properties, &obj); err != nil { + obj = map[string]any{} + } + } + + obj["restore"] = false + + for _, k := range computedKeys { + delete(obj, k) + } + + for k, v := range defaultProperties() { + if cur, set := obj[k]; !set || cur == nil { + obj[k] = v + } + } + + for k, v := range s.computedProperties() { + obj[k] = v + } + + raw, err := json.Marshal(obj) + if err != nil { + return + } + + s.Properties = raw +} + +// normalizeLocation folds an ARM location display name ("East US") to its +// programmatic form ("eastus"), the form ARM compares locations in. +func normalizeLocation(loc string) string { + return strings.ToLower(strings.ReplaceAll(loc, " ", "")) +} diff --git a/providers/azure/apimanagement/review_fixes_test.go b/providers/azure/apimanagement/review_fixes_test.go new file mode 100644 index 000000000..a5152b5a2 --- /dev/null +++ b/providers/azure/apimanagement/review_fixes_test.go @@ -0,0 +1,463 @@ +package apimanagement_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" +) + +func props(t *testing.T, s *apimanagement.Service) map[string]any { + t.Helper() + + out := map[string]any{} + if err := json.Unmarshal(s.Properties, &out); err != nil { + t.Fatalf("properties: %v", err) + } + + return out +} + +// TestLibraryServiceCarriesDefaultsAndComputed: the provider's service holds +// Azure's defaults and the computed fields, re-derived on every write (a move to +// Consumption drops the dedicated-tier endpoints), and a caller value for a +// defaulted field wins. +func TestLibraryServiceCarriesDefaultsAndComputed(t *testing.T) { + m, _ := newMock() + s := create(t, m, "apim1") + p := props(t, &s) + + want := map[string]any{ + "virtualNetworkType": "None", + "publicNetworkAccess": "Enabled", + "notificationSenderEmail": "apimgmt-noreply@mail.windowsazure.com", + "platformVersion": "stv2", + "gatewayRegionalUrl": "https://apim1-eastus-01.regional.azure-api.net", + "portalUrl": "https://apim1.portal.azure-api.net", + "provisioningState": "Succeeded", + "createdAtUtc": "2026-09-01T10:30:00Z", + "restore": false, + } + for k, v := range want { + if p[k] != v { + t.Errorf("%s = %v, want %v", k, p[k], v) + } + } + + in := devInput() + in.Properties = json.RawMessage(`{"publisherEmail":"a@b.test","publisherName":"C","virtualNetworkType":"External",` + + `"provisioningState":"Failed"}`) + + s, _, err := m.CreateOrUpdateService(context.Background(), sub, rg, "apim1", "East US", in) + if err != nil { + t.Fatal(err) + } + + if p := props(t, &s); p["virtualNetworkType"] != "External" || p["provisioningState"] != "Succeeded" { + t.Errorf("caller default must win and computed must not: %v", p) + } + + s, err = m.UpdateService(context.Background(), sub, rg, "apim1", &apimanagement.ServiceInput{ + SkuName: sptr("Consumption"), SkuCapacity: i32(0), + }) + if err != nil { + t.Fatal(err) + } + + if p := props(t, &s); p["portalUrl"] != nil || p["platformVersion"] != "mtv1" { + t.Errorf("Consumption must drop the dedicated endpoints: %v", p) + } +} + +// TestSKUCeilingsAndZones covers each tier's unit ceiling and the Premium-only +// zones rule, on create and on PATCH. +func TestSKUCeilingsAndZones(t *testing.T) { + ctx := context.Background() + + for sku, maxUnits := range map[string]int32{ + "Developer": 1, "Basic": 2, "Standard": 4, "Premium": 12, "BasicV2": 10, "StandardV2": 10, + } { + m, _ := newMock() + in := devInput() + in.SkuName, in.SkuCapacity = sptr(sku), i32(maxUnits+1) + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "svc", "eastus", in); !cerrors.IsInvalidArgument(err) { + t.Errorf("%s capacity %d: err = %v, want InvalidArgument", sku, maxUnits+1, err) + } + + in.SkuCapacity = i32(maxUnits) + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "svc", "eastus", in); err != nil { + t.Errorf("%s capacity %d: %v", sku, maxUnits, err) + } + } + + m, _ := newMock() + in := devInput() + in.SkuName, in.SkuCapacity = sptr("Isolated"), i32(20) + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "iso", "eastus", in); err != nil { + t.Errorf("Isolated has no emulator ceiling: %v", err) + } + + in = devInput() + in.Zones = []string{"1"} + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "zonal", "eastus", in); !cerrors.IsInvalidArgument(err) { + t.Errorf("zones on Developer: err = %v", err) + } + + create(t, m, "dev") + + if _, err := m.UpdateService(ctx, sub, rg, "dev", &apimanagement.ServiceInput{Zones: []string{"1"}}); !cerrors.IsInvalidArgument(err) { + t.Errorf("PATCH zones on Developer: err = %v", err) + } +} + +// TestNameLocationAndIfMatch covers the global name, the immutable location and +// the conditional writes. +func TestNameLocationAndIfMatch(t *testing.T) { + m, _ := newMock() + ctx := context.Background() + s := create(t, m, "apim1") + + if _, _, err := m.CreateOrUpdateService(ctx, "sub2", "rg2", "APIM1", "eastus", devInput()); !errors.Is(err, apimanagement.ErrNameNotAvailable) || + !cerrors.IsAlreadyExists(err) { + t.Errorf("same name elsewhere: err = %v", err) + } + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "westeurope", devInput()); !errors.Is(err, apimanagement.ErrLocationMismatch) { + t.Errorf("location change: err = %v", err) + } + + if v := m.CheckNameAvailability(ctx, "apim1"); v.Available || v.Reason != "AlreadyExists" { + t.Errorf("taken name = %+v", v) + } + + if v := m.CheckNameAvailability(ctx, "-bad"); v.Available || v.Reason != "Invalid" || v.Message == "" { + t.Errorf("invalid name = %+v", v) + } + + stale := devInput() + stale.IfMatch = `W/"not-it"` + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "eastus", stale); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale If-Match PUT: err = %v", err) + } + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "fresh", "eastus", stale); !cerrors.IsFailedPrecondition(err) { + t.Errorf("If-Match on a missing service: err = %v", err) + } + + if _, err := m.UpdateService(ctx, sub, rg, "apim1", &apimanagement.ServiceInput{IfMatch: "x"}); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale If-Match PATCH: err = %v", err) + } + + cur := devInput() + cur.IfMatch = `"` + s.Etag + `"` + + s2, _, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "eastus", cur) + if err != nil || s2.Etag == s.Etag { + t.Fatalf("current If-Match PUT: %v (etag %q)", err, s2.Etag) + } + + if _, err := m.DeleteServiceIfMatch(ctx, sub, rg, "apim1", s.Etag); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale If-Match DELETE: err = %v", err) + } + + if existed, err := m.DeleteService(ctx, sub, rg, "apim1"); err != nil || !existed { + t.Errorf("DELETE: %v %v", existed, err) + } + + if existed, err := m.DeleteService(ctx, sub, rg, "apim1"); err != nil || existed { + t.Errorf("second DELETE: %v %v", existed, err) + } +} + +// TestSoftDeleteRetentionPurgeAndRestore covers the soft-deleted lifecycle. +func TestSoftDeleteRetentionPurgeAndRestore(t *testing.T) { + m, clk := newMock() + ctx := context.Background() + create(t, m, "apim1") + + if _, err := m.DeleteService(ctx, sub, rg, "apim1"); err != nil { + t.Fatal(err) + } + + d, err := m.GetDeletedService(ctx, sub, "East US", "apim1") + if err != nil || !d.ScheduledPurgeDate.Equal(epoch.Add(48*time.Hour)) || + d.ARMID() != "/subscriptions/sub/providers/Microsoft.ApiManagement/locations/eastus/deletedservices/apim1" { + t.Fatalf("deleted = %+v %q, %v", d, d.ARMID(), err) + } + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "eastus", devInput()); !errors.Is(err, apimanagement.ErrSoftDeleted) { + t.Errorf("create over a soft-deleted name: err = %v", err) + } + + restore := &apimanagement.ServiceInput{Properties: json.RawMessage(`{"restore":true}`)} + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "westus", restore); !cerrors.IsNotFound(err) { + t.Errorf("restore in the wrong location: err = %v", err) + } + + got, created, err := m.CreateOrUpdateService(ctx, sub, rg, "apim1", "eastus", restore) + if err != nil || !created || got.SkuName != "Developer" { + t.Fatalf("restore: %+v %v %v", got, created, err) + } + + if apis, err := m.ListAPIs(ctx, sub, rg, "apim1"); err != nil || len(apis) != 1 { + t.Errorf("restore must bring the children back: %v %v", apis, err) + } + + if _, err := m.DeleteService(ctx, sub, rg, "apim1"); err != nil { + t.Fatal(err) + } + + if list, _ := m.ListDeletedServices(ctx, sub); len(list) != 1 { + t.Errorf("deleted list = %v", list) + } + + if list, _ := m.ListDeletedServices(ctx, "other"); len(list) != 0 { + t.Errorf("other subscription's deleted list = %v", list) + } + + clk.Advance(48 * time.Hour) + + if _, err := m.GetDeletedService(ctx, sub, "eastus", "apim1"); !cerrors.IsNotFound(err) { + t.Errorf("retention lapsed: err = %v", err) + } + + if _, err := m.PurgeDeletedService(ctx, sub, "eastus", "apim1"); !cerrors.IsNotFound(err) { + t.Errorf("purge after lapse: err = %v", err) + } + + create(t, m, "apim1") + + if err := m.PurgeResourceGroup(ctx, sub, rg); err != nil { + t.Fatal(err) + } + + if _, err := m.PurgeDeletedService(ctx, sub, "eastus", "apim1"); err != nil { + t.Errorf("a group delete soft-deletes; purge: %v", err) + } +} + +// TestChildResources covers the child surface's error paths through the +// library. +func TestChildResources(t *testing.T) { + m, _ := newMock() + ctx := context.Background() + create(t, m, "apim1") + + if _, err := m.ListAPIs(ctx, sub, rg, "missing"); !cerrors.IsNotFound(err) { + t.Errorf("children of a missing service: %v", err) + } + + p, err := m.GetProduct(ctx, sub, rg, "apim1", "Starter") + if err != nil || p.Name != "starter" { + t.Fatalf("GetProduct: %+v %v", p, err) + } + + if _, err := m.GetAPI(ctx, sub, rg, "apim1", "nope"); !cerrors.IsNotFound(err) { + t.Errorf("missing api: %v", err) + } + + if _, err := m.DeleteProduct(ctx, sub, rg, "apim1", "starter", `"stale"`); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale product delete: %v", err) + } + + if ok, err := m.DeleteProduct(ctx, sub, rg, "apim1", "starter", `"`+p.Etag+`"`); !ok || err != nil { + t.Errorf("product delete: %v %v", ok, err) + } + + if ok, err := m.DeleteAPI(ctx, sub, rg, "apim1", "nope", ""); ok || err != nil { + t.Errorf("missing api delete: %v %v", ok, err) + } + + for _, tc := range []struct{ value, format string }{ + {"", "xml-link"}, {"", "xml"}, {"", "rawxml"}, + } { + if _, _, err := m.PutPolicy(ctx, sub, rg, "apim1", tc.value, tc.format, ""); !cerrors.IsInvalidArgument(err) { + t.Errorf("policy %q/%q: err = %v", tc.value, tc.format, err) + } + } + + if _, _, err := m.PutPolicy(ctx, sub, rg, "apim1", "", "", `"x"`); !cerrors.IsFailedPrecondition(err) { + t.Errorf("If-Match on a missing policy: %v", err) + } + + pol, created, err := m.PutPolicy(ctx, sub, rg, "apim1", "", "rawxml", "*") + if err != nil || !created || !strings.Contains(string(pol.Properties), `"rawxml"`) { + t.Fatalf("PutPolicy rawxml: %s %v %v", pol.Properties, created, err) + } + + if ok, err := m.DeletePolicy(ctx, sub, rg, "apim1", ""); !ok || err != nil { + t.Errorf("DeletePolicy: %v %v", ok, err) + } + + if ok, err := m.DeletePolicy(ctx, sub, rg, "apim1", ""); ok || err != nil { + t.Errorf("second DeletePolicy: %v %v", ok, err) + } + + if _, err := m.GetPortalSetting(ctx, sub, rg, "apim1", "nope"); !cerrors.IsNotFound(err) { + t.Errorf("missing portal setting: %v", err) + } + + if _, err := m.PutPortalSetting(ctx, sub, rg, "apim1", apimanagement.PortalSignIn, json.RawMessage(`[]`), ""); !cerrors.IsInvalidArgument(err) { + t.Errorf("non-object portal setting: %v", err) + } + + if _, err := m.PutPortalSetting(ctx, sub, rg, "apim1", apimanagement.PortalSignIn, json.RawMessage(`{"enabled":true}`), `"x"`); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale portal setting: %v", err) + } + + if key, err := m.DelegationValidationKey(ctx, sub, rg, "apim1"); key != "" || err != nil { + t.Errorf("default delegation key = %q %v", key, err) + } + + if _, err := m.GetTenantAccess(ctx, sub, rg, "apim1", "nope"); !cerrors.IsNotFound(err) { + t.Errorf("missing tenant access: %v", err) + } + + if _, err := m.UpdateTenantAccess(ctx, sub, rg, "apim1", "access", nil, `"x"`); !cerrors.IsFailedPrecondition(err) { + t.Errorf("stale tenant access: %v", err) + } + + git, err := m.GetTenantAccess(ctx, sub, rg, "apim1", "GITACCESS") + if err != nil || git.PrincipalID != "git" || git.PrimaryKey == "" { + t.Errorf("gitAccess = %+v %v", git, err) + } + + in := devInput() + in.SkuName, in.SkuCapacity = sptr("BasicV2"), i32(1) + + if _, _, err := m.CreateOrUpdateService(ctx, sub, rg, "v2", "eastus", in); err != nil { + t.Fatal(err) + } + + for _, err := range []error{ + func() error { _, e := m.GetPortalSetting(ctx, sub, rg, "v2", "signin"); return e }(), + func() error { _, e := m.GetTenantAccess(ctx, sub, rg, "v2", "access"); return e }(), + } { + if !errors.Is(err, apimanagement.ErrTierNotSupported) { + t.Errorf("v2 tier: err = %v", err) + } + } +} + +// TestSnapshotRoundTripsChildrenAndDeleted: a snapshot carries the child +// resources and the soft-deleted services, and a restore re-materializes a +// service written before the provider owned the computed fields. +func TestSnapshotRoundTripsChildrenAndDeleted(t *testing.T) { + m, _ := newMock() + ctx := context.Background() + create(t, m, "live") + create(t, m, "gone") + + if _, _, err := m.PutPolicy(ctx, sub, rg, "live", "", "xml", ""); err != nil { + t.Fatal(err) + } + + if _, err := m.DeleteService(ctx, sub, rg, "gone"); err != nil { + t.Fatal(err) + } + + data, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatal(err) + } + + r, _ := newMock() + if err := r.Restore(ctx, data); err != nil { + t.Fatal(err) + } + + if _, err := r.GetPolicy(ctx, sub, rg, "live"); err != nil { + t.Errorf("policy lost across snapshot: %v", err) + } + + if _, err := r.GetDeletedService(ctx, sub, "eastus", "gone"); err != nil { + t.Errorf("soft-deleted service lost across snapshot: %v", err) + } + + legacy := `{"services":{"/subscriptions/sub/resourcegroups/rg/providers/microsoft.apimanagement/service/old":{"subscription":"sub","resourceGroup":"rg","name":"old","location":"eastus",` + + `"skuName":"Developer","skuCapacity":1,"properties":{"publisherEmail":"a@b","publisherName":"n"},` + + `"provisioningState":"Succeeded","etag":"e","createdAt":"2026-01-01T00:00:00Z"}}}` + + l, _ := newMock() + if err := l.Restore(ctx, json.RawMessage(legacy)); err != nil { + t.Fatal(err) + } + + s, err := l.GetService(ctx, "sub", "rg", "old") + if err != nil || props(t, &s)["gatewayRegionalUrl"] == nil { + t.Errorf("legacy snapshot not re-materialized: %s %v", s.Properties, err) + } + + if apis, err := l.ListAPIs(ctx, "sub", "rg", "old"); err != nil || len(apis) != 0 { + t.Errorf("a legacy service gets default settings, not sample APIs: %v %v", apis, err) + } + + if _, err := l.GetPortalSetting(ctx, "sub", "rg", "old", "signup"); err != nil { + t.Errorf("a legacy service gets default portal settings: %v", err) + } + + if err := l.Restore(ctx, nil); err != nil { + t.Errorf("empty restore: %v", err) + } + + if err := l.Restore(ctx, json.RawMessage(`{`)); err == nil { + t.Error("malformed snapshot must fail") + } +} + +// TestPortalAndTenantWrites covers the successful portal-setting and tenant +// access writes: the etag rotates and the delegation key never leaves via GET. +func TestPortalAndTenantWrites(t *testing.T) { + m, _ := newMock() + ctx := context.Background() + create(t, m, "apim1") + + if products, err := m.ListProducts(ctx, sub, rg, "apim1"); err != nil || len(products) != 2 { + t.Errorf("products = %v %v", products, err) + } + + if _, err := m.GetPolicy(ctx, sub, rg, "missing"); !cerrors.IsNotFound(err) { + t.Errorf("policy of a missing service: %v", err) + } + + before, err := m.GetPortalSetting(ctx, sub, rg, "apim1", apimanagement.PortalDelegation) + if err != nil { + t.Fatal(err) + } + + after, err := m.PutPortalSetting(ctx, sub, rg, "apim1", apimanagement.PortalDelegation, + json.RawMessage(`{"url":"https://d.test","validationKey":"k1"}`), `"`+before.Etag+`"`) + if err != nil || after.Etag == before.Etag || strings.Contains(string(after.Properties), "k1") { + t.Fatalf("PUT delegation: %s %v", after.Properties, err) + } + + if got, _ := m.GetPortalSetting(ctx, sub, rg, "apim1", apimanagement.PortalDelegation); strings.Contains(string(got.Properties), "k1") { + t.Errorf("GET must not reveal the key: %s", got.Properties) + } + + if key, err := m.DelegationValidationKey(ctx, sub, rg, "apim1"); key != "k1" || err != nil { + t.Errorf("listSecrets key = %q %v", key, err) + } + + ta, err := m.GetTenantAccess(ctx, sub, rg, "apim1", apimanagement.TenantAccessName) + if err != nil || ta.Enabled { + t.Fatalf("tenant access = %+v %v", ta, err) + } + + on := true + + upd, err := m.UpdateTenantAccess(ctx, sub, rg, "apim1", apimanagement.TenantAccessName, &on, ta.Etag) + if err != nil || !upd.Enabled || upd.Etag == ta.Etag { + t.Errorf("tenant access update = %+v %v", upd, err) + } +} diff --git a/providers/azure/apimanagement/snapshot.go b/providers/azure/apimanagement/snapshot.go index a0bc52a8d..65b14efc9 100644 --- a/providers/azure/apimanagement/snapshot.go +++ b/providers/azure/apimanagement/snapshot.go @@ -10,14 +10,18 @@ import ( var _ snapshot.Snapshottable = (*Mock)(nil) -// snapshotState is the on-disk shape: the service store keyed by its -// (lowercased) resource id. +// snapshotState is the on-disk shape: the live services and their child +// resources keyed by the (lowercased) service resource id, and the +// soft-deleted services keyed by subscription/location/name. type snapshotState struct { Services json.RawMessage `json:"services,omitempty"` + Children json.RawMessage `json:"children,omitempty"` + Deleted json.RawMessage `json:"deleted,omitempty"` } -// Snapshot captures every API Management service. includeAssets is unused: -// these resources hold no bulk object bodies. +// Snapshot captures every API Management service, its child resources and the +// soft-deleted services. includeAssets is unused: these resources hold no bulk +// object bodies. func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { m.mu.RLock() defer m.mu.RUnlock() @@ -27,7 +31,17 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { return nil, fmt.Errorf("apimanagement: snapshot services: %w", err) } - data, err := json.Marshal(snapshotState{Services: services}) + children, err := m.children.Snapshot() + if err != nil { + return nil, fmt.Errorf("apimanagement: snapshot children: %w", err) + } + + deleted, err := m.deleted.Snapshot() + if err != nil { + return nil, fmt.Errorf("apimanagement: snapshot deleted services: %w", err) + } + + data, err := json.Marshal(snapshotState{Services: services, Children: children, Deleted: deleted}) if err != nil { return nil, fmt.Errorf("apimanagement: marshal snapshot: %w", err) } @@ -35,7 +49,10 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { return data, nil } -// Restore rebuilds every service under its original id. +// Restore rebuilds every service, child resource and soft-deleted service under +// its original key. Each restored service's properties block is re-materialized, +// so a snapshot written before the provider owned the computed fields comes +// back with them. func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { m.mu.Lock() defer m.mu.Unlock() @@ -49,12 +66,28 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { return fmt.Errorf("apimanagement: unmarshal snapshot: %w", err) } - if len(state.Services) == 0 { - return nil + loads := []struct { + name string + raw json.RawMessage + load func([]byte) error + }{ + {"services", state.Services, m.services.LoadSnapshot}, + {"children", state.Children, m.children.LoadSnapshot}, + {"deleted services", state.Deleted, m.deleted.LoadSnapshot}, + } + + for _, l := range loads { + if len(l.raw) == 0 { + continue + } + + if err := l.load(l.raw); err != nil { + return fmt.Errorf("apimanagement: restore %s: %w", l.name, err) + } } - if err := m.services.LoadSnapshot(state.Services); err != nil { - return fmt.Errorf("apimanagement: restore services: %w", err) + for _, s := range m.services.All() { + s.materializeProperties() } return nil diff --git a/providers/azure/apimanagement/validate.go b/providers/azure/apimanagement/validate.go index d163725ce..bdb722cf4 100644 --- a/providers/azure/apimanagement/validate.go +++ b/providers/azure/apimanagement/validate.go @@ -2,6 +2,7 @@ package apimanagement import ( "encoding/json" + "errors" "regexp" "strings" @@ -14,40 +15,69 @@ const ( // skuConsumption is the serverless tier, the only one whose capacity is 0. skuConsumption = "Consumption" + // skuPremium is the only classic tier that can be deployed across + // availability zones. + skuPremium = "Premium" +) + +// Sentinels the HTTP layer maps to APIM's specific ARM error codes. Each is +// joined onto a canonical cerrors value, so cerrors.GetCode/Message still work. +var ( + // ErrNameNotAvailable: the service name (a global *.azure-api.net DNS + // label) is already used by another live service. + ErrNameNotAvailable = errors.New("api management service name not available") + // ErrSoftDeleted: the name belongs to a soft-deleted service that must be + // recovered (properties.restore) or purged first. + ErrSoftDeleted = errors.New("api management service is soft-deleted") + // ErrLocationMismatch: a PUT on an existing service names a different + // location, which is immutable. + ErrLocationMismatch = errors.New("api management service location is immutable") ) // serviceNamePattern is Azure's service-name rule: starts with a letter, then // letters, digits and hyphens, and does not end with a hyphen. var serviceNamePattern = regexp.MustCompile(`^[A-Za-z]([A-Za-z0-9-]*[A-Za-z0-9])?$`) +// skuInfo is a tier's canonical casing and its maximum unit count per region +// (0 = no emulator-enforced ceiling). +type skuInfo struct { + name string + maxCapacity int32 +} + // validSKUs is the armapimanagement v3 SKUType enum, keyed lowercase so the -// lookup is case-insensitive like ARM, mapped to the canonical casing. +// lookup is case-insensitive like ARM, mapped to the canonical casing and the +// per-region unit ceiling Azure publishes for the tier: Developer 1, Basic 2, +// Standard 4, Premium 12, BasicV2/StandardV2 10 (the azurerm provider's own +// sku_name validation uses the same ceilings, except that it lets Premium go to +// 99 for support-raised quotas). Isolated is by-request only and has no +// ceiling here. // -//nolint:gochecknoglobals // static lookup table -var validSKUs = map[string]string{ - "developer": "Developer", - "basic": "Basic", - "standard": "Standard", - "premium": "Premium", - "consumption": skuConsumption, - "isolated": "Isolated", - "basicv2": "BasicV2", - "standardv2": "StandardV2", +//nolint:gochecknoglobals // static lookup table of published limits +var validSKUs = map[string]skuInfo{ + "developer": {"Developer", 1}, + "basic": {"Basic", 2}, + "standard": {"Standard", 4}, + "premium": {skuPremium, 12}, + "consumption": {skuConsumption, 0}, + "isolated": {"Isolated", 0}, + "basicv2": {"BasicV2", 10}, + "standardv2": {"StandardV2", 10}, } // canonicalSKU returns the canonical casing of a known SKU name, or the input // unchanged when it is unknown (validation rejects unknown names first). func canonicalSKU(name string) string { if c, ok := validSKUs[strings.ToLower(name)]; ok { - return c + return c.name } return name } // validateCreate rejects a create/replace request with missing or malformed -// required fields: the path identity, location, the SKU block and the two -// publisher properties. +// required fields: the path identity, location, the SKU block, the zones and +// the two publisher properties. func validateCreate(sub, rg, name, location string, in *ServiceInput) error { switch { case sub == "": @@ -74,13 +104,17 @@ func validateCreate(sub, rg, name, location string, in *ServiceInput) error { return err } + if err := validateZones(*in.SkuName, in.Zones); err != nil { + return err + } + return validatePublisher(in.Properties) } // validateName enforces Azure's service-name rule (1-50 characters, starts with // a letter, letters/digits/hyphens, no trailing hyphen). func validateName(name string) error { - if name == "" || len(name) > maxServiceNameLen || !serviceNamePattern.MatchString(name) { + if !validName(name) { return cerrors.Newf(cerrors.InvalidArgument, "invalid API Management service name %q: it must be 1-%d characters, start with a letter, "+ "contain only letters, digits and hyphens, and not end with a hyphen", name, maxServiceNameLen) @@ -89,15 +123,21 @@ func validateName(name string) error { return nil } +// validName reports whether name satisfies Azure's service-name rule. +func validName(name string) bool { + return name != "" && len(name) <= maxServiceNameLen && serviceNamePattern.MatchString(name) +} + // validateSKU checks the SKU name is a known tier and the capacity fits it: the -// Consumption tier must be 0 units, every other tier at least 1. +// Consumption tier must be 0 units, every other tier at least 1 and at most the +// tier's published ceiling. func validateSKU(name string, capacity int32) error { - canon, ok := validSKUs[strings.ToLower(name)] + info, ok := validSKUs[strings.ToLower(name)] if !ok { return cerrors.Newf(cerrors.InvalidArgument, "invalid sku.name %q", name) } - if canon == skuConsumption { + if info.name == skuConsumption { if capacity != 0 { return cerrors.Newf(cerrors.InvalidArgument, "sku.capacity must be 0 for the Consumption tier, got %d", capacity) @@ -108,12 +148,27 @@ func validateSKU(name string, capacity int32) error { if capacity < 1 { return cerrors.Newf(cerrors.InvalidArgument, - "sku.capacity must be at least 1 for the %s tier, got %d", canon, capacity) + "sku.capacity must be at least 1 for the %s tier, got %d", info.name, capacity) + } + + if info.maxCapacity > 0 && capacity > info.maxCapacity { + return cerrors.Newf(cerrors.InvalidArgument, + "sku.capacity must be at most %d for the %s tier, got %d", info.maxCapacity, info.name, capacity) } return nil } +// validateZones allows availability zones only on the Premium tier. +func validateZones(skuName string, zones []string) error { + if len(zones) == 0 || canonicalSKU(skuName) == skuPremium { + return nil + } + + return cerrors.Newf(cerrors.InvalidArgument, + "availability zones are supported only in the Premium tier, not %s", canonicalSKU(skuName)) +} + // validatePublisher requires non-empty properties.publisherEmail and // properties.publisherName. func validatePublisher(props json.RawMessage) error { @@ -138,7 +193,27 @@ func validatePublisher(props json.RawMessage) error { } } -// invalid is an InvalidArgument error (ARM 400 InvalidParameter). +// restoreRequested reports whether the request body sets properties.restore, +// the flag that recovers a soft-deleted service instead of creating one. +func restoreRequested(props json.RawMessage) bool { + var p struct { + Restore bool `json:"restore"` + } + + if len(props) == 0 || json.Unmarshal(props, &p) != nil { + return false + } + + return p.Restore +} + +// invalid is an InvalidArgument error (APIM 400 ValidationError). func invalid(msg string) error { return cerrors.New(cerrors.InvalidArgument, msg) } + +// coded joins a sentinel onto a canonical error so the HTTP layer can pick the +// APIM-specific ARM code while generic callers still see the cerrors code. +func coded(sentinel error, err *cerrors.Error) error { + return errors.Join(err, sentinel) +} diff --git a/providers/azure/apimanagement_discovery.go b/providers/azure/apimanagement_discovery.go index 10c7edd13..0820f81c3 100644 --- a/providers/azure/apimanagement_discovery.go +++ b/providers/azure/apimanagement_discovery.go @@ -29,8 +29,6 @@ func (d apiManagementDiscovery) DiscoverResources( return projectDiscovery(items, func(s *apimanagement.Service) resourcediscovery.DiscoveredResource { props := map[string]any{ propProvisioningState: s.ProvisioningState, - "sku": s.SkuName, - "skuCapacity": s.SkuCapacity, "gatewayUrl": s.Endpoints().Gateway, } @@ -41,7 +39,12 @@ func (d apiManagementDiscovery) DiscoverResources( ARN: s.ARMID(), Region: s.Location, Tags: s.Tags, - Attrs: resourcediscovery.Attributes{Properties: props}, + Attrs: resourcediscovery.Attributes{ + SKU: s.SkuName, + SKUCapacity: int(s.SkuCapacity), + Zones: append([]string(nil), s.Zones...), + Properties: props, + }, } }), nil } diff --git a/server/azure/apimanagement/apimanagement_sdk_test.go b/server/azure/apimanagement/apimanagement_sdk_test.go index d616d2a54..e183da58e 100644 --- a/server/azure/apimanagement/apimanagement_sdk_test.go +++ b/server/azure/apimanagement/apimanagement_sdk_test.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/stackshy/cloudemu/v2" + azureprov "github.com/stackshy/cloudemu/v2/providers/azure" azureserver "github.com/stackshy/cloudemu/v2/server/azure" ) @@ -38,15 +39,19 @@ func (fakeCred) GetToken(context.Context, policy.TokenRequestOptions) (azcore.Ac } type fixture struct { - ts *httptest.Server - svc *armapimanagement.ServiceClient - rgs *armresources.ResourceGroupsClient + ts *httptest.Server + prov *azureprov.Provider + opts *arm.ClientOptions + cf *armapimanagement.ClientFactory + svc *armapimanagement.ServiceClient + rgs *armresources.ResourceGroupsClient } func newFixture(t *testing.T) *fixture { t.Helper() - ts := httptest.NewTLSServer(azureserver.NewFromProvider(cloudemu.NewAzure())) + prov := cloudemu.NewAzure() + ts := httptest.NewTLSServer(azureserver.NewFromProvider(prov)) t.Cleanup(ts.Close) opts := &arm.ClientOptions{ClientOptions: azcore.ClientOptions{ @@ -70,7 +75,7 @@ func newFixture(t *testing.T) *fixture { t.Fatalf("resource groups client: %v", err) } - f := &fixture{ts: ts, svc: cf.NewServiceClient(), rgs: rgs} + f := &fixture{ts: ts, prov: prov, opts: opts, cf: cf, svc: cf.NewServiceClient(), rgs: rgs} f.ensureRG(t, rgName) f.ensureRG(t, rgOther) @@ -249,8 +254,12 @@ func assertPatched(t *testing.T, got, before *armapimanagement.ServiceResource) t.Errorf("PATCH dropped unnamed properties: %v", got.Properties.CustomProperties) } - if deref(got.Identity.PrincipalID) != deref(before.Identity.PrincipalID) || deref(got.Etag) != deref(before.Etag) { - t.Error("PATCH re-minted the identity or etag") + if deref(got.Identity.PrincipalID) != deref(before.Identity.PrincipalID) { + t.Error("PATCH re-minted the identity") + } + + if deref(got.Etag) == deref(before.Etag) { + t.Error("PATCH must rotate the etag") } } @@ -315,8 +324,8 @@ func TestSDKConsumptionTier(t *testing.T) { } } -// TestSDKValidationErrors asserts every rejected create surfaces as an ARM 400 -// through the real client. +// TestSDKValidationErrors asserts every rejected create surfaces as APIM's 400 +// ValidationError through the real client. func TestSDKValidationErrors(t *testing.T) { f := newFixture(t) ctx := context.Background() @@ -346,7 +355,7 @@ func TestSDKValidationErrors(t *testing.T) { } _, err := f.svc.BeginCreateOrUpdate(ctx, rgName, tc.name, body, nil) - assertStatus(t, err, http.StatusBadRequest, "InvalidParameter") + assertStatus(t, err, http.StatusBadRequest, "ValidationError") }) } } diff --git a/server/azure/apimanagement/children.go b/server/azure/apimanagement/children.go new file mode 100644 index 000000000..d056c20e9 --- /dev/null +++ b/server/azure/apimanagement/children.go @@ -0,0 +1,370 @@ +package apimanagement + +import ( + "context" + "encoding/json" + "net/http" + "strings" + + "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" + "github.com/stackshy/cloudemu/v2/server/wire/azurearm" +) + +// Child collection path segments under .../service/{name}/. +const ( + segAPIs = "apis" + segProducts = "products" + segPolicies = "policies" + segPortal = "portalsettings" + segTenant = "tenant" + policyName = "policy" + actListSecrets = "listsecrets" +) + +// childStore is the child-resource half of Store. +type childStore interface { + ListAPIs(ctx context.Context, sub, rg, svc string) ([]apimanagement.ChildResource, error) + GetAPI(ctx context.Context, sub, rg, svc, apiID string) (apimanagement.ChildResource, error) + DeleteAPI(ctx context.Context, sub, rg, svc, apiID, ifMatch string) (bool, error) + ListProducts(ctx context.Context, sub, rg, svc string) ([]apimanagement.ChildResource, error) + GetProduct(ctx context.Context, sub, rg, svc, productID string) (apimanagement.ChildResource, error) + DeleteProduct(ctx context.Context, sub, rg, svc, productID, ifMatch string) (bool, error) + + GetPolicy(ctx context.Context, sub, rg, svc string) (apimanagement.ChildResource, error) + PutPolicy(ctx context.Context, sub, rg, svc, value, format, ifMatch string) (apimanagement.ChildResource, bool, error) + DeletePolicy(ctx context.Context, sub, rg, svc, ifMatch string) (bool, error) + + GetPortalSetting(ctx context.Context, sub, rg, svc, name string) (apimanagement.ChildResource, error) + PutPortalSetting( + ctx context.Context, sub, rg, svc, name string, props json.RawMessage, ifMatch string, + ) (apimanagement.ChildResource, error) + DelegationValidationKey(ctx context.Context, sub, rg, svc string) (string, error) + + GetTenantAccess(ctx context.Context, sub, rg, svc, name string) (apimanagement.TenantAccess, error) + UpdateTenantAccess( + ctx context.Context, sub, rg, svc, name string, enabled *bool, ifMatch string, + ) (apimanagement.TenantAccess, error) +} + +// childSegments returns the path segments after .../service/{name}/. ParsePath +// keeps only the first few trailing segments, so the child router reads the +// whole tail itself to tell .../apis/{id} from .../apis/{id}/operations. +func childSegments(urlPath string) []string { + parts := strings.Split(strings.Trim(urlPath, "/"), "/") + + for i := 0; i+3 < len(parts); i++ { + if strings.EqualFold(parts[i], "providers") && strings.EqualFold(parts[i+1], providerName) && + strings.EqualFold(parts[i+2], serviceType) { + return parts[i+4:] + } + } + + return nil +} + +// childScope is one child request: its service coordinates and its path tail. +type childScope struct { + sub, rg, svc string + serviceID string + segs []string +} + +// serveChild routes the service's child resources. +func (h *Handler) serveChild(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath, segs []string) { + if len(segs) == 0 { + unsupportedChild(w, rp.SubResource) + return + } + + c := childScope{ + sub: rp.Subscription, rg: rp.ResourceGroup, svc: rp.ResourceName, + serviceID: azurearm.BuildResourceID(rp.Subscription, rp.ResourceGroup, providerName, serviceType, rp.ResourceName), + segs: segs, + } + + switch strings.ToLower(segs[0]) { + case segAPIs, segProducts: + h.serveCollection(w, r, &c) + case segPolicies: + h.servePolicy(w, r, &c) + case segPortal: + h.servePortal(w, r, &c) + case segTenant: + h.serveTenant(w, r, &c) + default: + unsupportedChild(w, segs[0]) + } +} + +// unsupportedChild answers a child path the emulator does not model. +func unsupportedChild(w http.ResponseWriter, what string) { + azurearm.WriteError(w, http.StatusNotFound, "InvalidResourceType", + "unsupported API Management sub-resource "+what) +} + +// writeChild writes a child resource with its ETag header. +func writeChild(w http.ResponseWriter, r *http.Request, status int, etag string, body any) { + w.Header().Set("ETag", `"`+etag+`"`) + + if r.Method == http.MethodHead { + w.WriteHeader(status) + return + } + + azurearm.WriteJSON(w, status, body) +} + +// Path-tail lengths: .../{collection}, .../{collection}/{id} and +// .../{collection}/{id}/{action}. +const ( + collectionSegs = 1 + itemSegs = 2 + actionSegs = 3 +) + +// isRead reports a GET or HEAD. +func isRead(r *http.Request) bool { + return r.Method == http.MethodGet || r.Method == http.MethodHead +} + +// collectionOps are the list/get/delete operations of one child collection. +type collectionOps struct { + list func(ctx context.Context, sub, rg, svc string) ([]apimanagement.ChildResource, error) + get func(ctx context.Context, sub, rg, svc, id string) (apimanagement.ChildResource, error) + del func(ctx context.Context, sub, rg, svc, id, ifMatch string) (bool, error) +} + +// serveCollection serves the apis and products collections: list, get and +// delete. Creating an API or a product is out of scope. +func (h *Handler) serveCollection(w http.ResponseWriter, r *http.Request, c *childScope) { + seg := strings.ToLower(c.segs[0]) + + ops := collectionOps{list: h.store.ListProducts, get: h.store.GetProduct, del: h.store.DeleteProduct} + if seg == segAPIs { + ops = collectionOps{list: h.store.ListAPIs, get: h.store.GetAPI, del: h.store.DeleteAPI} + } + + switch len(c.segs) { + case collectionSegs: + if r.Method != http.MethodGet { + methodNotAllowed(w) + return + } + + h.listCollection(w, r, c, seg, &ops) + case itemSegs: + h.serveCollectionItem(w, r, c, seg, &ops) + default: + unsupportedChild(w, strings.Join(c.segs, "/")) + } +} + +func (*Handler) listCollection(w http.ResponseWriter, r *http.Request, c *childScope, seg string, ops *collectionOps) { + items, err := ops.list(r.Context(), c.sub, c.rg, c.svc) + if err != nil { + writeErr(w, err) + return + } + + out := make([]childResponse, 0, len(items)) + for i := range items { + out = append(out, toChildResponse(c.serviceID, seg, &items[i])) + } + + writeList(w, r, out, true) +} + +func (*Handler) serveCollectionItem(w http.ResponseWriter, r *http.Request, c *childScope, seg string, ops *collectionOps) { + switch { + case isRead(r): + item, err := ops.get(r.Context(), c.sub, c.rg, c.svc, c.segs[1]) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, item.Etag, toChildResponse(c.serviceID, seg, &item)) + case r.Method == http.MethodDelete: + existed, err := ops.del(r.Context(), c.sub, c.rg, c.svc, c.segs[1], r.Header.Get("If-Match")) + if err != nil { + writeErr(w, err) + return + } + + // An API delete answers 204 either way: the 2022-08-01 armapimanagement + // client accepts only 202/204 and go-azure-sdk (azurerm) only 200/204. + if seg == segAPIs { + w.WriteHeader(http.StatusNoContent) + return + } + + writeDeleted(w, existed) + default: + methodNotAllowed(w) + } +} + +// servePolicy serves the service-level policy (policies/policy) and the +// policies list. +func (h *Handler) servePolicy(w http.ResponseWriter, r *http.Request, c *childScope) { + switch { + case len(c.segs) == collectionSegs && r.Method == http.MethodGet: + h.listPolicies(w, r, c) + case len(c.segs) != itemSegs || !strings.EqualFold(c.segs[1], policyName): + unsupportedChild(w, strings.Join(c.segs, "/")) + case isRead(r): + p, err := h.store.GetPolicy(r.Context(), c.sub, c.rg, c.svc) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, p.Etag, toChildResponse(c.serviceID, segPolicies, &p)) + case r.Method == http.MethodPut: + h.putPolicy(w, r, c) + case r.Method == http.MethodDelete: + existed, err := h.store.DeletePolicy(r.Context(), c.sub, c.rg, c.svc, r.Header.Get("If-Match")) + if err != nil { + writeErr(w, err) + return + } + + writeDeleted(w, existed) + default: + methodNotAllowed(w) + } +} + +// listPolicies lists the service's policies: the service policy when one is +// set. +func (h *Handler) listPolicies(w http.ResponseWriter, r *http.Request, c *childScope) { + out := []childResponse{} + + p, err := h.store.GetPolicy(r.Context(), c.sub, c.rg, c.svc) + if err == nil { + out = append(out, toChildResponse(c.serviceID, segPolicies, &p)) + } else if _, gerr := h.store.GetService(r.Context(), c.sub, c.rg, c.svc); gerr != nil { + writeErr(w, gerr) + return + } + + writeList(w, r, out, true) +} + +func (h *Handler) putPolicy(w http.ResponseWriter, r *http.Request, c *childScope) { + var req policyRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + p, created, err := h.store.PutPolicy(r.Context(), c.sub, c.rg, c.svc, + req.Properties.Value, req.Properties.Format, r.Header.Get("If-Match")) + if err != nil { + writeErr(w, err) + return + } + + status := http.StatusOK + if created { + status = http.StatusCreated + } + + writeChild(w, r, status, p.Etag, toChildResponse(c.serviceID, segPolicies, &p)) +} + +// servePortal serves portalsettings/{signin|signup|delegation} (get, put) and +// portalsettings/delegation/listSecrets. +func (h *Handler) servePortal(w http.ResponseWriter, r *http.Request, c *childScope) { + switch { + case len(c.segs) == actionSegs && strings.EqualFold(c.segs[1], apimanagement.PortalDelegation) && + strings.EqualFold(c.segs[2], actListSecrets) && r.Method == http.MethodPost: + h.delegationSecrets(w, r, c) + case len(c.segs) != itemSegs: + unsupportedChild(w, strings.Join(c.segs, "/")) + case isRead(r): + s, err := h.store.GetPortalSetting(r.Context(), c.sub, c.rg, c.svc, strings.ToLower(c.segs[1])) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, s.Etag, toChildResponse(c.serviceID, segPortal, &s)) + case r.Method == http.MethodPut: + h.putPortal(w, r, c) + default: + methodNotAllowed(w) + } +} + +func (h *Handler) delegationSecrets(w http.ResponseWriter, r *http.Request, c *childScope) { + key, err := h.store.DelegationValidationKey(r.Context(), c.sub, c.rg, c.svc) + if err != nil { + writeErr(w, err) + return + } + + azurearm.WriteJSON(w, http.StatusOK, map[string]string{"validationKey": key}) +} + +func (h *Handler) putPortal(w http.ResponseWriter, r *http.Request, c *childScope) { + var req childRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + s, err := h.store.PutPortalSetting(r.Context(), c.sub, c.rg, c.svc, + strings.ToLower(c.segs[1]), req.Properties, r.Header.Get("If-Match")) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, s.Etag, toChildResponse(c.serviceID, segPortal, &s)) +} + +// serveTenant serves tenant/{access|gitAccess} (get, patch) and its +// listSecrets action. +func (h *Handler) serveTenant(w http.ResponseWriter, r *http.Request, c *childScope) { + switch { + case len(c.segs) == actionSegs && strings.EqualFold(c.segs[2], actListSecrets) && r.Method == http.MethodPost: + t, err := h.store.GetTenantAccess(r.Context(), c.sub, c.rg, c.svc, c.segs[1]) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, t.Etag, tenantAccessSecrets{ + ID: t.Name, PrincipalID: t.PrincipalID, PrimaryKey: t.PrimaryKey, SecondaryKey: t.SecondaryKey, Enabled: t.Enabled, + }) + case len(c.segs) != itemSegs: + unsupportedChild(w, strings.Join(c.segs, "/")) + case isRead(r): + t, err := h.store.GetTenantAccess(r.Context(), c.sub, c.rg, c.svc, c.segs[1]) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, t.Etag, toTenantAccessResponse(c.serviceID, &t)) + case r.Method == http.MethodPatch: + h.patchTenant(w, r, c) + default: + methodNotAllowed(w) + } +} + +func (h *Handler) patchTenant(w http.ResponseWriter, r *http.Request, c *childScope) { + var req tenantAccessRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + t, err := h.store.UpdateTenantAccess(r.Context(), c.sub, c.rg, c.svc, c.segs[1], + req.Properties.Enabled, r.Header.Get("If-Match")) + if err != nil { + writeErr(w, err) + return + } + + writeChild(w, r, http.StatusOK, t.Etag, toTenantAccessResponse(c.serviceID, &t)) +} diff --git a/server/azure/apimanagement/handler.go b/server/azure/apimanagement/handler.go index 598fe92b8..88d67a2b9 100644 --- a/server/azure/apimanagement/handler.go +++ b/server/azure/apimanagement/handler.go @@ -1,6 +1,7 @@ -// Package apimanagement serves the Azure API Management service ARM API -// (Microsoft.ApiManagement/service). Real armapimanagement ServiceClient -// requests hit this handler the same way they hit management.azure.com. +// Package apimanagement serves the Azure API Management ARM API +// (Microsoft.ApiManagement). Real armapimanagement clients, the azurerm +// Terraform provider's go-azure-sdk clients and the az CLI hit this handler the +// same way they hit management.azure.com. // // Real Azure runs service create, update and delete as long-running operations // (a Developer-tier create takes 30-45 minutes). The emulator completes them @@ -11,16 +12,21 @@ // header, which azcore treats as a completed no-op poll. So PollUntilDone // returns on the first call. // -// The service's child resources (apis, products, subscriptions, policies, -// backends, ...), the gateway data plane, backup/restore and the soft-deleted -// services (deletedservices) surface are out of scope. +// Served: the service CRUD surface, the soft-deleted services +// (locations/{l}/deletedservices/{name} get and purge, deletedservices list), +// checkNameAvailability, and the child resources around a service's create, +// refresh and destroy (apis and products list/get/delete, policies/policy, +// portalsettings signin/signup/delegation, tenant access). The behavior lives +// in the provider; this package only maps the wire. package apimanagement import ( "context" + "errors" "net/http" "strings" + cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" "github.com/stackshy/cloudemu/v2/server/wire/azurearm" ) @@ -29,23 +35,34 @@ const ( providerName = "Microsoft.ApiManagement" serviceType = "service" serviceArmType = providerName + "/" + serviceType + + typeLocations = "locations" + typeDeletedServices = "deletedservices" + typeCheckName = "checkNameAvailability" ) -// Store is the minimal API Management backend the handler needs. -// *apimanagement.Mock satisfies it. +// Store is the API Management backend the handler needs. *apimanagement.Mock +// satisfies it. type Store interface { CreateOrUpdateService( ctx context.Context, sub, rg, name, location string, in *apimanagement.ServiceInput, ) (apimanagement.Service, bool, error) UpdateService(ctx context.Context, sub, rg, name string, in *apimanagement.ServiceInput) (apimanagement.Service, error) GetService(ctx context.Context, sub, rg, name string) (apimanagement.Service, error) - DeleteService(ctx context.Context, sub, rg, name string) (bool, error) + DeleteServiceIfMatch(ctx context.Context, sub, rg, name, ifMatch string) (bool, error) ListServicesByResourceGroup(ctx context.Context, sub, rg string) ([]apimanagement.Service, error) ListServicesBySubscription(ctx context.Context, sub string) ([]apimanagement.Service, error) PurgeResourceGroup(ctx context.Context, sub, rg string) error + CheckNameAvailability(ctx context.Context, name string) apimanagement.NameAvailability + + GetDeletedService(ctx context.Context, sub, location, name string) (apimanagement.DeletedService, error) + ListDeletedServices(ctx context.Context, sub string) ([]apimanagement.DeletedService, error) + PurgeDeletedService(ctx context.Context, sub, location, name string) (apimanagement.DeletedService, error) + + childStore } -// Handler serves Microsoft.ApiManagement/service ARM requests. +// Handler serves Microsoft.ApiManagement ARM requests. type Handler struct { store Store } @@ -55,16 +72,23 @@ func New(store Store) *Handler { return &Handler{store: store} } -// Matches reports whether r targets an API Management service ARM URL. The +// Matches reports whether r targets an API Management ARM URL: a service (and +// its children), a soft-deleted service, or checkNameAvailability. The // provider and type are matched case-insensitively. func (*Handler) Matches(r *http.Request) bool { rp, ok := azurearm.ParsePath(r.URL.Path) - if !ok { + if !ok || !strings.EqualFold(rp.Provider, providerName) { return false } - return strings.EqualFold(rp.Provider, providerName) && - strings.EqualFold(rp.ResourceType, serviceType) + switch strings.ToLower(rp.ResourceType) { + case strings.ToLower(serviceType), strings.ToLower(typeDeletedServices), strings.ToLower(typeCheckName): + return true + case typeLocations: + return strings.EqualFold(rp.SubResource, typeDeletedServices) + default: + return false + } } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -74,23 +98,66 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + switch strings.ToLower(rp.ResourceType) { + case strings.ToLower(typeCheckName): + h.checkNameAvailability(w, r) + case strings.ToLower(typeDeletedServices): + h.listDeletedServices(w, r, &rp) + case typeLocations: + h.serveDeletedService(w, r, &rp) + default: + h.serveServiceTree(w, r, &rp) + } +} + +// serveServiceTree routes .../service, .../service/{name} and the service's +// child resources. +func (h *Handler) serveServiceTree(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { switch { case rp.ResourceName == "": - h.listServices(w, r, &rp) + h.listServices(w, r, rp) case rp.SubResource == "": - h.serveService(w, r, &rp) + h.serveService(w, r, rp) default: - azurearm.WriteError(w, http.StatusNotFound, "InvalidResourceType", - "unsupported API Management sub-resource "+rp.SubResource) + h.serveChild(w, r, rp, childSegments(r.URL.Path)) } } -// PurgeResourceGroup deletes every service under sub/rg so a resource-group -// delete cascades into them. +// PurgeResourceGroup soft-deletes every service under sub/rg so a +// resource-group delete cascades into them. func (h *Handler) PurgeResourceGroup(ctx context.Context, subscription, resourceGroup string) error { return h.store.PurgeResourceGroup(ctx, subscription, resourceGroup) } +// writeErr maps a provider error onto APIM's ARM error codes: validation +// failures are 400 ValidationError, a stale If-Match is 412 PreconditionFailed, +// and the three service conflicts carry their specific 409 codes. +func writeErr(w http.ResponseWriter, err error) { + msg := cerrors.Message(err) + + switch { + case errors.Is(err, apimanagement.ErrLocationMismatch): + azurearm.WriteError(w, http.StatusConflict, "InvalidResourceLocation", msg) + case errors.Is(err, apimanagement.ErrNameNotAvailable): + azurearm.WriteError(w, http.StatusConflict, "ServiceAlreadyExists", msg) + case errors.Is(err, apimanagement.ErrSoftDeleted): + azurearm.WriteError(w, http.StatusConflict, "ServiceAlreadyExistsInSoftDeletedState", msg) + case errors.Is(err, apimanagement.ErrTierNotSupported): + azurearm.WriteError(w, http.StatusBadRequest, "MethodNotAllowedInPricingTier", msg) + case cerrors.IsInvalidArgument(err): + azurearm.WriteError(w, http.StatusBadRequest, "ValidationError", msg) + case cerrors.IsFailedPrecondition(err): + azurearm.WriteError(w, http.StatusPreconditionFailed, "PreconditionFailed", msg) + default: + azurearm.WriteCErr(w, err) + } +} + +// methodNotAllowed writes the ARM 405. +func methodNotAllowed(w http.ResponseWriter) { + azurearm.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") +} + // serveService routes the top-level service CRUD surface. func (h *Handler) serveService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { switch r.Method { @@ -103,7 +170,7 @@ func (h *Handler) serveService(w http.ResponseWriter, r *http.Request, rp *azure case http.MethodDelete: h.deleteService(w, r, rp) default: - azurearm.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + methodNotAllowed(w) } } @@ -118,12 +185,12 @@ func (h *Handler) createService(w http.ResponseWriter, r *http.Request, rp *azur return } - in := serviceInputFromRequest(&req) + in := serviceInputFromRequest(&req, r.Header.Get("If-Match")) s, created, err := h.store.CreateOrUpdateService( r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName, req.Location, &in) if err != nil { - azurearm.WriteCErr(w, err) + writeErr(w, err) return } @@ -144,11 +211,11 @@ func (h *Handler) updateService(w http.ResponseWriter, r *http.Request, rp *azur return } - in := serviceInputFromRequest(&req) + in := serviceInputFromRequest(&req, r.Header.Get("If-Match")) s, err := h.store.UpdateService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName, &in) if err != nil { - azurearm.WriteCErr(w, err) + writeErr(w, err) return } @@ -158,22 +225,28 @@ func (h *Handler) updateService(w http.ResponseWriter, r *http.Request, rp *azur func (h *Handler) getService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { s, err := h.store.GetService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName) if err != nil { - azurearm.WriteCErr(w, err) + writeErr(w, err) return } azurearm.WriteJSON(w, http.StatusOK, toServiceResponse(&s)) } -// deleteService is the idempotent ARM DELETE: 200 when the service existed, -// 204 when it did not. +// deleteService is the idempotent ARM DELETE (a soft delete): 200 when the +// service existed, 204 when it did not. func (h *Handler) deleteService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { - existed, err := h.store.DeleteService(r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName) + existed, err := h.store.DeleteServiceIfMatch( + r.Context(), rp.Subscription, rp.ResourceGroup, rp.ResourceName, r.Header.Get("If-Match")) if err != nil { - azurearm.WriteCErr(w, err) + writeErr(w, err) return } + writeDeleted(w, existed) +} + +// writeDeleted answers a DELETE: 200 when the resource existed, 204 when not. +func writeDeleted(w http.ResponseWriter, existed bool) { if existed { w.WriteHeader(http.StatusOK) return @@ -184,7 +257,7 @@ func (h *Handler) deleteService(w http.ResponseWriter, r *http.Request, rp *azur func (h *Handler) listServices(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { if r.Method != http.MethodGet { - azurearm.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + methodNotAllowed(w) return } @@ -200,14 +273,102 @@ func (h *Handler) listServices(w http.ResponseWriter, r *http.Request, rp *azure } if err != nil { - azurearm.WriteCErr(w, err) + writeErr(w, err) return } - out := serviceListResponse{Value: make([]serviceResponse, 0, len(items))} + out := make([]serviceResponse, 0, len(items)) for i := range items { - out.Value = append(out.Value, toServiceResponse(&items[i])) + out = append(out, toServiceResponse(&items[i])) + } + + writeList(w, r, out, false) +} + +// writeList pages a list with $skip/$top and a nextLink. withCount adds the +// APIM collection's total count. +func writeList[T any](w http.ResponseWriter, r *http.Request, items []T, withCount bool) { + page, next := azurearm.Paginate(r, items, azurearm.DefaultPageSize) + + out := listResponse[T]{Value: page, NextLink: next} + + if withCount { + n := len(items) + out.Count = &n } azurearm.WriteJSON(w, http.StatusOK, out) } + +// checkNameAvailability answers POST .../providers/Microsoft.ApiManagement/checkNameAvailability. +func (h *Handler) checkNameAvailability(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w) + return + } + + var req nameAvailabilityRequest + if !azurearm.DecodeJSON(w, r, &req) { + return + } + + v := h.store.CheckNameAvailability(r.Context(), req.Name) + azurearm.WriteJSON(w, http.StatusOK, nameAvailabilityResponse{ + NameAvailable: v.Available, Reason: v.Reason, Message: v.Message, + }) +} + +// serveDeletedService routes .../locations/{l}/deletedservices/{name}: GET +// reads the soft-deleted service, DELETE purges it; both 404 when nothing by +// that name is soft-deleted there. +func (h *Handler) serveDeletedService(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + if rp.SubResourceName == "" || rp.SubResourceAction != "" { + azurearm.WriteError(w, http.StatusNotFound, "InvalidResourceType", + "unsupported API Management deleted-services path") + + return + } + + var ( + d apimanagement.DeletedService + err error + ) + + switch r.Method { + case http.MethodGet: + d, err = h.store.GetDeletedService(r.Context(), rp.Subscription, rp.ResourceName, rp.SubResourceName) + case http.MethodDelete: + d, err = h.store.PurgeDeletedService(r.Context(), rp.Subscription, rp.ResourceName, rp.SubResourceName) + default: + methodNotAllowed(w) + return + } + + if err != nil { + writeErr(w, err) + return + } + + azurearm.WriteJSON(w, http.StatusOK, toDeletedServiceResponse(&d)) +} + +// listDeletedServices answers GET .../providers/Microsoft.ApiManagement/deletedservices. +func (h *Handler) listDeletedServices(w http.ResponseWriter, r *http.Request, rp *azurearm.ResourcePath) { + if r.Method != http.MethodGet || rp.ResourceName != "" { + methodNotAllowed(w) + return + } + + items, err := h.store.ListDeletedServices(r.Context(), rp.Subscription) + if err != nil { + writeErr(w, err) + return + } + + out := make([]deletedServiceResponse, 0, len(items)) + for i := range items { + out = append(out, toDeletedServiceResponse(&items[i])) + } + + writeList(w, r, out, false) +} diff --git a/server/azure/apimanagement/review_fixes_sdk_test.go b/server/azure/apimanagement/review_fixes_sdk_test.go new file mode 100644 index 000000000..d588acf0e --- /dev/null +++ b/server/azure/apimanagement/review_fixes_sdk_test.go @@ -0,0 +1,556 @@ +package apimanagement_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +const apiVersion = "?api-version=2022-08-01" + +// serviceURL is the ARM URL of a service in the fixture subscription. +func (f *fixture) serviceURL(rg, name string) string { + return f.ts.URL + "/subscriptions/" + subID + "/resourceGroups/" + rg + + "/providers/Microsoft.ApiManagement/service/" + name +} + +// do sends a raw ARM request and returns the status, body and response headers. +func (f *fixture) do(t *testing.T, method, url, body string, hdr map[string]string) (int, string, http.Header) { + t.Helper() + + req, err := http.NewRequestWithContext(context.Background(), method, url, strings.NewReader(body)) + if err != nil { + t.Fatalf("new request: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + + for k, v := range hdr { + req.Header.Set(k, v) + } + + resp, err := f.ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer resp.Body.Close() + + raw, _ := io.ReadAll(resp.Body) + + return resp.StatusCode, string(raw), resp.Header +} + +// TestSDKEtagRotatesOnEveryWrite: PUT, PATCH and a replacing PUT each return a +// new etag, a read does not change it, and a stale If-Match is 412 on PUT, +// PATCH and DELETE while the current etag succeeds. On b0ebd3d3 every write +// returned the same etag and If-Match was ignored. +func TestSDKEtagRotatesOnEveryWrite(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + created := f.create(t, rgName, svcName, developerService()) + patched := patchService(t, f) + replaced := f.create2(t, rgName, svcName, developerService()) + + etags := map[string]bool{deref(created.Etag): true, deref(patched.Etag): true, deref(replaced.Etag): true} + if len(etags) != 3 { + t.Fatalf("PUT/PATCH/PUT must each return a new etag, got %q %q %q", + deref(created.Etag), deref(patched.Etag), deref(replaced.Etag)) + } + + got, err := f.svc.Get(ctx, rgName, svcName, nil) + if err != nil || deref(got.Etag) != deref(replaced.Etag) { + t.Fatalf("a read must not change the etag: %v %q", err, deref(got.Etag)) + } + + stale := map[string]string{"If-Match": `"` + deref(created.Etag) + `"`} + url := f.serviceURL(rgName, svcName) + apiVersion + + if code, body, _ := f.do(t, http.MethodPatch, url, `{"tags":{"a":"b"}}`, stale); code != http.StatusPreconditionFailed { + t.Errorf("PATCH with a stale If-Match = %d %s, want 412", code, body) + } + + putBody, _ := json.Marshal(developerService()) + if code, body, _ := f.do(t, http.MethodPut, url, string(putBody), stale); code != http.StatusPreconditionFailed { + t.Errorf("PUT with a stale If-Match = %d %s, want 412", code, body) + } + + if code, body, _ := f.do(t, http.MethodDelete, url, "", stale); code != http.StatusPreconditionFailed { + t.Errorf("DELETE with a stale If-Match = %d %s, want 412", code, body) + } + + current := map[string]string{"If-Match": deref(replaced.Etag)} + if code, body, _ := f.do(t, http.MethodPatch, url, `{"tags":{"a":"b"}}`, current); code != http.StatusOK { + t.Errorf("PATCH with the current If-Match = %d %s, want 200", code, body) + } +} + +// create2 is create for a service that already exists (a replace). +func (f *fixture) create2( + t *testing.T, rg, name string, body armapimanagement.ServiceResource, +) armapimanagement.ServiceResource { + t.Helper() + + return f.create(t, rg, name, body) +} + +// TestSDKCapacityCeilingsAndZones: each tier's unit ceiling is enforced (on +// create and on PATCH) and availability zones are Premium-only. On b0ebd3d3 +// Developer with capacity 5 and zones on Developer were both accepted. +func TestSDKCapacityCeilingsAndZones(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + ceilings := map[armapimanagement.SKUType]int32{ + armapimanagement.SKUTypeDeveloper: 1, + armapimanagement.SKUTypeBasic: 2, + armapimanagement.SKUTypeStandard: 4, + armapimanagement.SKUTypePremium: 12, + armapimanagement.SKUTypeBasicV2: 10, + armapimanagement.SKUType("StandardV2"): 10, + } + + for sku, maxUnits := range ceilings { + name := "cap-" + strings.ToLower(string(sku)) + + body := developerService() + body.SKU = &armapimanagement.ServiceSKUProperties{Name: to.Ptr(sku), Capacity: to.Ptr(maxUnits + 1)} + + _, err := f.svc.BeginCreateOrUpdate(ctx, rgName, name, body, nil) + assertStatus(t, err, http.StatusBadRequest, "ValidationError") + + body.SKU.Capacity = to.Ptr(maxUnits) + f.create(t, rgName, name, body) + } + + _, err := f.svc.BeginUpdate(ctx, rgName, "cap-developer", armapimanagement.ServiceUpdateParameters{ + SKU: &armapimanagement.ServiceSKUProperties{Name: to.Ptr(armapimanagement.SKUTypeDeveloper), Capacity: to.Ptr[int32](5)}, + }, nil) + assertStatus(t, err, http.StatusBadRequest, "ValidationError") + + zonal := developerService() + zonal.Zones = []*string{to.Ptr("1"), to.Ptr("2")} + + _, err = f.svc.BeginCreateOrUpdate(ctx, rgName, "zones-dev", zonal, nil) + assertStatus(t, err, http.StatusBadRequest, "ValidationError") + + zonal.SKU = &armapimanagement.ServiceSKUProperties{ + Name: to.Ptr(armapimanagement.SKUTypePremium), Capacity: to.Ptr[int32](2), + } + if s := f.create(t, rgName, "zones-premium", zonal); len(s.Zones) != 2 { + t.Errorf("Premium zones = %v, want 2", s.Zones) + } +} + +// TestSDKServiceNameIsGlobal: the service name is a global *.azure-api.net +// label, so a second service of that name in another group or another +// subscription is 409, and checkNameAvailability reports it. On b0ebd3d3 both +// creates succeeded with the same gatewayUrl and checkNameAvailability was 501. +func TestSDKServiceNameIsGlobal(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + f.create(t, rgName, "svc1", developerService()) + + _, err := f.svc.BeginCreateOrUpdate(ctx, rgOther, "svc1", developerService(), nil) + assertStatus(t, err, http.StatusConflict, "ServiceAlreadyExists") + + _, err = f.svc.BeginCreateOrUpdate(ctx, rgOther, "SVC1", developerService(), nil) + assertStatus(t, err, http.StatusConflict, "ServiceAlreadyExists") + + otherSub := "00000000-0000-0000-0000-0000000000b2" + + cf, err := armapimanagement.NewClientFactory(otherSub, fakeCred{}, f.opts) + if err != nil { + t.Fatalf("client factory: %v", err) + } + + rgs, err := armresources.NewResourceGroupsClient(otherSub, fakeCred{}, f.opts) + if err != nil { + t.Fatalf("rg client: %v", err) + } + + if _, err := rgs.CreateOrUpdate(ctx, "rg-b", armresources.ResourceGroup{Location: to.Ptr("eastus")}, nil); err != nil { + t.Fatalf("create rg in other subscription: %v", err) + } + + _, err = cf.NewServiceClient().BeginCreateOrUpdate(ctx, "rg-b", "svc1", developerService(), nil) + assertStatus(t, err, http.StatusConflict, "ServiceAlreadyExists") + + check := func(name string) armapimanagement.ServiceNameAvailabilityResult { + res, err := f.svc.CheckNameAvailability(ctx, + armapimanagement.ServiceCheckNameAvailabilityParameters{Name: to.Ptr(name)}, nil) + if err != nil { + t.Fatalf("checkNameAvailability %s: %v", name, err) + } + + return res.ServiceNameAvailabilityResult + } + + if r := check("svc1"); deref(r.NameAvailable) || deref(r.Reason) != armapimanagement.NameAvailabilityReasonAlreadyExists { + t.Errorf("svc1 availability = %v/%v, want false/AlreadyExists", deref(r.NameAvailable), deref(r.Reason)) + } + + if r := check("free-name"); !deref(r.NameAvailable) || deref(r.Reason) != armapimanagement.NameAvailabilityReasonValid { + t.Errorf("free-name availability = %v/%v, want true/Valid", deref(r.NameAvailable), deref(r.Reason)) + } + + if r := check("1bad"); deref(r.NameAvailable) || deref(r.Reason) != armapimanagement.NameAvailabilityReasonInvalid { + t.Errorf("1bad availability = %v/%v, want false/Invalid", deref(r.NameAvailable), deref(r.Reason)) + } + + // A soft-deleted service still holds its name until it is purged. + if _, err := f.svc.BeginDelete(ctx, rgName, "svc1", nil); err != nil { + t.Fatalf("delete svc1: %v", err) + } + + if r := check("svc1"); deref(r.NameAvailable) { + t.Error("a soft-deleted service must still hold its name") + } + + _, err = f.svc.BeginCreateOrUpdate(ctx, rgOther, "svc1", developerService(), nil) + assertStatus(t, err, http.StatusConflict, "ServiceAlreadyExistsInSoftDeletedState") +} + +// TestSDKLocationIsImmutable: a PUT naming another location on an existing +// service is 409 InvalidResourceLocation and leaves it unchanged; the same +// location in another spelling is accepted. On b0ebd3d3 the PUT answered 200 and +// silently kept eastus. +func TestSDKLocationIsImmutable(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + f.create(t, rgName, svcName, developerService()) + + moved := developerService() + moved.Location = to.Ptr("West Europe") + + _, err := f.svc.BeginCreateOrUpdate(ctx, rgName, svcName, moved, nil) + assertStatus(t, err, http.StatusConflict, "InvalidResourceLocation") + + same := developerService() + same.Location = to.Ptr("eastus") + f.create(t, rgName, svcName, same) + + got, err := f.svc.Get(ctx, rgName, svcName, nil) + if err != nil || deref(got.Location) != "East US" { + t.Fatalf("location = %q (%v), want East US", deref(got.Location), err) + } +} + +// TestLibraryAndServerReturnSameResource: the Go library's service carries the +// same properties block as the HTTP GET (the defaults and computed fields are +// the provider's). On b0ebd3d3 the library returned only the caller's +// properties and the server added virtualNetworkType, platformVersion, the +// regional gateway URL and the rest. +func TestLibraryAndServerReturnSameResource(t *testing.T) { + f := newFixture(t) + f.create(t, rgName, svcName, developerService()) + + lib, err := f.prov.APIManagement.GetService(context.Background(), subID, rgName, svcName) + if err != nil { + t.Fatalf("library get: %v", err) + } + + code, body, _ := f.do(t, http.MethodGet, f.serviceURL(rgName, svcName)+apiVersion, "", nil) + if code != http.StatusOK { + t.Fatalf("GET = %d %s", code, body) + } + + var wire struct { + Etag string `json:"etag"` + Properties map[string]any `json:"properties"` + } + if err := json.Unmarshal([]byte(body), &wire); err != nil { + t.Fatalf("decode: %v", err) + } + + var libProps map[string]any + if err := json.Unmarshal(lib.Properties, &libProps); err != nil { + t.Fatalf("decode library properties: %v", err) + } + + for _, k := range []string{ + "virtualNetworkType", "publicNetworkAccess", "notificationSenderEmail", "platformVersion", + "gatewayRegionalUrl", "gatewayUrl", "developerPortalUrl", "provisioningState", "createdAtUtc", + } { + if libProps[k] == nil || libProps[k] != wire.Properties[k] { + t.Errorf("property %s: library %v, server %v", k, libProps[k], wire.Properties[k]) + } + } + + if lib.Etag != wire.Etag { + t.Errorf("etag: library %q, server %q", lib.Etag, wire.Etag) + } + + if got := libProps["gatewayRegionalUrl"]; got != "https://contoso-apim-eastus-01.regional.azure-api.net" { + t.Errorf("gatewayRegionalUrl = %v", got) + } +} + +// TestResourceGraphTopLevelSKU: the Resource Graph row carries a top-level +// sku{name,capacity} and zones, not properties.sku. On b0ebd3d3 sku and +// skuCapacity sat in properties as strings/numbers and there was no sku block. +func TestResourceGraphTopLevelSKU(t *testing.T) { + f := newFixture(t) + + body := developerService() + body.SKU = &armapimanagement.ServiceSKUProperties{Name: to.Ptr(armapimanagement.SKUTypePremium), Capacity: to.Ptr[int32](3)} + body.Zones = []*string{to.Ptr("1")} + f.create(t, rgName, svcName, body) + + q := `{"subscriptions":["` + subID + `"],"query":"Resources | where type =~ 'microsoft.apimanagement/service'"}` + + code, raw, _ := f.do(t, http.MethodPost, + f.ts.URL+"/providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01", q, nil) + if code != http.StatusOK { + t.Fatalf("resource graph = %d %s", code, raw) + } + + var res struct { + Data []struct { + Sku map[string]any `json:"sku"` + Zones []string `json:"zones"` + Properties map[string]any `json:"properties"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(raw), &res); err != nil || len(res.Data) != 1 { + t.Fatalf("decode %s: %v", raw, err) + } + + row := res.Data[0] + if row.Sku["name"] != "Premium" || row.Sku["capacity"] != float64(3) { + t.Errorf("sku = %v, want {Premium 3}", row.Sku) + } + + if len(row.Zones) != 1 || row.Zones[0] != "1" { + t.Errorf("zones = %v", row.Zones) + } + + if _, ok := row.Properties["sku"]; ok { + t.Errorf("sku must not be a property: %v", row.Properties) + } +} + +// TestSDKListsPageWithNextLink: a list honours $top and emits a nextLink that +// the official pager follows to the end. On b0ebd3d3 there was no nextLink. +func TestSDKListsPageWithNextLink(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + for _, n := range []string{"page-a", "page-b", "page-c"} { + f.create(t, rgName, n, developerService()) + } + + code, body, _ := f.do(t, http.MethodGet, f.ts.URL+"/subscriptions/"+subID+"/resourceGroups/"+rgName+ + "/providers/Microsoft.ApiManagement/service"+apiVersion+"&$top=2", "", nil) + + var page struct { + Value []json.RawMessage `json:"value"` + NextLink string `json:"nextLink"` + } + if err := json.Unmarshal([]byte(body), &page); err != nil || code != http.StatusOK { + t.Fatalf("list = %d %s", code, body) + } + + if len(page.Value) != 2 || !strings.Contains(page.NextLink, "%24skip=2") { + t.Fatalf("first page = %d items, nextLink %q", len(page.Value), page.NextLink) + } + + var names []string + + pager := f.cf.NewProductClient().NewListByServicePager(rgName, "page-a", + &armapimanagement.ProductClientListByServiceOptions{Top: to.Ptr[int32](1)}) + + pages := 0 + for pager.More() { + p, err := pager.NextPage(ctx) + if err != nil { + t.Fatalf("products page: %v", err) + } + + pages++ + + for _, v := range p.Value { + names = append(names, deref(v.Name)) + } + } + + if pages != 2 || strings.Join(names, ",") != "starter,unlimited" { + t.Errorf("paged products = %v over %d pages, want starter,unlimited over 2", names, pages) + } +} + +// TestSDKPolicyLifecycle covers the service-level policy: PUT (201 then 200), +// GET, a stale If-Match (412), malformed XML (400 ValidationError) and DELETE. +func TestSDKPolicyLifecycle(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + f.create(t, rgName, svcName, developerService()) + + pc := f.cf.NewPolicyClient() + doc := `` + + put := func(value string, opts *armapimanagement.PolicyClientCreateOrUpdateOptions) (armapimanagement.PolicyClientCreateOrUpdateResponse, error) { + return pc.CreateOrUpdate(ctx, rgName, svcName, armapimanagement.PolicyIDNamePolicy, armapimanagement.PolicyContract{ + Properties: &armapimanagement.PolicyContractProperties{ + Value: to.Ptr(value), Format: to.Ptr(armapimanagement.PolicyContentFormatXML), + }, + }, opts) + } + + first, err := put(doc, nil) + if err != nil || first.ETag == nil { + t.Fatalf("PUT policy: %v", err) + } + + second, err := put(doc, &armapimanagement.PolicyClientCreateOrUpdateOptions{IfMatch: first.ETag}) + if err != nil || deref(second.ETag) == deref(first.ETag) { + t.Fatalf("PUT policy with current If-Match: %v (etag %q -> %q)", err, deref(first.ETag), deref(second.ETag)) + } + + _, err = put(doc, &armapimanagement.PolicyClientCreateOrUpdateOptions{IfMatch: first.ETag}) + assertStatus(t, err, http.StatusPreconditionFailed, "PreconditionFailed") + + _, err = put("", nil) + assertStatus(t, err, http.StatusBadRequest, "ValidationError") + + got, err := pc.Get(ctx, rgName, svcName, armapimanagement.PolicyIDNamePolicy, nil) + if err != nil || deref(got.Properties.Value) != doc { + t.Fatalf("GET policy = %v, %v", got.Properties, err) + } + + _, err = pc.Delete(ctx, rgName, svcName, armapimanagement.PolicyIDNamePolicy, "\"stale\"", nil) + assertStatus(t, err, http.StatusPreconditionFailed, "PreconditionFailed") + + if _, err := pc.Delete(ctx, rgName, svcName, armapimanagement.PolicyIDNamePolicy, "*", nil); err != nil { + t.Fatalf("DELETE policy: %v", err) + } + + _, err = pc.Get(ctx, rgName, svcName, armapimanagement.PolicyIDNamePolicy, nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") +} + +// TestSDKTenantAccessAndPortal covers tenant access GET/PATCH (keys only from +// listSecrets), the delegation key staying out of GET, and the tiers without a +// developer portal refusing both. +func TestSDKTenantAccessAndPortal(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + f.create(t, rgName, svcName, developerService()) + + ta := f.cf.NewTenantAccessClient() + + got, err := ta.Get(ctx, rgName, svcName, armapimanagement.AccessIDNameAccess, nil) + if err != nil || deref(got.Properties.Enabled) { + t.Fatalf("GET tenant access = %+v, %v", got.Properties, err) + } + + upd, err := ta.Update(ctx, rgName, svcName, armapimanagement.AccessIDNameAccess, "*", + armapimanagement.AccessInformationUpdateParameters{ + Properties: &armapimanagement.AccessInformationUpdateParameterProperties{Enabled: to.Ptr(true)}, + }, nil) + if err != nil || !deref(upd.Properties.Enabled) { + t.Fatalf("PATCH tenant access = %+v, %v", upd.Properties, err) + } + + secrets, err := ta.ListSecrets(ctx, rgName, svcName, armapimanagement.AccessIDNameAccess, nil) + if err != nil || !deref(secrets.Enabled) { + t.Fatalf("listSecrets = %+v, %v", secrets, err) + } + + dc := f.cf.NewDelegationSettingsClient() + if _, err := dc.CreateOrUpdate(ctx, rgName, svcName, armapimanagement.PortalDelegationSettings{ + Properties: &armapimanagement.PortalDelegationSettingsProperties{ + URL: to.Ptr("https://delegate.test"), ValidationKey: to.Ptr("c2VjcmV0"), + }, + }, nil); err != nil { + t.Fatalf("PUT delegation: %v", err) + } + + code, body, _ := f.do(t, http.MethodGet, f.serviceURL(rgName, svcName)+"/portalsettings/delegation"+apiVersion, "", nil) + if code != http.StatusOK || strings.Contains(body, "c2VjcmV0") { + t.Errorf("GET delegation must not reveal the validation key: %d %s", code, body) + } + + key, err := dc.ListSecrets(ctx, rgName, svcName, nil) + if err != nil || deref(key.ValidationKey) != "c2VjcmV0" { + t.Errorf("delegation listSecrets = %q, %v", deref(key.ValidationKey), err) + } + + consumption := developerService() + consumption.SKU = &armapimanagement.ServiceSKUProperties{ + Name: to.Ptr(armapimanagement.SKUTypeConsumption), Capacity: to.Ptr[int32](0), + } + f.create(t, rgName, "serverless", consumption) + + _, err = f.cf.NewSignInSettingsClient().Get(ctx, rgName, "serverless", nil) + assertStatus(t, err, http.StatusBadRequest, "MethodNotAllowedInPricingTier") + + _, err = ta.Get(ctx, rgName, "serverless", armapimanagement.AccessIDNameAccess, nil) + assertStatus(t, err, http.StatusBadRequest, "MethodNotAllowedInPricingTier") + + apis := f.cf.NewAPIClient().NewListByServicePager(rgName, "serverless", nil) + + p, err := apis.NextPage(ctx) + if err != nil || len(p.Value) != 0 { + t.Errorf("a Consumption service has no sample API: %d, %v", len(p.Value), err) + } +} + +// TestSDKRestoreSoftDeleted: a PUT with properties.restore recovers a +// soft-deleted service (ignoring the rest of the body), and a resource-group +// delete soft-deletes its services too. +func TestSDKRestoreSoftDeleted(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + + orig := f.create(t, rgName, svcName, developerService()) + + if _, err := f.svc.BeginDelete(ctx, rgName, svcName, nil); err != nil { + t.Fatalf("delete: %v", err) + } + + list := f.cf.NewDeletedServicesClient().NewListBySubscriptionPager(nil) + + page, err := list.NextPage(ctx) + if err != nil || len(page.Value) != 1 || deref(page.Value[0].Name) != svcName { + t.Fatalf("deleted services list = %v, %v", page.Value, err) + } + + restore := armapimanagement.ServiceResource{ + Location: to.Ptr("eastus"), + SKU: &armapimanagement.ServiceSKUProperties{Name: to.Ptr(armapimanagement.SKUTypeDeveloper), Capacity: to.Ptr[int32](1)}, + Properties: &armapimanagement.ServiceProperties{Restore: to.Ptr(true), PublisherEmail: to.Ptr(""), PublisherName: to.Ptr("")}, + } + + got := f.create(t, rgName, svcName, restore) + if deref(got.Properties.PublisherName) != "Contoso" || deref(got.Tags["env"]) != "dev" || + !got.Properties.CreatedAtUTC.Equal(*orig.Properties.CreatedAtUTC) { + t.Errorf("restore must bring the service back as it was: %+v", got.Properties) + } + + if _, err := f.cf.NewDeletedServicesClient().GetByName(ctx, svcName, "eastus", nil); err == nil { + t.Error("a recovered service must leave the deleted list") + } + + poller, err := f.rgs.BeginDelete(ctx, rgName, nil) + if err != nil { + t.Fatalf("delete rg: %v", err) + } + + if _, err := poller.PollUntilDone(ctx, pollFast); err != nil { + t.Fatalf("delete rg poll: %v", err) + } + + if _, err := f.cf.NewDeletedServicesClient().GetByName(ctx, svcName, "eastus", nil); err != nil { + t.Errorf("a resource-group delete must soft-delete its services: %v", err) + } +} diff --git a/server/azure/apimanagement/terraform_flow_sdk_test.go b/server/azure/apimanagement/terraform_flow_sdk_test.go new file mode 100644 index 000000000..00cdc63bd --- /dev/null +++ b/server/azure/apimanagement/terraform_flow_sdk_test.go @@ -0,0 +1,276 @@ +package apimanagement_test + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3" +) + +// pollFast is the poll frequency for the synchronous emulator LROs. +// +//nolint:gochecknoglobals // shared poll option +var pollFast = &runtime.PollUntilDoneOptions{Frequency: time.Millisecond} + +// TestSDKTerraformCreateReadDestroy replays, through the official clients, the +// request sequence terraform-provider-azurerm v4 makes for azurerm_api_management +// (internal/services/apimanagement/api_management_resource.go at 3cfd078, with +// the provider's default features: recover_soft_deleted and +// purge_soft_delete_on_destroy both true): +// +// create: GET service (404) -> GET deletedservices (404) -> PUT service -> +// list+delete apis -> list+delete products -> PUT signin -> PUT signup +// read: GET service -> GET policies/policy?format=xml (404 tolerated) -> +// GET signin, signup, delegation -> POST delegation/listSecrets -> +// POST tenant/access/listSecrets +// destroy: GET service -> DELETE service -> GET deletedservices (must be 200) +// -> DELETE deletedservices (purge) +// +// On b0ebd3d3 the first deletedservices GET answered 501, and every child call +// answered 404 InvalidResourceType. +func TestSDKTerraformCreateReadDestroy(t *testing.T) { + f := newFixture(t) + ctx := context.Background() + name := "tf-apim" + + // create + _, err := f.svc.Get(ctx, rgName, name, nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") + + deleted := f.cf.NewDeletedServicesClient() + _, err = deleted.GetByName(ctx, name, "eastus", nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") + + f.create(t, rgName, name, developerService()) + f.terraformPruneSamples(t, name) + f.terraformPortalWrites(t, name) + + // read (twice: apply's read and the no-op plan's refresh must agree) + first := f.terraformRead(t, name) + if again := f.terraformRead(t, name); again != first { + t.Errorf("refresh drifted:\n first %+v\n again %+v", first, again) + } + + // destroy + f.terraformDestroy(t, name) + + // the purged name is free again + f.create(t, rgName, name, developerService()) +} + +// terraformPruneSamples lists and deletes the sample Echo API and the Starter +// and Unlimited products Azure provisions with a new service. +func (f *fixture) terraformPruneSamples(t *testing.T, name string) { + t.Helper() + + ctx := context.Background() + apis := f.cf.NewAPIClient() + + var apiIDs []string + + for pager := apis.NewListByServicePager(rgName, name, nil); pager.More(); { + page, err := pager.NextPage(ctx) + if err != nil { + t.Fatalf("list apis: %v", err) + } + + for _, a := range page.Value { + apiIDs = append(apiIDs, deref(a.Name)) + } + } + + if len(apiIDs) != 1 || apiIDs[0] != "echo-api" { + t.Fatalf("a new Developer service must carry the sample echo-api, got %v", apiIDs) + } + + for _, id := range apiIDs { + poller, err := apis.BeginDelete(ctx, rgName, name, id, "*", + &armapimanagement.APIClientBeginDeleteOptions{DeleteRevisions: to.Ptr(true)}) + if err != nil { + t.Fatalf("delete api %s: %v", id, err) + } + + if _, err := poller.PollUntilDone(ctx, pollFast); err != nil { + t.Fatalf("delete api %s poll: %v", id, err) + } + } + + products := f.cf.NewProductClient() + + var productIDs []string + + for pager := products.NewListByServicePager(rgName, name, nil); pager.More(); { + page, err := pager.NextPage(ctx) + if err != nil { + t.Fatalf("list products: %v", err) + } + + for _, p := range page.Value { + productIDs = append(productIDs, deref(p.Name)) + } + } + + if len(productIDs) != 2 || productIDs[0] != "starter" || productIDs[1] != "unlimited" { + t.Fatalf("a new Developer service must carry starter+unlimited, got %v", productIDs) + } + + for _, id := range productIDs { + if _, err := products.Delete(ctx, rgName, name, id, "*", + &armapimanagement.ProductClientDeleteOptions{DeleteSubscriptions: to.Ptr(true)}); err != nil { + t.Fatalf("delete product %s: %v", id, err) + } + } + + if _, err := apis.Get(ctx, rgName, name, "echo-api", nil); err == nil { + t.Error("echo-api still readable after delete") + } +} + +// terraformPortalWrites is the sign_in / sign_up PUT azurerm always sends for a +// tier with a developer portal (the expanded defaults: both disabled). +func (f *fixture) terraformPortalWrites(t *testing.T, name string) { + t.Helper() + + ctx := context.Background() + + _, err := f.cf.NewSignInSettingsClient().CreateOrUpdate(ctx, rgName, name, armapimanagement.PortalSigninSettings{ + Properties: &armapimanagement.PortalSigninSettingProperties{Enabled: to.Ptr(false)}, + }, nil) + if err != nil { + t.Fatalf("PUT signin: %v", err) + } + + _, err = f.cf.NewSignUpSettingsClient().CreateOrUpdate(ctx, rgName, name, armapimanagement.PortalSignupSettings{ + Properties: &armapimanagement.PortalSignupSettingsProperties{ + Enabled: to.Ptr(false), + TermsOfService: &armapimanagement.TermsOfServiceProperties{ + ConsentRequired: to.Ptr(false), Enabled: to.Ptr(false), Text: to.Ptr(""), + }, + }, + }, nil) + if err != nil { + t.Fatalf("PUT signup: %v", err) + } +} + +// tfState is the part of azurerm's read that depends on the child calls. +type tfState struct { + etag, gateway, regional, portal string + signIn, signUp, tosEnabled, delegation bool + validationKey, tenantID, primary, secnd string +} + +// terraformRead performs azurerm's read calls and flattens what it stores. +func (f *fixture) terraformRead(t *testing.T, name string) tfState { + t.Helper() + + ctx := context.Background() + + svc, err := f.svc.Get(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("GET service: %v", err) + } + + if svc.Properties.CustomProperties == nil { + t.Fatal("customProperties must be present: azurerm dereferences it on read") + } + + _, err = f.cf.NewPolicyClient().Get(ctx, rgName, name, armapimanagement.PolicyIDNamePolicy, + &armapimanagement.PolicyClientGetOptions{Format: to.Ptr(armapimanagement.PolicyExportFormatXML)}) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") + + signIn, err := f.cf.NewSignInSettingsClient().Get(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("GET signin: %v", err) + } + + signUp, err := f.cf.NewSignUpSettingsClient().Get(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("GET signup: %v", err) + } + + delegation := f.cf.NewDelegationSettingsClient() + + del, err := delegation.Get(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("GET delegation: %v", err) + } + + key, err := delegation.ListSecrets(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("delegation listSecrets: %v", err) + } + + secrets, err := f.cf.NewTenantAccessClient().ListSecrets(ctx, rgName, name, armapimanagement.AccessIDNameAccess, nil) + if err != nil { + t.Fatalf("tenant access listSecrets: %v", err) + } + + if deref(secrets.PrimaryKey) == "" || deref(secrets.SecondaryKey) == "" || deref(secrets.ID) != "access" { + t.Errorf("tenant access secrets incomplete: %+v", secrets.AccessInformationSecretsContract) + } + + return tfState{ + etag: deref(svc.Etag), + gateway: deref(svc.Properties.GatewayURL), + regional: deref(svc.Properties.GatewayRegionalURL), + portal: deref(svc.Properties.DeveloperPortalURL), + signIn: deref(signIn.Properties.Enabled), + signUp: deref(signUp.Properties.Enabled), + tosEnabled: deref(signUp.Properties.TermsOfService.Enabled), + delegation: deref(del.Properties.Subscriptions.Enabled), + validationKey: deref(key.ValidationKey), + tenantID: deref(secrets.ID), + primary: deref(secrets.PrimaryKey), + secnd: deref(secrets.SecondaryKey), + } +} + +// terraformDestroy deletes the service and purges the soft-deleted record. +func (f *fixture) terraformDestroy(t *testing.T, name string) { + t.Helper() + + ctx := context.Background() + + existing, err := f.svc.Get(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("GET before delete: %v", err) + } + + poller, err := f.svc.BeginDelete(ctx, rgName, name, nil) + if err != nil { + t.Fatalf("delete service: %v", err) + } + + if _, err := poller.PollUntilDone(ctx, pollFast); err != nil { + t.Fatalf("delete service poll: %v", err) + } + + deleted := f.cf.NewDeletedServicesClient() + + got, err := deleted.GetByName(ctx, name, "eastus", nil) + if err != nil { + t.Fatalf("the deleted service must be soft-deleted and readable before purge: %v", err) + } + + if deref(got.Properties.ServiceID) != deref(existing.ID) || got.Properties.ScheduledPurgeDate == nil || + !got.Properties.ScheduledPurgeDate.After(*got.Properties.DeletionDate) { + t.Errorf("deleted service record = %+v", got.Properties) + } + + purge, err := deleted.BeginPurge(ctx, name, "eastus", nil) + if err != nil { + t.Fatalf("purge: %v", err) + } + + if _, err := purge.PollUntilDone(ctx, pollFast); err != nil { + t.Fatalf("purge poll: %v", err) + } + + _, err = deleted.GetByName(ctx, name, "eastus", nil) + assertStatus(t, err, http.StatusNotFound, "ResourceNotFound") +} diff --git a/server/azure/apimanagement/types.go b/server/azure/apimanagement/types.go index 13fb25e72..cf13c75a4 100644 --- a/server/azure/apimanagement/types.go +++ b/server/azure/apimanagement/types.go @@ -9,21 +9,6 @@ import ( "github.com/stackshy/cloudemu/v2/providers/azure/apimanagement" ) -const ( - // skuConsumption is the serverless tier, which has no dedicated portal, - // management or SCM endpoints and runs on the multi-tenant platform. - skuConsumption = "Consumption" - - // platformDedicated / platformConsumption are the computePlatform versions - // Azure reports for dedicated tiers and for the Consumption tier. - platformDedicated = "stv2" - platformConsumption = "mtv1" - - // notificationSenderDefault is the sender address Azure assigns when the - // caller sets none. - notificationSenderDefault = "apimgmt-noreply@mail.windowsazure.com" -) - // serviceRequest is the ARM service PUT/PATCH body. location, tags, zones, sku // and identity are top-level; the writable service properties live under // properties and round-trip verbatim. @@ -73,16 +58,98 @@ type serviceResponse struct { Properties json.RawMessage `json:"properties"` } -// serviceListResponse is the ARM service list envelope. nextLink is omitted: -// the emulator returns a single page. -type serviceListResponse struct { - Value []serviceResponse `json:"value"` +// listResponse is the ARM/APIM list envelope; nextLink continues a paged list. +type listResponse[T any] struct { + Value []T `json:"value"` + Count *int `json:"count,omitempty"` + NextLink string `json:"nextLink,omitempty"` +} + +// deletedServiceResponse is a soft-deleted service +// (Microsoft.ApiManagement/deletedservices). +type deletedServiceResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Location string `json:"location"` + Properties deletedServiceProperties `json:"properties"` +} + +type deletedServiceProperties struct { + ServiceID string `json:"serviceId"` + DeletionDate string `json:"deletionDate"` + ScheduledPurgeDate string `json:"scheduledPurgeDate"` +} + +// childResponse is a service child resource (api, product, policy, portal +// setting). +type childResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Properties json.RawMessage `json:"properties"` +} + +// childRequest is a child resource PUT/PATCH body. +type childRequest struct { + Properties json.RawMessage `json:"properties"` +} + +// policyRequest is the service policy PUT body. +type policyRequest struct { + Properties struct { + Value string `json:"value"` + Format string `json:"format"` + } `json:"properties"` +} + +// tenantAccessResponse is a tenant access entity as GET/PATCH return it (no +// keys). +type tenantAccessResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Properties tenantAccessProperties `json:"properties"` +} + +type tenantAccessProperties struct { + ID string `json:"id"` + PrincipalID string `json:"principalId"` + Enabled bool `json:"enabled"` +} + +// tenantAccessSecrets is the tenant access listSecrets body. +type tenantAccessSecrets struct { + ID string `json:"id"` + PrincipalID string `json:"principalId"` + PrimaryKey string `json:"primaryKey"` + SecondaryKey string `json:"secondaryKey"` + Enabled bool `json:"enabled"` +} + +// tenantAccessRequest is the tenant access PATCH body. +type tenantAccessRequest struct { + Properties struct { + Enabled *bool `json:"enabled"` + } `json:"properties"` +} + +// nameAvailabilityRequest / nameAvailabilityResponse are the +// checkNameAvailability body and verdict. +type nameAvailabilityRequest struct { + Name string `json:"name"` +} + +type nameAvailabilityResponse struct { + NameAvailable bool `json:"nameAvailable"` + Reason string `json:"reason"` + Message string `json:"message,omitempty"` } // serviceInputFromRequest builds a service create/update Input from a request -// body. -func serviceInputFromRequest(req *serviceRequest) apimanagement.ServiceInput { - in := apimanagement.ServiceInput{Tags: req.Tags, Zones: req.Zones, Properties: req.Properties} +// body and its If-Match header. +func serviceInputFromRequest(req *serviceRequest, ifMatch string) apimanagement.ServiceInput { + in := apimanagement.ServiceInput{Tags: req.Tags, Zones: req.Zones, Properties: req.Properties, IfMatch: ifMatch} if req.Sku != nil { if req.Sku.Name != "" { @@ -119,7 +186,8 @@ func userAssignedKeys(m map[string]json.RawMessage) []string { } // toServiceResponse projects a stored service onto the ARM wire -// representation, injecting the computed read-only properties. +// representation. The properties block is the provider's, verbatim: it already +// carries the defaults and computed fields. func toServiceResponse(s *apimanagement.Service) serviceResponse { return serviceResponse{ ID: s.ARMID(), @@ -131,80 +199,43 @@ func toServiceResponse(s *apimanagement.Service) serviceResponse { Sku: skuResponse{Name: s.SkuName, Capacity: s.SkuCapacity}, Identity: toIdentityWire(s.Identity), Etag: s.Etag, - Properties: responseProperties(s), + Properties: s.Properties, } } -// responseProperties overlays the computed read-only fields onto the stored, -// verbatim properties block, filling Azure's defaults for the few writable -// fields the caller left unset. -func responseProperties(s *apimanagement.Service) json.RawMessage { - obj := map[string]any{} - if len(s.Properties) > 0 { - if err := json.Unmarshal(s.Properties, &obj); err != nil { - obj = map[string]any{} - } - } - - for k, v := range map[string]any{ - "virtualNetworkType": "None", - "publicNetworkAccess": "Enabled", - "notificationSenderEmail": notificationSenderDefault, - "disableGateway": false, - } { - if _, set := obj[k]; !set { - obj[k] = v - } - } - - for k, v := range computedProperties(s) { - obj[k] = v - } - - raw, err := json.Marshal(obj) - if err != nil { - return s.Properties +// toDeletedServiceResponse projects a soft-deleted service onto the wire. +func toDeletedServiceResponse(d *apimanagement.DeletedService) deletedServiceResponse { + return deletedServiceResponse{ + ID: d.ARMID(), + Name: d.Service.Name, + Type: providerName + "/deletedservices", + Location: d.Service.Location, + Properties: deletedServiceProperties{ + ServiceID: d.Service.ARMID(), + DeletionDate: d.DeletionDate.UTC().Format(time.RFC3339), + ScheduledPurgeDate: d.ScheduledPurgeDate.UTC().Format(time.RFC3339), + }, } - - return raw } -// computedProperties returns the read-only properties Azure mints: the -// provisioning state, creation time, platform version and the endpoint URLs. -// The Consumption tier has only a gateway, so its portal, management and SCM -// endpoints are absent. -func computedProperties(s *apimanagement.Service) map[string]any { - out := map[string]any{ - "provisioningState": s.ProvisioningState, - "targetProvisioningState": "", - "createdAtUtc": s.CreatedAt.UTC().Format(time.RFC3339), - "gatewayUrl": s.Endpoints().Gateway, - "publicIPAddresses": []string{}, - "platformVersion": platformDedicated, - } - - if s.SkuName == skuConsumption { - out["platformVersion"] = platformConsumption - - return out +// toChildResponse projects a child resource under serviceID/segment. +func toChildResponse(serviceID, segment string, c *apimanagement.ChildResource) childResponse { + return childResponse{ + ID: serviceID + "/" + segment + "/" + c.Name, + Name: c.Name, + Type: serviceArmType + "/" + segment, + Properties: c.Properties, } - - ep := s.Endpoints() - out["gatewayRegionalUrl"] = regionalGatewayURL(s) - out["portalUrl"] = ep.Portal - out["developerPortalUrl"] = ep.DeveloperPortal - out["managementApiUrl"] = ep.ManagementAPI - out["scmUrl"] = ep.Scm - - return out } -// regionalGatewayURL renders the primary region's gateway endpoint, -// https://--01.regional.azure-api.net. -func regionalGatewayURL(s *apimanagement.Service) string { - region := strings.ToLower(strings.ReplaceAll(s.Location, " ", "")) - - return "https://" + strings.ToLower(s.Name) + "-" + region + "-01.regional.azure-api.net" +// toTenantAccessResponse projects a tenant access entity without its keys. +func toTenantAccessResponse(serviceID string, t *apimanagement.TenantAccess) tenantAccessResponse { + return tenantAccessResponse{ + ID: serviceID + "/tenant/" + t.Name, + Name: t.Name, + Type: serviceArmType + "/tenant", + Properties: tenantAccessProperties{ID: t.Name, PrincipalID: t.PrincipalID, Enabled: t.Enabled}, + } } // toIdentityWire projects a stored managed identity onto the wire block, diff --git a/server/azure/apimanagement/wire_errors_sdk_test.go b/server/azure/apimanagement/wire_errors_sdk_test.go new file mode 100644 index 000000000..f653c7eac --- /dev/null +++ b/server/azure/apimanagement/wire_errors_sdk_test.go @@ -0,0 +1,106 @@ +package apimanagement_test + +import ( + "net/http" + "strings" + "testing" +) + +// TestWireRoutesAndErrors walks the handler's routing and error mapping over +// raw ARM requests: unsupported paths and methods, missing parents, malformed +// bodies, and the deleted-services and checkNameAvailability endpoints. +func TestWireRoutesAndErrors(t *testing.T) { + f := newFixture(t) + f.create(t, rgName, svcName, developerService()) + + svc := f.serviceURL(rgName, svcName) + missing := f.serviceURL(rgName, "missing-svc") + subProv := f.ts.URL + "/subscriptions/" + subID + "/providers/Microsoft.ApiManagement" + + cases := []struct { + name, method, url, body string + hdr map[string]string + status int + contains string + }{ + {"service bad method", http.MethodPost, svc, "", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"list bad method", http.MethodPost, f.ts.URL + "/subscriptions/" + subID + "/resourceGroups/" + rgName + + "/providers/Microsoft.ApiManagement/service", "", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"subscription list", http.MethodGet, subProv + "/service", "", nil, http.StatusOK, svcName}, + {"malformed body", http.MethodPut, svc, "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"patch malformed body", http.MethodPatch, svc, "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"patch missing", http.MethodPatch, missing, "{}", nil, http.StatusNotFound, "ResourceNotFound"}, + {"delete missing", http.MethodDelete, missing, "", nil, http.StatusNoContent, ""}, + {"unsupported child", http.MethodGet, svc + "/backends", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"api operations", http.MethodGet, svc + "/apis/echo-api/operations", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"api bad method", http.MethodPut, svc + "/apis/echo-api", "{}", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"api head", http.MethodHead, svc + "/apis/echo-api", "", nil, http.StatusOK, ""}, + {"api missing", http.MethodGet, svc + "/apis/nope", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"apis of missing svc", http.MethodGet, missing + "/apis", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"api delete of missing svc", http.MethodDelete, missing + "/apis/x", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"product stale delete", http.MethodDelete, svc + "/products/starter", "", + map[string]string{"If-Match": `"old"`}, http.StatusPreconditionFailed, "PreconditionFailed"}, + {"product get", http.MethodGet, svc + "/products/unlimited", "", nil, http.StatusOK, `"approvalRequired":true`}, + {"policies list empty", http.MethodGet, svc + "/policies", "", nil, http.StatusOK, `"count":0`}, + {"policies list missing svc", http.MethodGet, missing + "/policies", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"policy bad name", http.MethodGet, svc + "/policies/other", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"policy bad method", http.MethodPatch, svc + "/policies/policy", "{}", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"policy malformed", http.MethodPut, svc + "/policies/policy", "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"policy put missing svc", http.MethodPut, missing + "/policies/policy", + `{"properties":{"value":""}}`, nil, http.StatusNotFound, "ResourceNotFound"}, + {"policy delete missing svc", http.MethodDelete, missing + "/policies/policy", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"policy delete none", http.MethodDelete, svc + "/policies/policy", "", nil, http.StatusNoContent, ""}, + {"portal deep path", http.MethodGet, svc + "/portalsettings/signin/x/y", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"portal bad method", http.MethodDelete, svc + "/portalsettings/signin", "", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"portal malformed", http.MethodPut, svc + "/portalsettings/signin", "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"portal put missing svc", http.MethodPut, missing + "/portalsettings/signin", `{"properties":{}}`, nil, + http.StatusNotFound, "ResourceNotFound"}, + {"portal get missing", http.MethodGet, svc + "/portalsettings/nope", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"portal head", http.MethodHead, svc + "/portalsettings/signup", "", nil, http.StatusOK, ""}, + {"delegation secrets missing svc", http.MethodPost, missing + "/portalsettings/delegation/listSecrets", "", nil, + http.StatusNotFound, "ResourceNotFound"}, + {"tenant bad method", http.MethodPut, svc + "/tenant/access", "{}", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"tenant deep path", http.MethodGet, svc + "/tenant/access/x/y", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"tenant get missing", http.MethodGet, svc + "/tenant/nope", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"tenant secrets missing", http.MethodPost, svc + "/tenant/nope/listSecrets", "", nil, http.StatusNotFound, "ResourceNotFound"}, + {"tenant patch malformed", http.MethodPatch, svc + "/tenant/access", "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"tenant patch stale", http.MethodPatch, svc + "/tenant/access", `{"properties":{"enabled":true}}`, + map[string]string{"If-Match": `"old"`}, http.StatusPreconditionFailed, "PreconditionFailed"}, + {"deleted list", http.MethodGet, subProv + "/deletedservices", "", nil, http.StatusOK, `"value":[]`}, + {"deleted list bad method", http.MethodPost, subProv + "/deletedservices", "", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"deleted no name", http.MethodGet, subProv + "/locations/eastus/deletedservices", "", nil, http.StatusNotFound, "InvalidResourceType"}, + {"deleted bad method", http.MethodPut, subProv + "/locations/eastus/deletedservices/x", "{}", nil, + http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"deleted purge missing", http.MethodDelete, subProv + "/locations/eastus/deletedservices/x", "", nil, + http.StatusNotFound, "ResourceNotFound"}, + {"check name bad method", http.MethodGet, subProv + "/checkNameAvailability", "", nil, http.StatusMethodNotAllowed, "MethodNotAllowed"}, + {"check name malformed", http.MethodPost, subProv + "/checkNameAvailability", "{", nil, http.StatusBadRequest, "InvalidRequestContent"}, + {"other locations type", http.MethodGet, subProv + "/locations/eastus/operationResults/x", "", nil, http.StatusNotImplemented, ""}, + {"no rg on PUT", http.MethodPut, subProv + "/service/x", "{}", nil, http.StatusBadRequest, "InvalidPath"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + code, body, _ := f.do(t, tc.method, tc.url+apiVersion, tc.body, tc.hdr) + if code != tc.status || !strings.Contains(body, tc.contains) { + t.Fatalf("%s %s = %d %s, want %d containing %q", tc.method, tc.url, code, body, tc.status, tc.contains) + } + }) + } +} + +// TestWireUserAssignedIdentity round-trips a user-assigned identity. +func TestWireUserAssignedIdentity(t *testing.T) { + f := newFixture(t) + + uai := "/subscriptions/" + subID + "/resourceGroups/" + rgName + + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1" + body := `{"location":"eastus","sku":{"name":"Developer","capacity":1},` + + `"identity":{"type":"UserAssigned","userAssignedIdentities":{"` + uai + `":{}}},` + + `"properties":{"publisherEmail":"a@b.test","publisherName":"n"}}` + + code, resp, _ := f.do(t, http.MethodPut, f.serviceURL(rgName, "uai-svc")+apiVersion, body, nil) + if code != http.StatusCreated || !strings.Contains(resp, "clientId") { + t.Fatalf("PUT = %d %s", code, resp) + } +} diff --git a/server/azure/azure.go b/server/azure/azure.go index bad0d963a..78b9f9ade 100644 --- a/server/azure/azure.go +++ b/server/azure/azure.go @@ -220,8 +220,9 @@ type Drivers struct { // IoTHub serves Microsoft.Devices/IotHubs plus its listkeys / // getKeysForKeyName actions and the nested event-hub consumer groups. IoTHub iothubsrv.Store - // APIManagement serves Microsoft.ApiManagement/service (the API Management - // service control plane). + // APIManagement serves Microsoft.ApiManagement: the service control plane, + // its soft-deleted services, checkNameAvailability and the child resources + // around a service's create, refresh and destroy. APIManagement apimanagementsrv.Store // SQLVirtualMachine serves Microsoft.SqlVirtualMachine/sqlVirtualMachines: // the SQL-management overlay on a compute VM. @@ -1127,7 +1128,8 @@ func New(d Drivers) http.Handler { srv.Register(iotHubHandler) } - // API Management claims Microsoft.ApiManagement/service: a distinct ARM + // API Management claims Microsoft.ApiManagement (service, deletedservices, + // locations/{l}/deletedservices, checkNameAvailability): a distinct ARM // provider name from every other Azure handler, so registration order is // unconstrained. if apiManagementHandler != nil { diff --git a/server/azure/echo_properties.go b/server/azure/echo_properties.go index b85a5658c..f12f4fc40 100644 --- a/server/azure/echo_properties.go +++ b/server/azure/echo_properties.go @@ -422,8 +422,9 @@ func captureUnmodeled( // (primaryKey/secondaryKey/accessKeys/connectionString) never enter the // request-capture path and are correctly left alone. // -// 2. Exact-match object keys: the Notification Hubs PNS credential blocks, which -// carry secrets but do not end in the suffixes. toHubJSON (notificationhubs) +// 2. Exact-match keys: the Notification Hubs PNS credential blocks, which +// carry secrets but do not end in the suffixes, and the API Management +// delegation validationKey (served only by its listSecrets action). toHubJSON (notificationhubs) // models only name/registrationTtl and drops these; real Azure serves them only // via GetPnsCredentials, never the generic hub GET. Each is an object, so // denylisting the key skips the whole credential subtree. @@ -456,7 +457,10 @@ func writeOnlyProperty(parent, key string) bool { switch lower { case "gcmcredential", "apnscredential", "wnscredential", - "admcredential", "baiducredential", "mpnscredential": + "admcredential", "baiducredential", "mpnscredential", + // API Management delegation settings: the validation key is accepted on + // PUT and served only by portalsettings/delegation/listSecrets. + "validationkey": return true default: return false diff --git a/server/wire/azurearm/paging.go b/server/wire/azurearm/paging.go new file mode 100644 index 000000000..0cb6979f9 --- /dev/null +++ b/server/wire/azurearm/paging.go @@ -0,0 +1,70 @@ +package azurearm + +import ( + "net/http" + "strconv" +) + +// DefaultPageSize is how many items a paged ARM list returns before it emits a +// nextLink. +const DefaultPageSize = 100 + +// Query parameters a paged ARM list reads: $skip resumes a listing at an +// offset, $top caps how many items one page holds. +const ( + skipParam = "$skip" + topParam = "$top" +) + +// Paginate returns the page of items a list request asks for and the nextLink +// that continues it. The page starts at the request's $skip offset and holds at +// most pageSize items (fewer when the request sets a smaller $top). When items +// remain, nextLink is an absolute URL that repeats the request (api-version, +// $top and any filter included) with $skip advanced; ARM SDK pagers GET it +// verbatim until it is empty, so it carries the scheme and host. A missing or +// malformed $skip/$top is ignored. +func Paginate[T any](r *http.Request, items []T, pageSize int) (page []T, nextLink string) { + if top := queryInt(r, topParam); top > 0 && top < pageSize { + pageSize = top + } + + skip := queryInt(r, skipParam) + if skip >= len(items) { + return []T{}, "" + } + + end := skip + pageSize + if end >= len(items) { + return items[skip:], "" + } + + return items[skip:end], nextPageLink(r, end) +} + +// queryInt reads a non-negative integer query parameter, 0 when it is missing +// or malformed. +func queryInt(r *http.Request, name string) int { + n, err := strconv.Atoi(r.URL.Query().Get(name)) + if err != nil || n < 0 { + return 0 + } + + return n +} + +// nextPageLink builds the absolute URL that continues a listing at offset skip. +func nextPageLink(r *http.Request, skip int) string { + next := *r.URL + next.Host = r.Host + + next.Scheme = "http" + if r.TLS != nil { + next.Scheme = "https" + } + + q := next.Query() + q.Set(skipParam, strconv.Itoa(skip)) + next.RawQuery = q.Encode() + + return next.String() +} diff --git a/server/wire/azurearm/paging_test.go b/server/wire/azurearm/paging_test.go new file mode 100644 index 000000000..9d4761455 --- /dev/null +++ b/server/wire/azurearm/paging_test.go @@ -0,0 +1,51 @@ +package azurearm_test + +import ( + "crypto/tls" + "net/http/httptest" + "testing" + + "github.com/stackshy/cloudemu/v2/server/wire/azurearm" +) + +func TestPaginate(t *testing.T) { + items := []int{0, 1, 2, 3, 4} + + cases := map[string]struct { + url string + tls bool + size int + want []int + wantNext string + }{ + "single page": {"/x?api-version=1", false, 10, []int{0, 1, 2, 3, 4}, ""}, + "first page": {"/x?api-version=1", false, 2, []int{0, 1}, "http://h/x?%24skip=2&api-version=1"}, + "middle page": {"/x?api-version=1&$skip=2", true, 2, []int{2, 3}, "https://h/x?%24skip=4&api-version=1"}, + "top narrows page": {"/x?$top=1", false, 100, []int{0}, "http://h/x?%24skip=1&%24top=1"}, + "last page": {"/x?$skip=4", false, 2, []int{4}, ""}, + "past the end": {"/x?$skip=9", false, 2, []int{}, ""}, + "bad skip ignored": {"/x?$skip=-3&$top=zz", false, 10, []int{0, 1, 2, 3, 4}, ""}, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + r := httptest.NewRequest("GET", tc.url, nil) + r.Host = "h" + + if tc.tls { + r.TLS = &tls.ConnectionState{} + } + + page, next := azurearm.Paginate(r, items, tc.size) + if len(page) != len(tc.want) || next != tc.wantNext { + t.Fatalf("page=%v next=%q, want %v %q", page, next, tc.want, tc.wantNext) + } + + for i := range page { + if page[i] != tc.want[i] { + t.Fatalf("page=%v, want %v", page, tc.want) + } + } + }) + } +} From 314e5ba804826bf19377a62735870c2958134cd3 Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 17:24:43 +0530 Subject: [PATCH 23/24] chore(contrib): tidy the contrib modules for the root dependency bump The root module now requires cloud.google.com/go/longrunning v1.2.0, iam v1.11.0, managedkafka and backupdr. contrib/server's go.mod still pinned longrunning v1.0.0, so its 'go build' stopped with 'updates to go.mod needed' (the Contrib (server) CI job). dockerengine and realengine only gain the matching go.sum lines. --- contrib/dockerengine/go.sum | 14 ++++++++++---- contrib/realengine/go.sum | 6 ++++++ contrib/server/go.mod | 2 +- contrib/server/go.sum | 10 ++++++++-- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/contrib/dockerengine/go.sum b/contrib/dockerengine/go.sum index a75dfc6f0..17dbccee7 100644 --- a/contrib/dockerengine/go.sum +++ b/contrib/dockerengine/go.sum @@ -8,6 +8,8 @@ cloud.google.com/go/auth v0.23.0 h1:6Gg1CMgpgubRG7DGz5Vf1pcoNo8RfiRiRAPS4crTp54= cloud.google.com/go/auth v0.23.0/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/backupdr v1.16.0 h1:K2xLA4ZKu+jmX8V81r2M5X0mXCMXnNMoNwAP1egogLA= +cloud.google.com/go/backupdr v1.16.0/go.mod h1:VHHb3pxFlkJ28RvyZAEbECG6Sq5JH9JfZIjZqdFTP+U= cloud.google.com/go/compute v1.60.0 h1:CqGt23ysz990ZZe1vq/9aDPKKnmwM6kcC7Y1Q05H2kI= cloud.google.com/go/compute v1.60.0/go.mod h1:Xm6PbsLgBpAg4va77ljbBdpMjzuU+uPp5Ze2dnZq7lw= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= @@ -16,10 +18,12 @@ cloud.google.com/go/eventarc v1.18.0 h1:8WWG1/ogInYur1NQjML6EMHQ0ZBzAdMDGlUVpLD5 cloud.google.com/go/eventarc v1.18.0/go.mod h1:/6SDoqh5+9QNUqCX4/oQcJVK16fG/snHBSXu7lrJtO8= cloud.google.com/go/firestore v1.22.0 h1:avooeboIq37vKXobrbPUFhFBxS/c3FqmWoX0xs8dO6E= cloud.google.com/go/firestore v1.22.0/go.mod h1:PaM4i7i7ruALSKmlpHXXZaPObcZw0W7ie5UOPr72iTU= -cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= -cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= -cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= -cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/managedkafka v1.0.0 h1:wKZdHCPUr3mQDo/9sbPI2+vhXXw9AgC9EfSsZnxbUvY= +cloud.google.com/go/managedkafka v1.0.0/go.mod h1:Phh9R6RQhGCNq36Uifsi+HHFIOlRFUE8Z3Xl2fYelBo= cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= @@ -46,6 +50,8 @@ github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 h1:j0hhYS006eJ54vusoa github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1/go.mod h1:AdtInaXmK8eYmbjezRWgLz+Qs46nc9Up9GWGwteWNfw= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0 h1:5aCs3yc/Ftlv/Le+Dr0P+oOuo/clB6fsfFwrm+DbNms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0/go.mod h1:4/JvtEOgU0r2lkVz8BImrFd0e7ZNQ74x+3k9swJzSmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 h1:1PD0CnFSl1m1TCwudP3cIiyTABCWVzHXtYc6Vi5J0JY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0/go.mod h1:xCGT95xV5ei4ahSgJWy31pPGE3xWfaWpr9uRzwTzsmg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v3 v3.0.0 h1:FErSe/vQGefbSuVwBV9JlrRXgG1uOFyW6TCXERX89s4= diff --git a/contrib/realengine/go.sum b/contrib/realengine/go.sum index 07237f5e8..69ecce4de 100644 --- a/contrib/realengine/go.sum +++ b/contrib/realengine/go.sum @@ -8,6 +8,8 @@ cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/backupdr v1.16.0 h1:K2xLA4ZKu+jmX8V81r2M5X0mXCMXnNMoNwAP1egogLA= +cloud.google.com/go/backupdr v1.16.0/go.mod h1:VHHb3pxFlkJ28RvyZAEbECG6Sq5JH9JfZIjZqdFTP+U= cloud.google.com/go/compute v1.63.0 h1:KsBourH0wajM4RhzwPwRMKbxHVdvzGsk7StvACoWXD8= cloud.google.com/go/compute v1.63.0/go.mod h1:Xm6PbsLgBpAg4va77ljbBdpMjzuU+uPp5Ze2dnZq7lw= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= @@ -22,6 +24,8 @@ cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9 cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/managedkafka v1.0.0 h1:wKZdHCPUr3mQDo/9sbPI2+vhXXw9AgC9EfSsZnxbUvY= +cloud.google.com/go/managedkafka v1.0.0/go.mod h1:Phh9R6RQhGCNq36Uifsi+HHFIOlRFUE8Z3Xl2fYelBo= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= @@ -48,6 +52,8 @@ github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 h1:j0hhYS006eJ54vusoa github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1/go.mod h1:AdtInaXmK8eYmbjezRWgLz+Qs46nc9Up9GWGwteWNfw= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0 h1:5aCs3yc/Ftlv/Le+Dr0P+oOuo/clB6fsfFwrm+DbNms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0/go.mod h1:4/JvtEOgU0r2lkVz8BImrFd0e7ZNQ74x+3k9swJzSmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 h1:1PD0CnFSl1m1TCwudP3cIiyTABCWVzHXtYc6Vi5J0JY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0/go.mod h1:xCGT95xV5ei4ahSgJWy31pPGE3xWfaWpr9uRzwTzsmg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v3 v3.0.0 h1:FErSe/vQGefbSuVwBV9JlrRXgG1uOFyW6TCXERX89s4= diff --git a/contrib/server/go.mod b/contrib/server/go.mod index 60e113d7c..444caf235 100644 --- a/contrib/server/go.mod +++ b/contrib/server/go.mod @@ -27,7 +27,7 @@ require ( cloud.google.com/go/bigtable v1.47.0 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.11.0 // indirect - cloud.google.com/go/longrunning v1.0.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/alicebob/miniredis/v2 v2.38.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect diff --git a/contrib/server/go.sum b/contrib/server/go.sum index 93e974f1b..578fe54eb 100644 --- a/contrib/server/go.sum +++ b/contrib/server/go.sum @@ -8,6 +8,8 @@ cloud.google.com/go/auth v0.23.0 h1:6Gg1CMgpgubRG7DGz5Vf1pcoNo8RfiRiRAPS4crTp54= cloud.google.com/go/auth v0.23.0/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/backupdr v1.16.0 h1:K2xLA4ZKu+jmX8V81r2M5X0mXCMXnNMoNwAP1egogLA= +cloud.google.com/go/backupdr v1.16.0/go.mod h1:VHHb3pxFlkJ28RvyZAEbECG6Sq5JH9JfZIjZqdFTP+U= cloud.google.com/go/bigtable v1.47.0 h1:NGLgDSr/i79BTGCjxH/maPKxyvl5q8/SsBsyLK52kdI= cloud.google.com/go/bigtable v1.47.0/go.mod h1:GUM6PdkG3rrDse9kugqvX5+ktwo3ldfLtLi1VFn5Wj4= cloud.google.com/go/compute v1.63.0 h1:KsBourH0wajM4RhzwPwRMKbxHVdvzGsk7StvACoWXD8= @@ -20,8 +22,10 @@ cloud.google.com/go/firestore v1.22.0 h1:avooeboIq37vKXobrbPUFhFBxS/c3FqmWoX0xs8 cloud.google.com/go/firestore v1.22.0/go.mod h1:PaM4i7i7ruALSKmlpHXXZaPObcZw0W7ie5UOPr72iTU= cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY= -cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/managedkafka v1.0.0 h1:wKZdHCPUr3mQDo/9sbPI2+vhXXw9AgC9EfSsZnxbUvY= +cloud.google.com/go/managedkafka v1.0.0/go.mod h1:Phh9R6RQhGCNq36Uifsi+HHFIOlRFUE8Z3Xl2fYelBo= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= @@ -48,6 +52,8 @@ github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 h1:j0hhYS006eJ54vusoa github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1/go.mod h1:AdtInaXmK8eYmbjezRWgLz+Qs46nc9Up9GWGwteWNfw= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0 h1:5aCs3yc/Ftlv/Le+Dr0P+oOuo/clB6fsfFwrm+DbNms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement/v3 v3.0.0/go.mod h1:4/JvtEOgU0r2lkVz8BImrFd0e7ZNQ74x+3k9swJzSmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 h1:1PD0CnFSl1m1TCwudP3cIiyTABCWVzHXtYc6Vi5J0JY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0/go.mod h1:xCGT95xV5ei4ahSgJWy31pPGE3xWfaWpr9uRzwTzsmg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v3 v3.0.0 h1:FErSe/vQGefbSuVwBV9JlrRXgG1uOFyW6TCXERX89s4= From 987019ed822c8ea4f7a0be5b0004deab5af1223a Mon Sep 17 00:00:00 2001 From: aryanmehrotra Date: Sun, 27 Sep 2026 17:24:43 +0530 Subject: [PATCH 24/24] fix(gcp-lb): size the resource map from the body alone gcpResourceJSON sized its output map as len(res.Body)+internalFieldCount. CodeQL flags that addition as a size computation that may overflow (go/allocation-size-overflow). The hint only needs to be approximate; the map grows for the few server-injected members, so the constant is gone. --- server/gcp/loadbalancer/resources.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/gcp/loadbalancer/resources.go b/server/gcp/loadbalancer/resources.go index 2523611ec..498bc5a2a 100644 --- a/server/gcp/loadbalancer/resources.go +++ b/server/gcp/loadbalancer/resources.go @@ -306,7 +306,9 @@ func (h *Handler) deleteGCPResource(w http.ResponseWriter, r *http.Request, rp g // //nolint:gocritic // rp is a request-scoped value func gcpResourceJSON(res *lbdriver.GCPResource, rp gcprest.ResourcePath, host string) map[string]any { - out := make(map[string]any, len(res.Body)+internalFieldCount) + // Size hint from the body alone: adding to a caller-sized length is an + // unchecked addition, and the map grows for the few server-injected members. + out := make(map[string]any, len(res.Body)) for k, v := range res.Body { // Reserved internal members (e.g. instance-group membership) are stored in @@ -340,11 +342,6 @@ func gcpResourceJSON(res *lbdriver.GCPResource, rp gcprest.ResourcePath, host st return out } -// internalFieldCount is the number of server-injected members gcpResourceJSON -// adds on top of the stored body (kind, id, name, creationTimestamp, selfLink, -// region/zone, size). -const internalFieldCount = 7 - // healthCheckInUse returns the name of a same-scope backend service whose // healthChecks[] references the health check being deleted, or "" when none // does.